admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update FILE_ARCHIVER package to version 3.2.0 with TRASH retention control and new statuses

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-10 08:23:06 +01:00
parent cdd9dff32d
commit d443c4f07d
10 changed files with 257 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
View File

@@ -1,14 +1,17 @@
-- ===================================================================
-- MARS-828: Install FILE_ARCHIVER Package Specification v3.0.0
-- MARS-828: Install FILE_ARCHIVER Package Specification v3.2.0
-- ===================================================================
-- Purpose: Deploy updated package specification with version 3.0.0
-- Purpose: Deploy updated package specification with version 3.2.0
-- Author: Grzegorz Michalski
-- Date: 2026-01-27
-- Date: 2026-02-06
-- Changes:
-- - Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA
-- - TRASH folder retention control for safety and compliance
-- ===================================================================
@@new_version/FILE_ARCHIVER.pkg
PROMPT ========================================
PROMPT FILE_ARCHIVER Specification v3.0.0 ready for installation
PROMPT FILE_ARCHIVER Specification v3.2.0 installed successfully
PROMPT ========================================

14
MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
View File

@@ -1,13 +1,15 @@
-- ===================================================================
-- MARS-828: Install FILE_ARCHIVER Package Body v3.0.0
-- MARS-828: Install FILE_ARCHIVER Package Body v3.2.0
-- ===================================================================
-- Purpose: Deploy updated package body with GET_ARCHIVAL_WHERE_CLAUSE function
-- Purpose: Deploy updated package body with TRASH folder retention control
-- Author: Grzegorz Michalski
-- Date: 2026-01-27
-- Date: 2026-02-06
-- Changes:
-- - Added GET_ARCHIVAL_WHERE_CLAUSE private function
-- - Updated ARCHIVE_TABLE_DATA to use strategy-based filtering
-- - Updated GATHER_TABLE_STAT to use strategy-based statistics
-- - v3.0.0: Added GET_ARCHIVAL_WHERE_CLAUSE for flexible archival strategies
-- - v3.1.0: Added function overloads for Python integration
-- - v3.1.1: Fixed ORA-01422 for multiple parquet files (directory prefix storage)
-- - v3.1.2: Fixed PARTITION_YEAR/PARTITION_MONTH assignments and circular dependency
-- - v3.2.0: Added pKeepInTrash parameter for TRASH folder retention control
-- ===================================================================
@@new_version/FILE_ARCHIVER.pkb

45
MARS_Packages/REL01_ADDITIONS/MARS-828/07_MARS_828_install_add_trash_retention_statuses.sql Normal file
View File

@@ -0,0 +1,45 @@
-- MARS-828: Add TRASH retention statuses to A_SOURCE_FILE_RECEIVED
-- Author: Grzegorz Michalski
-- Date: 2026-02-09
-- Description: Adds ARCHIVED_AND_TRASHED and ARCHIVED_AND_PURGED statuses to support TRASH retention feature
PROMPT ========================================
PROMPT MARS-828: Adding TRASH retention statuses
PROMPT ========================================
-- Drop old constraint
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK;
-- Add new constraint with TRASH retention statuses
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK
CHECK (PROCESSING_STATUS IN (
'RECEIVED',
'VALIDATED',
'READY_FOR_INGESTION',
'INGESTED',
'ARCHIVED', -- Legacy status (backward compatibility)
'ARCHIVED_AND_TRASHED', -- Files archived to Parquet and kept in TRASH folder (DATA bucket subfolder)
'ARCHIVED_AND_PURGED' -- Files archived to Parquet and deleted from TRASH folder
));
-- Add comment
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)';
-- Verify constraint
SELECT constraint_name, constraint_type, search_condition
FROM user_constraints
WHERE table_name = 'A_SOURCE_FILE_RECEIVED'
AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK';
-- Show current status distribution
SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
GROUP BY PROCESSING_STATUS
ORDER BY PROCESSING_STATUS;
PROMPT ========================================
PROMPT TRASH retention statuses added successfully
PROMPT Status flow: INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
PROMPT Legacy ARCHIVED status maintained for backward compatibility
PROMPT ========================================

77
MARS_Packages/REL01_ADDITIONS/MARS-828/90_MARS_828_rollback_trash_retention_statuses.sql Normal file
View File

@@ -0,0 +1,77 @@
-- MARS-828 ROLLBACK: Remove TRASH retention statuses from A_SOURCE_FILE_RECEIVED
-- Author: Grzegorz Michalski
-- Date: 2026-02-09
-- Description: Rollback TRASH retention statuses to original constraint
PROMPT ========================================
PROMPT MARS-828 ROLLBACK: Removing TRASH retention statuses
PROMPT ========================================
-- Check if any files have new statuses
DECLARE
vTrashCount NUMBER;
vPurgedCount NUMBER;
BEGIN
SELECT COUNT(*) INTO vTrashCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
SELECT COUNT(*) INTO vPurgedCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_PURGED';
DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_TRASHED status: ' || vTrashCount);
DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_PURGED status: ' || vPurgedCount);
IF vTrashCount > 0 OR vPurgedCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Files exist with new statuses!');
DBMS_OUTPUT.PUT_LINE('Migrating statuses to ARCHIVED before rollback...');
-- Migrate new statuses back to ARCHIVED
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'ARCHIVED'
WHERE PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
COMMIT;
DBMS_OUTPUT.PUT_LINE('Migrated ' || SQL%ROWCOUNT || ' records to ARCHIVED status');
ELSE
DBMS_OUTPUT.PUT_LINE('No files with new statuses - safe to rollback');
END IF;
END;
/
-- Drop new constraint
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK;
-- Restore original constraint
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK
CHECK (PROCESSING_STATUS IN (
'RECEIVED',
'VALIDATED',
'READY_FOR_INGESTION',
'INGESTED',
'ARCHIVED'
));
-- Remove comment
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED';
-- Verify constraint restored
SELECT constraint_name, constraint_type, search_condition
FROM user_constraints
WHERE table_name = 'A_SOURCE_FILE_RECEIVED'
AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK';
-- Show current status distribution
SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
GROUP BY PROCESSING_STATUS
ORDER BY PROCESSING_STATUS;
PROMPT ========================================
PROMPT TRASH retention statuses rollback completed
PROMPT Original constraint restored
PROMPT All ARCHIVED_AND_TRASHED/ARCHIVED_AND_PURGED migrated to ARCHIVED
PROMPT ========================================

38
MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql
View File

@@ -35,9 +35,9 @@ PROMPT =========================================================================
PROMPT MARS-828 Installation Starting
PROMPT ============================================================================
PROMPT Package: CT_MRDS.FILE_ARCHIVER
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID)
PROMPT Purpose: Flexible archival policies per data source
PROMPT Steps: 8 (DDL, Trigger, Package v3.1.2, Verify, Track, Configure)
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention control
PROMPT Purpose: Flexible archival policies per data source with file retention management
PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.2.0, Verify, Track, Configure)
PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
PROMPT ============================================================================
@@ -55,42 +55,47 @@ WHENEVER SQLERROR CONTINUE
-- Installation steps
PROMPT
PROMPT Step 1/8: Adding archival strategy columns to A_SOURCE_FILE_CONFIG
PROMPT Step 1/9: Adding archival strategy columns to A_SOURCE_FILE_CONFIG
PROMPT ===================================================================
@@01_MARS_828_install_add_archival_strategy_columns.sql
PROMPT
PROMPT Step 2/8: Creating validation trigger
PROMPT Step 2/9: Creating validation trigger
PROMPT ======================================
@@02_MARS_828_install_archival_strategy_trigger.sql
PROMPT
PROMPT Step 3/8: Deploying FILE_ARCHIVER Package Specification v3.1.2
PROMPT Step 3/9: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED
PROMPT =====================================================================
@@07_MARS_828_install_add_trash_retention_statuses.sql
PROMPT
PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.2.0
PROMPT ================================================================
@@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
PROMPT
PROMPT Step 4/8: Deploying FILE_ARCHIVER Package Body v3.1.2
PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.2.0
PROMPT ======================================================
@@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
PROMPT
PROMPT Step 5/8: Verifying installation
PROMPT Step 6/9: Verifying installation
PROMPT =================================
@@05_MARS_828_verify_installation.sql
PROMPT
PROMPT Step 6/8: Tracking package versions
PROMPT Step 7/9: Tracking package versions
PROMPT ====================================
@@track_package_versions.sql
PROMPT
PROMPT Step 7/8: Verifying tracked packages
PROMPT Step 8/9: Verifying tracked packages
PROMPT =====================================
@@verify_packages_version.sql
PROMPT
PROMPT Step 8/8: Configuring Release 01 tables archival strategies
PROMPT Step 9/9: Configuring Release 01 tables archival strategies
PROMPT ============================================================
@@06_MARS_828_configure_release01_tables.sql
@@ -103,16 +108,21 @@ SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL;
PROMPT
PROMPT Installation Summary:
PROMPT - Package: CT_MRDS.FILE_ARCHIVER
PROMPT - Version: 3.1.2 (final - includes all fixes)
PROMPT - Version: 3.2.0 (includes TRASH folder retention control)
PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID
PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED preserved)
PROMPT - TRASH Retention: pKeepInTrash parameter (DEFAULT TRUE for safety)
PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
PROMPT * Files kept in TRASH by default for compliance and rollback capability
PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED
PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED + TRASH retention preserved)
PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB)
PROMPT - Includes Fixes:
PROMPT * v3.1.1: ORA-01422 for multiple parquet files
PROMPT * v3.1.2: PARTITION_YEAR/PARTITION_MONTH assignments
PROMPT * v3.1.2: Export query circular dependency
PROMPT * v3.2.0: TRASH retention control (pKeepInTrash parameter)
PROMPT
PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2) available in patches/
PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2->v3.2.0) available in patches/
PROMPT
PROMPT Log file: &_filename
PROMPT ============================================================================

29
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql Normal file
View File

@@ -0,0 +1,29 @@
-- ====================================================================
-- A_SOURCE_FILE_RECEIVED Table
-- ====================================================================
-- Purpose: Track received files and their processing status
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE,
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL,
CHECKSUM VARCHAR2(128),
CREATED TIMESTAMP(6) WITH TIME ZONE,
BYTES NUMBER,
RECEPTION_DATE DATE NOT NULL,
PROCESSING_STATUS VARCHAR2(200),
EXTERNAL_TABLE_NAME VARCHAR2(200),
PARTITION_YEAR VARCHAR2(4),
PARTITION_MONTH VARCHAR2(2),
ARCH_FILE_NAME VARCHAR2(1000),
CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY),
CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED'))
) TABLESPACE "DATA";
-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)
CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1
ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES);
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE;

64
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb
View File

@@ -87,7 +87,8 @@ AS
----------------------------------------------------------------------------------------------------
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
@@ -104,7 +105,10 @@ AS
vProcessControlStatus VARCHAR2(60) := 'OK';
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
@@ -244,17 +248,17 @@ AS
END;
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
-- move file to trash before dropping
-- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => f.pathname||'/'||f.filename,
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
rollback;
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
@@ -264,24 +268,38 @@ AS
END LOOP;
--------------------------------------------------------------------
-- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
-- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART)
-- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME)
IF vProcessControlStatus = 'OK' THEN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
--Drop file from TRASH
DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
END LOOP;
IF NOT pKeepInTrash THEN
-- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
-- Update status to ARCHIVED_AND_PURGED
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
WHERE r.a_source_file_config_key = pSourceFileConfigKey
AND r.source_file_name = f.filename
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO');
ELSE
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO');
END IF;
--ROLLBACK PART
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket)
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
FOR f in ( SELECT vf.filename, vf.pathname
FROM TABLE(vfiles) vf
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
ON r.source_file_name = vf.filename
AND r.a_source_file_config_key = pSourceFileConfigKey
AND r.PROCESSING_STATUS = 'ARCHIVED'
AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')
AND vf.year = ym_loop.year
AND vf.month = ym_loop.month
) LOOP
@@ -291,7 +309,7 @@ AS
target_object_uri => f.pathname||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'INGESTED'
@@ -302,7 +320,7 @@ AS
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
END;
@@ -363,9 +381,7 @@ AS
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
END;
-- END of "Try to drop EXPORTED FILES"
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO');
-- END of "Try to drop EXPORTED FILES"
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
END LOOP; --ym_loop end (YEAR_MONTH)
@@ -523,15 +539,19 @@ AS
----------------------------------------------------------------------------------------------------
FUNCTION ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey);
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;

13
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg
View File

@@ -17,12 +17,13 @@ AS
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.1.2';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-06 11:30:00';
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-06 14:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
'3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) ||
'3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) ||
@@ -38,9 +39,11 @@ AS
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
**/
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
);
/**
@@ -48,11 +51,13 @@ AS
* @desc Function overload for ARCHIVE_TABLE_DATA procedure.
* Returns SQLCODE for Python library integration.
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
* @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER;

24
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql
View File

@@ -32,10 +32,11 @@ PROMPT =========================================================================
PROMPT This will restore FILE_ARCHIVER to v2.0.0
PROMPT
PROMPT Rollback steps:
PROMPT 1. Remove validation trigger
PROMPT 2. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns
PROMPT 3. Restore FILE_ARCHIVER package to v2.0.0
PROMPT 4. Revert all archival strategies to THRESHOLD_BASED
PROMPT 1. Rollback TRASH retention statuses
PROMPT 2. Remove validation trigger
PROMPT 3. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns
PROMPT 4. Restore FILE_ARCHIVER package to v2.0.0
PROMPT 5. Revert all archival strategies to THRESHOLD_BASED
PROMPT
PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL;
@@ -54,27 +55,32 @@ WHENEVER SQLERROR CONTINUE
-- Rollback steps (in reverse order)
PROMPT
PROMPT Step 1/5: Dropping validation trigger
PROMPT Step 1/6: Rolling back TRASH retention statuses
PROMPT ================================================
@@90_MARS_828_rollback_trash_retention_statuses.sql
PROMPT
PROMPT Step 2/6: Dropping validation trigger
PROMPT ======================================
@@93_MARS_828_rollback_trigger.sql
PROMPT
PROMPT Step 2/5: Dropping archival strategy columns
PROMPT Step 3/6: Dropping archival strategy columns
PROMPT =============================================
@@94_MARS_828_rollback_columns.sql
PROMPT
PROMPT Step 3/5: Restoring FILE_ARCHIVER Package Specification v2.0.0
PROMPT Step 4/6: Restoring FILE_ARCHIVER Package Specification v2.0.0
PROMPT ===============================================================
@@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql
PROMPT
PROMPT Step 4/5: Restoring FILE_ARCHIVER Package Body v2.0.0
PROMPT Step 5/6: Restoring FILE_ARCHIVER Package Body v2.0.0
PROMPT ======================================================
@@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql
PROMPT
PROMPT Step 5/5: Verifying tracked packages
PROMPT Step 6/6: Verifying tracked packages
PROMPT =====================================
@@verify_packages_version.sql

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql
View File

@@ -19,7 +19,7 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
ARCH_FILE_NAME VARCHAR2(1000),
CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY),
CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED'))
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED'))
) TABLESPACE "DATA";
-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)