From d443c4f07df8562e7bab6b59615d434a074342ac Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Tue, 10 Feb 2026 08:23:06 +0100 Subject: [PATCH] Update FILE_ARCHIVER package to version 3.2.0 with TRASH retention control and new statuses --- ...828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql | 11 ++- ...828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql | 14 ++-- ...8_install_add_trash_retention_statuses.sql | 45 +++++++++++ ..._828_rollback_trash_retention_statuses.sql | 77 +++++++++++++++++++ .../MARS-828/install_mars828.sql | 38 +++++---- .../new_version/A_SOURCE_FILE_RECEIVED.sql | 29 +++++++ .../MARS-828/new_version/FILE_ARCHIVER.pkb | 64 +++++++++------ .../MARS-828/new_version/FILE_ARCHIVER.pkg | 13 +++- .../MARS-828/rollback_mars828.sql | 24 +++--- .../SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql | 2 +- 10 files changed, 257 insertions(+), 60 deletions(-) create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-828/07_MARS_828_install_add_trash_retention_statuses.sql create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-828/90_MARS_828_rollback_trash_retention_statuses.sql create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql index 9ff3c4c..edfab99 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql @@ -1,14 +1,17 @@ -- =================================================================== --- MARS-828: Install FILE_ARCHIVER Package Specification v3.0.0 +-- MARS-828: Install FILE_ARCHIVER Package Specification v3.2.0 -- =================================================================== --- Purpose: Deploy updated package specification with version 3.0.0 +-- Purpose: Deploy updated package specification with version 3.2.0 -- Author: Grzegorz Michalski --- Date: 2026-01-27 +-- Date: 2026-02-06 +-- Changes: +-- - Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA +-- - TRASH folder retention control for safety and compliance -- =================================================================== @@new_version/FILE_ARCHIVER.pkg PROMPT ======================================== -PROMPT FILE_ARCHIVER Specification v3.0.0 ready for installation +PROMPT FILE_ARCHIVER Specification v3.2.0 installed successfully PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql index d7d8792..78a5fd7 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@ -1,13 +1,15 @@ -- =================================================================== --- MARS-828: Install FILE_ARCHIVER Package Body v3.0.0 +-- MARS-828: Install FILE_ARCHIVER Package Body v3.2.0 -- =================================================================== --- Purpose: Deploy updated package body with GET_ARCHIVAL_WHERE_CLAUSE function +-- Purpose: Deploy updated package body with TRASH folder retention control -- Author: Grzegorz Michalski --- Date: 2026-01-27 +-- Date: 2026-02-06 -- Changes: --- - Added GET_ARCHIVAL_WHERE_CLAUSE private function --- - Updated ARCHIVE_TABLE_DATA to use strategy-based filtering --- - Updated GATHER_TABLE_STAT to use strategy-based statistics +-- - v3.0.0: Added GET_ARCHIVAL_WHERE_CLAUSE for flexible archival strategies +-- - v3.1.0: Added function overloads for Python integration +-- - v3.1.1: Fixed ORA-01422 for multiple parquet files (directory prefix storage) +-- - v3.1.2: Fixed PARTITION_YEAR/PARTITION_MONTH assignments and circular dependency +-- - v3.2.0: Added pKeepInTrash parameter for TRASH folder retention control -- =================================================================== @@new_version/FILE_ARCHIVER.pkb diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/07_MARS_828_install_add_trash_retention_statuses.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/07_MARS_828_install_add_trash_retention_statuses.sql new file mode 100644 index 0000000..368450d --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/07_MARS_828_install_add_trash_retention_statuses.sql @@ -0,0 +1,45 @@ +-- MARS-828: Add TRASH retention statuses to A_SOURCE_FILE_RECEIVED +-- Author: Grzegorz Michalski +-- Date: 2026-02-09 +-- Description: Adds ARCHIVED_AND_TRASHED and ARCHIVED_AND_PURGED statuses to support TRASH retention feature + +PROMPT ======================================== +PROMPT MARS-828: Adding TRASH retention statuses +PROMPT ======================================== + +-- Drop old constraint +ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK; + +-- Add new constraint with TRASH retention statuses +ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK + CHECK (PROCESSING_STATUS IN ( + 'RECEIVED', + 'VALIDATED', + 'READY_FOR_INGESTION', + 'INGESTED', + 'ARCHIVED', -- Legacy status (backward compatibility) + 'ARCHIVED_AND_TRASHED', -- Files archived to Parquet and kept in TRASH folder (DATA bucket subfolder) + 'ARCHIVED_AND_PURGED' -- Files archived to Parquet and deleted from TRASH folder + )); + +-- Add comment +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)'; + +-- Verify constraint +SELECT constraint_name, constraint_type, search_condition +FROM user_constraints +WHERE table_name = 'A_SOURCE_FILE_RECEIVED' + AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK'; + +-- Show current status distribution +SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +GROUP BY PROCESSING_STATUS +ORDER BY PROCESSING_STATUS; + +PROMPT ======================================== +PROMPT TRASH retention statuses added successfully +PROMPT Status flow: INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional) +PROMPT Legacy ARCHIVED status maintained for backward compatibility +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/90_MARS_828_rollback_trash_retention_statuses.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/90_MARS_828_rollback_trash_retention_statuses.sql new file mode 100644 index 0000000..588184c --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/90_MARS_828_rollback_trash_retention_statuses.sql @@ -0,0 +1,77 @@ +-- MARS-828 ROLLBACK: Remove TRASH retention statuses from A_SOURCE_FILE_RECEIVED +-- Author: Grzegorz Michalski +-- Date: 2026-02-09 +-- Description: Rollback TRASH retention statuses to original constraint + +PROMPT ======================================== +PROMPT MARS-828 ROLLBACK: Removing TRASH retention statuses +PROMPT ======================================== + +-- Check if any files have new statuses +DECLARE + vTrashCount NUMBER; + vPurgedCount NUMBER; +BEGIN + SELECT COUNT(*) INTO vTrashCount + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + SELECT COUNT(*) INTO vPurgedCount + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'; + + DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_TRASHED status: ' || vTrashCount); + DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_PURGED status: ' || vPurgedCount); + + IF vTrashCount > 0 OR vPurgedCount > 0 THEN + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('WARNING: Files exist with new statuses!'); + DBMS_OUTPUT.PUT_LINE('Migrating statuses to ARCHIVED before rollback...'); + + -- Migrate new statuses back to ARCHIVED + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED' + WHERE PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED'); + + COMMIT; + DBMS_OUTPUT.PUT_LINE('Migrated ' || SQL%ROWCOUNT || ' records to ARCHIVED status'); + ELSE + DBMS_OUTPUT.PUT_LINE('No files with new statuses - safe to rollback'); + END IF; +END; +/ + +-- Drop new constraint +ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK; + +-- Restore original constraint +ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK + CHECK (PROCESSING_STATUS IN ( + 'RECEIVED', + 'VALIDATED', + 'READY_FOR_INGESTION', + 'INGESTED', + 'ARCHIVED' + )); + +-- Remove comment +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED'; + +-- Verify constraint restored +SELECT constraint_name, constraint_type, search_condition +FROM user_constraints +WHERE table_name = 'A_SOURCE_FILE_RECEIVED' + AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK'; + +-- Show current status distribution +SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +GROUP BY PROCESSING_STATUS +ORDER BY PROCESSING_STATUS; + +PROMPT ======================================== +PROMPT TRASH retention statuses rollback completed +PROMPT Original constraint restored +PROMPT All ARCHIVED_AND_TRASHED/ARCHIVED_AND_PURGED migrated to ARCHIVED +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql index 7105116..e0b8ab1 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql @@ -35,9 +35,9 @@ PROMPT ========================================================================= PROMPT MARS-828 Installation Starting PROMPT ============================================================================ PROMPT Package: CT_MRDS.FILE_ARCHIVER -PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) -PROMPT Purpose: Flexible archival policies per data source -PROMPT Steps: 8 (DDL, Trigger, Package v3.1.2, Verify, Track, Configure) +PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention control +PROMPT Purpose: Flexible archival policies per data source with file retention management +PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.2.0, Verify, Track, Configure) PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; PROMPT ============================================================================ @@ -55,42 +55,47 @@ WHENEVER SQLERROR CONTINUE -- Installation steps PROMPT -PROMPT Step 1/8: Adding archival strategy columns to A_SOURCE_FILE_CONFIG +PROMPT Step 1/9: Adding archival strategy columns to A_SOURCE_FILE_CONFIG PROMPT =================================================================== @@01_MARS_828_install_add_archival_strategy_columns.sql PROMPT -PROMPT Step 2/8: Creating validation trigger +PROMPT Step 2/9: Creating validation trigger PROMPT ====================================== @@02_MARS_828_install_archival_strategy_trigger.sql PROMPT -PROMPT Step 3/8: Deploying FILE_ARCHIVER Package Specification v3.1.2 +PROMPT Step 3/9: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED +PROMPT ===================================================================== +@@07_MARS_828_install_add_trash_retention_statuses.sql + +PROMPT +PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.2.0 PROMPT ================================================================ @@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 4/8: Deploying FILE_ARCHIVER Package Body v3.1.2 +PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.2.0 PROMPT ====================================================== @@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql PROMPT -PROMPT Step 5/8: Verifying installation +PROMPT Step 6/9: Verifying installation PROMPT ================================= @@05_MARS_828_verify_installation.sql PROMPT -PROMPT Step 6/8: Tracking package versions +PROMPT Step 7/9: Tracking package versions PROMPT ==================================== @@track_package_versions.sql PROMPT -PROMPT Step 7/8: Verifying tracked packages +PROMPT Step 8/9: Verifying tracked packages PROMPT ===================================== @@verify_packages_version.sql PROMPT -PROMPT Step 8/8: Configuring Release 01 tables archival strategies +PROMPT Step 9/9: Configuring Release 01 tables archival strategies PROMPT ============================================================ @@06_MARS_828_configure_release01_tables.sql @@ -103,16 +108,21 @@ SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL; PROMPT PROMPT Installation Summary: PROMPT - Package: CT_MRDS.FILE_ARCHIVER -PROMPT - Version: 3.1.2 (final - includes all fixes) +PROMPT - Version: 3.2.0 (includes TRASH folder retention control) PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID -PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED preserved) +PROMPT - TRASH Retention: pKeepInTrash parameter (DEFAULT TRUE for safety) +PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME) +PROMPT * Files kept in TRASH by default for compliance and rollback capability +PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED +PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED + TRASH retention preserved) PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB) PROMPT - Includes Fixes: PROMPT * v3.1.1: ORA-01422 for multiple parquet files PROMPT * v3.1.2: PARTITION_YEAR/PARTITION_MONTH assignments PROMPT * v3.1.2: Export query circular dependency +PROMPT * v3.2.0: TRASH retention control (pKeepInTrash parameter) PROMPT -PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2) available in patches/ +PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2->v3.2.0) available in patches/ PROMPT PROMPT Log file: &_filename PROMPT ============================================================================ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql new file mode 100644 index 0000000..b5be987 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql @@ -0,0 +1,29 @@ +-- ==================================================================== +-- A_SOURCE_FILE_RECEIVED Table +-- ==================================================================== +-- Purpose: Track received files and their processing status +-- ==================================================================== + +CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ( + A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE, + A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE, + SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL, + CHECKSUM VARCHAR2(128), + CREATED TIMESTAMP(6) WITH TIME ZONE, + BYTES NUMBER, + RECEPTION_DATE DATE NOT NULL, + PROCESSING_STATUS VARCHAR2(200), + EXTERNAL_TABLE_NAME VARCHAR2(200), + PARTITION_YEAR VARCHAR2(4), + PARTITION_MONTH VARCHAR2(2), + ARCH_FILE_NAME VARCHAR2(1000), + CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY), + CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')) +) TABLESPACE "DATA"; + +-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation) +CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1 +ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES); + +GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb index b450c57..14450b8 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb @@ -87,7 +87,8 @@ AS ---------------------------------------------------------------------------------------------------- PROCEDURE ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE ) IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; @@ -104,7 +105,10 @@ AS vProcessControlStatus VARCHAR2(60) := 'OK'; BEGIN - vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + )); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); @@ -244,17 +248,17 @@ AS END; EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; - -- move file to trash before dropping + -- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping BEGIN DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, source_object_uri => f.pathname||'/'||f.filename, target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, target_credential_name => ENV_MANAGER.gvCredentialName ); - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename); EXCEPTION WHEN OTHERS THEN - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); rollback; vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; @@ -264,24 +268,38 @@ AS END LOOP; -------------------------------------------------------------------- - -- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART) + -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) + -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) IF vProcessControlStatus = 'OK' THEN - FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP - --Drop file from TRASH - DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, - object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename); - END LOOP; + IF NOT pKeepInTrash THEN + -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + -- Update status to ARCHIVED_AND_PURGED + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + END LOOP; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO'); + ELSE + -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO'); + END IF; --ROLLBACK PART - --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH) + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket) ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN FOR f in ( SELECT vf.filename, vf.pathname FROM TABLE(vfiles) vf JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.source_file_name = vf.filename AND r.a_source_file_config_key = pSourceFileConfigKey - AND r.PROCESSING_STATUS = 'ARCHIVED' + AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED') AND vf.year = ym_loop.year AND vf.month = ym_loop.month ) LOOP @@ -291,7 +309,7 @@ AS target_object_uri => f.pathname||'/'||f.filename, target_credential_name => ENV_MANAGER.gvCredentialName ); - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r SET PROCESSING_STATUS = 'INGESTED' @@ -302,7 +320,7 @@ AS EXCEPTION WHEN OTHERS THEN - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; END; @@ -363,9 +381,7 @@ AS CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); END; - -- END of "Try to drop EXPORTED FILES" - - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO'); + -- END of "Try to drop EXPORTED FILES" CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); END LOOP; --ym_loop end (YEAR_MONTH) @@ -523,15 +539,19 @@ AS ---------------------------------------------------------------------------------------------------- FUNCTION ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE ) RETURN PLS_INTEGER IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN - vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + )); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ---- - ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey); + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash); ---- CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); RETURN SQLCODE; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg index 2b3e065..4a93eaf 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg @@ -17,12 +17,13 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.1.2'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-06 11:30:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-06 14:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || '3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) || '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || @@ -38,9 +39,11 @@ AS * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. **/ PROCEDURE ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE ); /** @@ -48,11 +51,13 @@ AS * @desc Function overload for ARCHIVE_TABLE_DATA procedure. * Returns SQLCODE for Python library integration. * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. + * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. * @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; * @ex_rslt 0 (success) or error code **/ FUNCTION ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE ) RETURN PLS_INTEGER; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql index 220ba1c..f1a48dc 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql @@ -32,10 +32,11 @@ PROMPT ========================================================================= PROMPT This will restore FILE_ARCHIVER to v2.0.0 PROMPT PROMPT Rollback steps: -PROMPT 1. Remove validation trigger -PROMPT 2. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns -PROMPT 3. Restore FILE_ARCHIVER package to v2.0.0 -PROMPT 4. Revert all archival strategies to THRESHOLD_BASED +PROMPT 1. Rollback TRASH retention statuses +PROMPT 2. Remove validation trigger +PROMPT 3. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns +PROMPT 4. Restore FILE_ARCHIVER package to v2.0.0 +PROMPT 5. Revert all archival strategies to THRESHOLD_BASED PROMPT PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; @@ -54,27 +55,32 @@ WHENEVER SQLERROR CONTINUE -- Rollback steps (in reverse order) PROMPT -PROMPT Step 1/5: Dropping validation trigger +PROMPT Step 1/6: Rolling back TRASH retention statuses +PROMPT ================================================ +@@90_MARS_828_rollback_trash_retention_statuses.sql + +PROMPT +PROMPT Step 2/6: Dropping validation trigger PROMPT ====================================== @@93_MARS_828_rollback_trigger.sql PROMPT -PROMPT Step 2/5: Dropping archival strategy columns +PROMPT Step 3/6: Dropping archival strategy columns PROMPT ============================================= @@94_MARS_828_rollback_columns.sql PROMPT -PROMPT Step 3/5: Restoring FILE_ARCHIVER Package Specification v2.0.0 +PROMPT Step 4/6: Restoring FILE_ARCHIVER Package Specification v2.0.0 PROMPT =============================================================== @@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 4/5: Restoring FILE_ARCHIVER Package Body v2.0.0 +PROMPT Step 5/6: Restoring FILE_ARCHIVER Package Body v2.0.0 PROMPT ====================================================== @@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql PROMPT -PROMPT Step 5/5: Verifying tracked packages +PROMPT Step 6/6: Verifying tracked packages PROMPT ===================================== @@verify_packages_version.sql diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql index 4e5bbea..b5be987 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql @@ -19,7 +19,7 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ( ARCH_FILE_NAME VARCHAR2(1000), CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY), CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), - CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED')) + CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')) ) TABLESPACE "DATA"; -- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)