diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql index 9e057d3..0b45bf1 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql @@ -10,15 +10,23 @@ PROMPT ======================================== -- Add new columns ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD ( ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL, - MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL + MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL, + ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL, + KEEP_IN_TRASH CHAR(1) DEFAULT 'Y' NOT NULL ); --- Add check constraint for valid strategies +-- Add check constraints ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK ( ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID') ); +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT + CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')); + +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT + CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')); + -- Add comments COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED (days), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; @@ -26,6 +34,12 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS + 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS + 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0'; + -- Verify columns added SELECT column_name, @@ -36,7 +50,7 @@ SELECT FROM all_tab_columns WHERE owner = 'CT_MRDS' AND table_name = 'A_SOURCE_FILE_CONFIG' - AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS') + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'ARCHIVE_ENABLED', 'KEEP_IN_TRASH') ORDER BY column_id; PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql index edfab99..db3eb27 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql @@ -1,17 +1,29 @@ --- =================================================================== --- MARS-828: Install FILE_ARCHIVER Package Specification v3.2.0 --- =================================================================== --- Purpose: Deploy updated package specification with version 3.2.0 +--============================================================================================================================= +-- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0 +--============================================================================================================================= +-- Purpose: Deploy FILE_ARCHIVER Package Specification with FN_ function names -- Author: Grzegorz Michalski --- Date: 2026-02-06 --- Changes: --- - Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA --- - TRASH folder retention control for safety and compliance --- =================================================================== +-- Date: 2026-02-11 +-- Related: MARS-828 Archival Strategy Implementation +--============================================================================================================================= + +SET SERVEROUTPUT ON + +PROMPT ======================================================================== +PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0 +PROMPT ======================================================================== @@new_version/FILE_ARCHIVER.pkg +-- Verify package compilation (check specific schema when installing as ADMIN) +SELECT OBJECT_NAME, OBJECT_TYPE, STATUS +FROM ALL_OBJECTS +WHERE OWNER = 'CT_MRDS' + AND OBJECT_NAME = 'FILE_ARCHIVER' + AND OBJECT_TYPE = 'PACKAGE'; -PROMPT ======================================== -PROMPT FILE_ARCHIVER Specification v3.2.0 installed successfully -PROMPT ======================================== +PROMPT SUCCESS: FILE_ARCHIVER Package Specification v3.3.0 installed + +--============================================================================================================================= +-- End of Script +--============================================================================================================================= diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql index 78a5fd7..2c124bc 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@ -1,16 +1,38 @@ --- =================================================================== --- MARS-828: Install FILE_ARCHIVER Package Body v3.2.0 --- =================================================================== --- Purpose: Deploy updated package body with TRASH folder retention control +--============================================================================================================================= +-- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Body v3.3.0 +--============================================================================================================================= +-- Purpose: Deploy FILE_ARCHIVER Package Body with config-based archival and FN_ function names -- Author: Grzegorz Michalski --- Date: 2026-02-06 --- Changes: --- - v3.0.0: Added GET_ARCHIVAL_WHERE_CLAUSE for flexible archival strategies --- - v3.1.0: Added function overloads for Python integration --- - v3.1.1: Fixed ORA-01422 for multiple parquet files (directory prefix storage) --- - v3.1.2: Fixed PARTITION_YEAR/PARTITION_MONTH assignments and circular dependency --- - v3.2.0: Added pKeepInTrash parameter for TRASH folder retention control --- =================================================================== +-- Date: 2026-02-11 +-- Related: MARS-828 Archival Strategy Implementation +--============================================================================================================================= + +SET SERVEROUTPUT ON + +PROMPT ======================================================================== +PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Body v3.3.0 +PROMPT ======================================================================== @@new_version/FILE_ARCHIVER.pkb +-- Verify package compilation (check specific schema when installing as ADMIN) +SELECT OBJECT_NAME, OBJECT_TYPE, STATUS +FROM ALL_OBJECTS +WHERE OWNER = 'CT_MRDS' + AND OBJECT_NAME = 'FILE_ARCHIVER' + AND OBJECT_TYPE IN ('PACKAGE', 'PACKAGE BODY') +ORDER BY OBJECT_TYPE; + +-- Check for any compilation errors +SELECT 'COMPILATION ERRORS FOUND' AS WARNING +FROM ALL_ERRORS +WHERE OWNER = 'CT_MRDS' + AND NAME = 'FILE_ARCHIVER' + AND TYPE = 'PACKAGE BODY' + AND ROWNUM = 1; + +PROMPT SUCCESS: FILE_ARCHIVER Package Body v3.3.0 installed + +--============================================================================================================================= +-- End of Script +--============================================================================================================================= diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql index 630379e..a1af3c9 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql @@ -60,7 +60,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000, BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824, -- 1 GB - HOURS_TO_EXPIRE_STATISTICS = 24 + HOURS_TO_EXPIRE_STATISTICS = 24, + ARCHIVE_ENABLED = 'Y', -- Enable archival for all LM tables + KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention) WHERE SOURCE_FILE_TYPE = 'INPUT' AND A_SOURCE_KEY = 'LM' AND TABLE_ID IN ( @@ -104,7 +106,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 5, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000, BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912, -- 512 MB - HOURS_TO_EXPIRE_STATISTICS = 48 + HOURS_TO_EXPIRE_STATISTICS = 48, + ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB DEBT tables + KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention) WHERE SOURCE_FILE_TYPE = 'INPUT' AND A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY'); @@ -128,7 +132,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000, BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456, -- 256 MB - HOURS_TO_EXPIRE_STATISTICS = 72 + HOURS_TO_EXPIRE_STATISTICS = 72, + ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB rating/description tables + KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention) WHERE SOURCE_FILE_TYPE = 'INPUT' AND A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( @@ -168,6 +174,8 @@ SELECT ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, + ARCHIVE_ENABLED, + KEEP_IN_TRASH, CASE WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' AND MINIMUM_AGE_MONTHS = 0 @@ -175,6 +183,8 @@ SELECT AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824 AND HOURS_TO_EXPIRE_STATISTICS = 24 + AND ARCHIVE_ENABLED = 'Y' + AND KEEP_IN_TRASH = 'N' THEN 'OK' ELSE 'ERROR' END AS STATUS @@ -196,6 +206,8 @@ SELECT ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, + ARCHIVE_ENABLED, + KEEP_IN_TRASH, CASE WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' AND MINIMUM_AGE_MONTHS = 6 @@ -203,6 +215,8 @@ SELECT AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912 AND HOURS_TO_EXPIRE_STATISTICS = 48 + AND ARCHIVE_ENABLED = 'Y' + AND KEEP_IN_TRASH = 'N' THEN 'OK' ELSE 'ERROR' END AS STATUS @@ -224,6 +238,8 @@ SELECT ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, + ARCHIVE_ENABLED, + KEEP_IN_TRASH, CASE WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' AND MINIMUM_AGE_MONTHS = 0 @@ -231,6 +247,8 @@ SELECT AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456 AND HOURS_TO_EXPIRE_STATISTICS = 72 + AND ARCHIVE_ENABLED = 'Y' + AND KEEP_IN_TRASH = 'N' THEN 'OK' ELSE 'ERROR' END AS STATUS @@ -252,7 +270,9 @@ SELECT SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_FILE_THRESHOLD, SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_ROWS_THRESHOLD, SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_BYTES_THRESHOLD, - SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY + SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY, + SUM(CASE WHEN ARCHIVE_ENABLED = 'Y' THEN 1 ELSE 0 END) AS ARCHIVAL_ENABLED, + SUM(CASE WHEN KEEP_IN_TRASH = 'N' THEN 1 ELSE 0 END) AS IMMEDIATE_DELETE FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%') @@ -269,6 +289,8 @@ PROMPT - WITH_FILE_THRESHOLD = 25 PROMPT - WITH_ROWS_THRESHOLD = 25 PROMPT - WITH_BYTES_THRESHOLD = 25 PROMPT - WITH_STATS_EXPIRY = 25 +PROMPT - ARCHIVAL_ENABLED = 25 (all tables enabled for archival) +PROMPT - IMMEDIATE_DELETE = 25 (all tables delete files immediately, no TRASH retention) PROMPT PROMPT ===================================================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql index fc481eb..6e17191 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql @@ -1,20 +1,28 @@ -- MARS-828: Rollback archival strategy columns -- Author: Grzegorz Michalski -- Date: 2026-01-27 --- Description: Remove ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns +-- Description: Remove ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, ARCHIVE_ENABLED, and KEEP_IN_TRASH columns PROMPT ======================================== -PROMPT MARS-828: Removing archival strategy columns +PROMPT MARS-828: Removing archival strategy and config columns PROMPT ======================================== --- Drop check constraint first +-- Drop check constraints first ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY; +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG +DROP CONSTRAINT CHK_ARCHIVE_ENABLED; + +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG +DROP CONSTRAINT CHK_KEEP_IN_TRASH; + -- Drop columns ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP ( ARCHIVAL_STRATEGY, - MINIMUM_AGE_MONTHS + MINIMUM_AGE_MONTHS, + ARCHIVE_ENABLED, + KEEP_IN_TRASH ); -- Verify columns dropped @@ -23,8 +31,8 @@ SELECT FROM all_tab_columns WHERE owner = 'CT_MRDS' AND table_name = 'A_SOURCE_FILE_CONFIG' - AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS'); + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'ARCHIVE_ENABLED', 'KEEP_IN_TRASH'); PROMPT ======================================== -PROMPT Archival strategy columns removed successfully +PROMPT Archival strategy and config columns removed successfully PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql index e0b8ab1..3af4f80 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql @@ -35,9 +35,9 @@ PROMPT ========================================================================= PROMPT MARS-828 Installation Starting PROMPT ============================================================================ PROMPT Package: CT_MRDS.FILE_ARCHIVER -PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention control -PROMPT Purpose: Flexible archival policies per data source with file retention management -PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.2.0, Verify, Track, Configure) +PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving +PROMPT Purpose: Flexible archival policies per data source with file retention and config-based control +PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.3.0, Verify, Track, Configure) PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; PROMPT ============================================================================ @@ -55,8 +55,8 @@ WHENEVER SQLERROR CONTINUE -- Installation steps PROMPT -PROMPT Step 1/9: Adding archival strategy columns to A_SOURCE_FILE_CONFIG -PROMPT =================================================================== +PROMPT Step 1/9: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG +PROMPT ============================================================================= @@01_MARS_828_install_add_archival_strategy_columns.sql PROMPT @@ -70,12 +70,12 @@ PROMPT ===================================================================== @@07_MARS_828_install_add_trash_retention_statuses.sql PROMPT -PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.2.0 +PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.3.0 PROMPT ================================================================ @@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.2.0 +PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.3.0 PROMPT ====================================================== @@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@ -108,21 +108,21 @@ SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL; PROMPT PROMPT Installation Summary: PROMPT - Package: CT_MRDS.FILE_ARCHIVER -PROMPT - Version: 3.2.0 (includes TRASH folder retention control) +PROMPT - Version: 3.3.0 (includes selective archiving and config-based TRASH policy) PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID -PROMPT - TRASH Retention: pKeepInTrash parameter (DEFAULT TRUE for safety) +PROMPT - Selective Archiving: ARCHIVE_ENABLED column (Y=archive, N=skip) +PROMPT - TRASH Policy: KEEP_IN_TRASH column (Y=keep files, N=delete immediately) +PROMPT * Default: ARCHIVE_ENABLED='Y', KEEP_IN_TRASH='N' (archiving enabled, immediate deletion) PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME) -PROMPT * Files kept in TRASH by default for compliance and rollback capability +PROMPT * No more pKeepInTrash parameter - policy from config only +PROMPT - New Procedure: ARCHIVE_ALL_FOR_SOURCE(pSourceKey) for batch processing +PROMPT - TRASH Management: RESTORE_FILE_FROM_TRASH, PURGE_TRASH_FOLDER (3-level granularity) PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED -PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED + TRASH retention preserved) +PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED, existing behavior preserved) PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB) -PROMPT - Includes Fixes: -PROMPT * v3.1.1: ORA-01422 for multiple parquet files -PROMPT * v3.1.2: PARTITION_YEAR/PARTITION_MONTH assignments -PROMPT * v3.1.2: Export query circular dependency -PROMPT * v3.2.0: TRASH retention control (pKeepInTrash parameter) +PROMPT - Includes All Fixes from v3.0.0 through v3.2.1 PROMPT -PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2->v3.2.0) available in patches/ +PROMPT Note: Incremental patches available in patches/ directory PROMPT PROMPT Log file: &_filename PROMPT ============================================================================ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql new file mode 100644 index 0000000..0aedf80 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql @@ -0,0 +1,56 @@ +-- ==================================================================== +-- A_SOURCE_FILE_CONFIG Table +-- ==================================================================== +-- Purpose: Store source file configuration and processing rules +-- MARS-1049: Added ENCODING column for CSV character set support +-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation +-- ==================================================================== + +CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( + A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE, + A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE, + SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG' + SOURCE_FILE_ID VARCHAR2(200), + SOURCE_FILE_DESC VARCHAR2(2000), + SOURCE_FILE_NAME_PATTERN VARCHAR2(200), + TABLE_ID VARCHAR2(200), + TEMPLATE_TABLE_NAME VARCHAR2(200), + CONTAINER_FILE_KEY NUMBER(38,0), + DAYS_FOR_ARCHIVE_THRESHOLD NUMBER(4,0), + FILES_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), + BYTES_SUM_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), + ODS_SCHEMA_NAME VARCHAR2(100), + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), + HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3), + ARCHIVAL_STRATEGY VARCHAR2(50), + MINIMUM_AGE_MONTHS NUMBER(3,0), + ENCODING VARCHAR2(50) DEFAULT 'UTF8', + ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL, + KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL, + CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')), + CONSTRAINT CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')), + CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), + CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), + CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID) +) TABLESPACE "DATA"; + +-- Primary key index (from production export) +CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK" +ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY") +TABLESPACE "DATA"; + +-- Unique constraint index (from production export) +CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1" +ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID") +TABLESPACE "DATA"; + +-- Column comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0'; + +GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb index 7e17f44..67a8339 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb @@ -87,8 +87,7 @@ AS ---------------------------------------------------------------------------------------------------- PROCEDURE ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, - pKeepInTrash IN BOOLEAN DEFAULT TRUE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; @@ -103,15 +102,27 @@ AS vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; vProcessControlStatus VARCHAR2(60) := 'OK'; + vKeepInTrash BOOLEAN; -- Derived from config BEGIN vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( - 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), - 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') )); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + -- Check if archiving is enabled for this configuration + IF vSourceFileConfig.ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archiving disabled for this configuration (ARCHIVE_ENABLED=N). Skipping.', 'WARNING', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN; + END IF; + + -- Get TRASH policy from configuration + vKeepInTrash := (vSourceFileConfig.KEEP_IN_TRASH = 'Y'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: KEEP_IN_TRASH=' || vSourceFileConfig.KEEP_IN_TRASH, 'INFO', vParameters); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then @@ -271,7 +282,7 @@ AS -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) IF vProcessControlStatus = 'OK' THEN - IF NOT pKeepInTrash THEN + IF NOT vKeepInTrash THEN -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, @@ -285,10 +296,10 @@ AS AND r.source_file_name = f.filename AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; END LOOP; - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: KEEP_IN_TRASH=N).','INFO'); ELSE -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) - CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO'); END IF; --ROLLBACK PART @@ -947,20 +958,18 @@ AS ---------------------------------------------------------------------------------------------------- - FUNCTION ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, - pKeepInTrash IN BOOLEAN DEFAULT TRUE + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) RETURN PLS_INTEGER IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( - 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), - 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') )); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ---- - ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash); + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey); ---- CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); RETURN SQLCODE; @@ -968,11 +977,11 @@ AS WHEN OTHERS THEN CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RETURN SQLCODE; - END ARCHIVE_TABLE_DATA; + END FN_ARCHIVE_TABLE_DATA; ---------------------------------------------------------------------------------------------------- - FUNCTION GATHER_TABLE_STAT ( + FUNCTION FN_GATHER_TABLE_STAT ( pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) RETURN PLS_INTEGER IS @@ -989,7 +998,284 @@ AS WHEN OTHERS THEN CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RETURN SQLCODE; - END GATHER_TABLE_STAT; + END FN_GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- BATCH ARCHIVAL PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesArchived PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pArchiveAll THEN + vProcessingLevel := 'LEVEL 3: Archive All'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Archive All Enabled Tables', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pArchiveAll=TRUE'); + END IF; + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + ARCHIVE_ENABLED, + KEEP_IN_TRASH, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pArchiveAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pArchiveAll = TRUE) + ) + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF config_rec.ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', KEEP_IN_TRASH=' || config_rec.KEEP_IN_TRASH || ']', + 'INFO' + ); + + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesArchived := vTablesArchived + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully archived table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to archive table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch archival summary: Archived=' || vTablesArchived || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed, + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pArchiveAll => pArchiveAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + -- BATCH STATISTICS GATHERING PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesProcessed PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + vEnabledFilter VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pGatherAll THEN + vProcessingLevel := 'LEVEL 3: Gather All Statistics'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Gather All Enabled Statistics', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pGatherAll=TRUE'); + END IF; + + -- Set enabled filter info + vEnabledFilter := CASE WHEN pOnlyEnabled THEN 'ARCHIVE_ENABLED=Y only' ELSE 'All tables' END; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Filter mode: ' || vEnabledFilter, 'INFO'); + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + ARCHIVE_ENABLED, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pGatherAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pGatherAll = TRUE) + ) + -- Apply ARCHIVE_ENABLED filter if pOnlyEnabled = TRUE + AND (pOnlyEnabled = FALSE OR ARCHIVE_ENABLED = 'Y') + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF pOnlyEnabled AND config_rec.ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Gathering statistics for table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', ARCHIVE_ENABLED=' || config_rec.ARCHIVE_ENABLED || ']', + 'INFO' + ); + + GATHER_TABLE_STAT(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesProcessed := vTablesProcessed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully gathered statistics for table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to gather statistics for table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch statistics gathering summary: Processed=' || vTablesProcessed || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed || ' [Filter: ' || vEnabledFilter || ']', + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pGatherAll => pGatherAll, + pOnlyEnabled => pOnlyEnabled + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_GATHER_TABLE_STAT_ALL; ---------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg index 4e2e199..febbf3e 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg @@ -17,12 +17,13 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.1'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-10 09:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.3.0 (2026-02-11): Added ARCHIVE_ENABLED and KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) || '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || @@ -35,30 +36,38 @@ AS cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + /** + * @name GET_TABLE_STAT + * @desc Private function to retrieve table statistics for archival processing. + * Returns A_TABLE_STAT record with table metadata and row counts. + * @param pSourceFileConfigKey - Configuration key for source file + * @return CT_MRDS.A_TABLE_STAT%ROWTYPE - Table statistics record + * @private Internal function for archival operations + **/ + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE; + /** * @name ARCHIVE_TABLE_DATA * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). - * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). **/ PROCEDURE ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, - pKeepInTrash IN BOOLEAN DEFAULT TRUE + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ); /** - * @name ARCHIVE_TABLE_DATA - * @desc Function overload for ARCHIVE_TABLE_DATA procedure. + * @name FN_ARCHIVE_TABLE_DATA + * @desc Function wrapper for ARCHIVE_TABLE_DATA procedure. * Returns SQLCODE for Python library integration. * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. - * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. - * @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; * @ex_rslt 0 (success) or error code **/ - FUNCTION ARCHIVE_TABLE_DATA ( - pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, - pKeepInTrash IN BOOLEAN DEFAULT TRUE + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) RETURN PLS_INTEGER; @@ -73,17 +82,92 @@ AS ); /** - * @name GATHER_TABLE_STAT - * @desc Function overload for GATHER_TABLE_STAT procedure. + * @name FN_GATHER_TABLE_STAT + * @desc Function wrapper for GATHER_TABLE_STAT procedure. * Returns SQLCODE for Python library integration. * Calls the main GATHER_TABLE_STAT procedure and captures execution result. - * @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; * @ex_rslt 0 (success) or error code **/ - FUNCTION GATHER_TABLE_STAT ( + FUNCTION FN_GATHER_TABLE_STAT ( pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) RETURN PLS_INTEGER; + /** + * @name GATHER_TABLE_STAT_ALL + * @desc Multi-level batch statistics gathering procedure with three granularity levels. + * Processes configurations based on ARCHIVE_ENABLED setting (when pOnlyEnabled=TRUE). + * Gathers statistics for external tables and inserts data into A_TABLE_STAT and A_TABLE_STAT_HIST. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (e.g., 'LM', 'C2D') (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with ARCHIVE_ENABLED='Y' + * @example -- Level 1: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE); + * @example -- All tables regardless of ARCHIVE_ENABLED: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE, pOnlyEnabled => FALSE); + **/ + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ); + + /** + * @name FN_GATHER_TABLE_STAT_ALL + * @desc Function wrapper for GATHER_TABLE_STAT_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with ARCHIVE_ENABLED='Y' + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER; + + /** + * @name ARCHIVE_ALL + * @desc Multi-level batch archival procedure with three granularity levels. + * Only processes configurations where ARCHIVE_ENABLED='Y'. + * TRASH policy for each table is controlled by individual KEEP_IN_TRASH column. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example -- Level 1: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.ARCHIVE_ALL(pArchiveAll => TRUE); + **/ + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name FN_ARCHIVE_ALL + * @desc Function wrapper for ARCHIVE_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + /** * @name RESTORE_FILE_FROM_TRASH * @desc Restores files from TRASH folder back to ODS at three different granularity levels. diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql index f1a48dc..0a3822a 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql @@ -34,7 +34,7 @@ PROMPT PROMPT Rollback steps: PROMPT 1. Rollback TRASH retention statuses PROMPT 2. Remove validation trigger -PROMPT 3. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns +PROMPT 3. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, ARCHIVE_ENABLED, KEEP_IN_TRASH) PROMPT 4. Restore FILE_ARCHIVER package to v2.0.0 PROMPT 5. Revert all archival strategies to THRESHOLD_BASED PROMPT @@ -65,8 +65,8 @@ PROMPT ====================================== @@93_MARS_828_rollback_trigger.sql PROMPT -PROMPT Step 3/6: Dropping archival strategy columns -PROMPT ============================================= +PROMPT Step 3/6: Dropping all archival configuration columns +PROMPT ====================================================== @@94_MARS_828_rollback_columns.sql PROMPT diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..7e17f44 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkb @@ -0,0 +1,998 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE + ---------------------------------------------------------------------------------------------------- + /** + * @name GET_ARCHIVAL_WHERE_CLAUSE + * @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration. + * Supports three strategies: THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID. + * @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY + * @return VARCHAR2 - WHERE clause for filtering archival candidates + **/ + FUNCTION GET_ARCHIVAL_WHERE_CLAUSE( + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) RETURN VARCHAR2 + IS + vWhereClause VARCHAR2(4000); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + CASE pSourceFileConfig.ARCHIVAL_STRATEGY + -- Legacy threshold-based strategy (backward compatible) + WHEN 'THRESHOLD_BASED' THEN + vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD; + + -- Archive data older than X months (0 = current month only) + WHEN 'MINIMUM_AGE_MONTHS' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); + END IF; + vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + -- Hybrid: Current month exclusion AND minimum age requirement + WHEN 'HYBRID' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy'); + END IF; + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' || + 'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + ELSE + RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY); + END CASE; + + RETURN vWhereClause; + END GET_ARCHIVAL_WHERE_CLAUSE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + -- Strategy-based trigger logic (MARS-828) + IF vSourceFileConfig.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN + -- MINIMUM_AGE_MONTHS: Archive based on age only, ignore thresholds + vArchivalTriggeredBy := 'AGE_BASED'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival strategy: MINIMUM_AGE_MONTHS (threshold-independent)','INFO'); + ELSE + -- THRESHOLD_BASED and HYBRID: Check thresholds + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + END IF; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + + -- Use strategy-based WHERE clause (MARS-828) + -- Using GROUP BY instead of DISTINCT to avoid ORA-22950 (object type ordering issue) + vQuery := ' + select CT_MRDS.t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' + group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')' + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + and r.PROCESSING_STATUS = ''INGESTED'' + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||''' + and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||''' + ' + ; + vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month,'INFO', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + -- Note: DBMS_CLOUD.EXPORT_DATA may create multiple parquet files (parallel execution) + -- Instead of tracking individual files, we store the archive directory prefix + -- ARCH_FILE_NAME will contain the directory URI where all parquet files are located + vFilename := vUri; -- Store directory prefix instead of individual filename + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status to ARCHIVED_AND_TRASHED (file will be moved to TRASH folder) + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -- Status reflects file is archived and kept in TRASH + ,ARCH_FILE_NAME = vFilename -- Now contains directory prefix, not individual file + ,PARTITION_YEAR = ym_loop.year -- Record which partition year the data was archived to + ,PARTITION_MONTH = ym_loop.month -- Record which partition month the data was archived to + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + + -- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) + -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) + IF vProcessControlStatus = 'OK' THEN + IF NOT pKeepInTrash THEN + -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + -- Update status to ARCHIVED_AND_PURGED + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + END LOOP; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO'); + ELSE + -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO'); + END IF; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED') + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_FILE_NAME = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + vWhereClause VARCHAR2(4000); + vOdsBucketUri VARCHAR2(1000); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + + -- Get WHERE clause based on archival strategy (MARS-828) + vWhereClause := GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vWhereClause','DEBUG',vWhereClause); + + -- Get ODS bucket URI before building query + vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + -- Use strategy-based WHERE clause for statistics (MARS-828) + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES + ,'||COALESCE(TO_CHAR(vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD), 'NULL')||' as DAYS_FOR_ARCHIVE_THRESHOLD + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''', + location_uri => '''||vOdsBucketUri||''' + ) + ) r + on t.filename = r.object_name' + ; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into CT_MRDS.A_TABLE_STAT_HIST values vStats; + delete from CT_MRDS.A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into CT_MRDS.A_TABLE_STAT values vStats; + COMMIT; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- TRASH FOLDER MANAGEMENT PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashPath VARCHAR2(1000); + vOdsPath VARCHAR2(1000); + vFilesRestored PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vRestoreLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath); + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Restore all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + FOR file_rec IN ( + SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pRestoreAll THEN + vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations + FOR file_rec IN ( + SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME, + c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashLocationUri VARCHAR2(1000); + vFilesDeleted PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vPurgeLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri + ); + vFilesDeleted := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Delete all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pPurgeAll THEN + vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations + FOR config_rec IN ( + SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_CONFIG c + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PURGE_TRASH_FOLDER( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pPurgeAll => pPurgeAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + RESTORE_FILE_FROM_TRASH( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pRestoreAll => pRestoreAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..4e2e199 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkg @@ -0,0 +1,195 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.1'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-10 09:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || + '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || + '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || + '3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) || + '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE + ); + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Function overload for ARCHIVE_TABLE_DATA procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. + * @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. + * @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, + pKeepInTrash IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER; + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name GATHER_TABLE_STAT + * @desc Function overload for GATHER_TABLE_STAT procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT procedure and captures execution result. + * @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Restores files from TRASH folder back to ODS at three different granularity levels. + * Moves files from TRASH subfolder back to ODS subfolder in DATA bucket. + * Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345); + * @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341); + * @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE); + **/ + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Function overload for RESTORE_FILE_FROM_TRASH procedure. + * Returns SQLCODE for Python library integration. + * Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name PURGE_TRASH_FOLDER + * @desc Deletes files from TRASH folder at three different granularity levels. + * Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345); + * @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341); + * @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE); + **/ + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name PURGE_TRASH_FOLDER + * @desc Function overload for PURGE_TRASH_FOLDER procedure. + * Returns SQLCODE for Python library integration. + * Calls the main PURGE_TRASH_FOLDER procedure and captures execution result. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql index a7bb59b..7d41071 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql @@ -25,7 +25,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( ARCHIVAL_STRATEGY VARCHAR2(50), MINIMUM_AGE_MONTHS NUMBER(3,0), ENCODING VARCHAR2(50) DEFAULT 'UTF8', + ARCHIVE_ENABLED CHAR(1) DEFAULT 'Y' NOT NULL, + KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL, CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')), + CONSTRAINT CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')), CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), @@ -46,5 +50,7 @@ TABLESPACE "DATA"; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0'; GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; \ No newline at end of file