Merge branch 'main' of https://git.itbi.mywire.org/admin/mars

MARS_Packages/REL02_POST/MARS-1409/01_MARS_1409_add_workflow_history_key_column.sql

@@ -35,5 +35,21 @@ END;
 /
 
 PROMPT
-PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added to A_SOURCE_FILE_RECEIVED.
+PROMPT Renaming IS_KEEP_IN_TRASH to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG...
+
+BEGIN
+    EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEEP_IN_TRASH TO IS_KEPT_IN_TRASH';
+    DBMS_OUTPUT.PUT_LINE('Column IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG.');
+EXCEPTION
+    WHEN OTHERS THEN
+        IF SQLCODE = -904 THEN
+            DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEEP_IN_TRASH does not exist (already renamed or not present).');
+        ELSE
+            RAISE;
+        END IF;
+END;
+/
+
+PROMPT
+PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added and IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH.
 PROMPT

MARS_Packages/REL02_POST/MARS-1409/99_MARS_1409_rollback_workflow_history_key_column.sql

@@ -26,5 +26,21 @@ END;
 /
 
 PROMPT
-PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY column removed from A_SOURCE_FILE_RECEIVED.
+PROMPT Renaming IS_KEPT_IN_TRASH back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG...
+
+BEGIN
+    EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEPT_IN_TRASH TO IS_KEEP_IN_TRASH';
+    DBMS_OUTPUT.PUT_LINE('Column IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG.');
+EXCEPTION
+    WHEN OTHERS THEN
+        IF SQLCODE = -904 THEN
+            DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEPT_IN_TRASH does not exist (already renamed back or not present).');
+        ELSE
+            RAISE;
+        END IF;
+END;
+/
+
+PROMPT
+PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY removed and IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH.
 PROMPT

MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql

@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
     MINIMUM_AGE_MONTHS NUMBER(3,0),
     ENCODING VARCHAR2(50) DEFAULT 'UTF8',
     IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
-    IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
+    IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
     IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
     CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
     CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
-    CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
+    CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
     CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
     CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
     CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
@@ -97,7 +97,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
 COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS 
     'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
 
-COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS 
+COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS 
     'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
 
 COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS 

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb

@@ -120,8 +120,8 @@ AS
       END IF;
       
       -- Get TRASH policy from configuration
-      vKeepInTrash := (vSourceFileConfig.IS_KEEP_IN_TRASH = 'Y');
+      vKeepInTrash := (vSourceFileConfig.IS_KEPT_IN_TRASH = 'Y');
-      CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEEP_IN_TRASH=' || vSourceFileConfig.IS_KEEP_IN_TRASH, 'INFO', vParameters);
+      CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEPT_IN_TRASH=' || vSourceFileConfig.IS_KEPT_IN_TRASH, 'INFO', vParameters);
       
       vTableStat        := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
 
@@ -309,10 +309,10 @@ AS
                           AND r.source_file_name = f.filename
                           AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
                      END LOOP;
-                     CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEEP_IN_TRASH=N).','INFO', vParameters);
+                     CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEPT_IN_TRASH=N).','INFO', vParameters);
                   ELSE
                      -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
-                     CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO', vParameters);
+                     CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEPT_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO', vParameters);
                   END IF;
 
                --ROLLBACK PART
@@ -1112,7 +1112,7 @@ AS
             A_SOURCE_FILE_CONFIG_KEY, 
             TABLE_ID, 
             IS_ARCHIVE_ENABLED,
-            IS_KEEP_IN_TRASH,
+            IS_KEPT_IN_TRASH,
             A_SOURCE_KEY
          FROM CT_MRDS.A_SOURCE_FILE_CONFIG
          WHERE SOURCE_FILE_TYPE = 'INPUT'
@@ -1137,7 +1137,7 @@ AS
          ELSE
             BEGIN
                CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
-                  'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEEP_IN_TRASH=' || config_rec.IS_KEEP_IN_TRASH || ']',
+                  'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEPT_IN_TRASH=' || config_rec.IS_KEPT_IN_TRASH || ']',
                   'INFO'
                );
                

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg

@@ -27,7 +27,7 @@ AS
       'Y=standard DBT flow (WORKFLOW_SUCCESSFUL=Y required), N=bypass for manual/non-DBT sources. ' ||
       'Flag value stored in A_TABLE_STAT and A_TABLE_STAT_HIST for full audit of statistics basis.' || CHR(13)||CHR(10) ||
       '3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) ||
-      '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
+      '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEPT_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
       '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
       '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
       '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
@@ -55,7 +55,7 @@ AS
       * @desc     Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
       *           Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
       *           Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
-      *           TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
+      *           TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
    **/
    PROCEDURE ARCHIVE_TABLE_DATA (
       pSourceFileConfigKey   IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
@@ -66,7 +66,7 @@ AS
       * @desc     Function wrapper for ARCHIVE_TABLE_DATA procedure.
       *           Returns SQLCODE for Python library integration.
       *           Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
-      *           TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
+      *           TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
       * @example  SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
       * @ex_rslt  0 (success) or error code
    **/
@@ -141,7 +141,7 @@ AS
       * @name     ARCHIVE_ALL
       * @desc     Multi-level batch archival procedure with three granularity levels.
       *           Only processes configurations where IS_ARCHIVE_ENABLED='Y'.
-      *           TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column.
+      *           TRASH policy for each table is controlled by individual IS_KEPT_IN_TRASH column.
       * @param    pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
       * @param    pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority)
       * @param    pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb

@@ -1634,7 +1634,7 @@ AS
       ,pEncoding                    IN    CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE                        DEFAULT NULL               -- MARS-1049
       ,pIsWorkflowSuccessRequired  IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE  DEFAULT 'Y'                -- MARS-1409
       ,pIsArchiveEnabled           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE             DEFAULT 'N'                -- MARS-828
-      ,pIsKeepInTrash              IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
+      ,pIsKeepInTrash              IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
       ,pArchivalStrategy           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE              DEFAULT 'THRESHOLD_BASED'  -- MARS-828
       ,pMinimumAgeMonths           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE             DEFAULT 0                  -- MARS-828
    ) IS
@@ -1659,7 +1659,7 @@ AS
                                                             ,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||''''  -- MARS-828
                                                             ));
       ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-      INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
+      INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
       VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths);
       COMMIT;
       ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
@@ -1770,7 +1770,7 @@ AS
                      ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS           = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
                      ||cgBL||pLevel||'ENCODING                             = '||pSourceFileConfig.ENCODING
                      ||cgBL||pLevel||'IS_ARCHIVE_ENABLED                   = '||pSourceFileConfig.IS_ARCHIVE_ENABLED
-                     ||cgBL||pLevel||'IS_KEEP_IN_TRASH                     = '||pSourceFileConfig.IS_KEEP_IN_TRASH
+                     ||cgBL||pLevel||'IS_KEPT_IN_TRASH                     = '||pSourceFileConfig.IS_KEPT_IN_TRASH
                      ||cgBL||pLevel||'ARCHIVAL_STRATEGY                    = '||pSourceFileConfig.ARCHIVAL_STRATEGY
                      ||cgBL||pLevel||'MINIMUM_AGE_MONTHS                   = '||pSourceFileConfig.MINIMUM_AGE_MONTHS
                      ||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED         = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg

@@ -476,7 +476,7 @@ AS
       ,pEncoding                   IN   CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE                        DEFAULT NULL               -- MARS-1049
       ,pIsWorkflowSuccessRequired  IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE  DEFAULT 'Y'                -- MARS-1409
       ,pIsArchiveEnabled           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE             DEFAULT 'N'                -- MARS-828
-      ,pIsKeepInTrash              IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
+      ,pIsKeepInTrash              IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
       ,pArchivalStrategy           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE              DEFAULT 'THRESHOLD_BASED'  -- MARS-828
       ,pMinimumAgeMonths           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE             DEFAULT 0                  -- MARS-828
    );

MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql

@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
     MINIMUM_AGE_MONTHS NUMBER(3,0),
     ENCODING VARCHAR2(50) DEFAULT 'UTF8',
     IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
-    IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
+    IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
     IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
     CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
     CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
-    CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
+    CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
     CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
     CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
     CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
@@ -107,7 +107,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
 COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS 
     'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
 
-COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS 
+COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS 
     'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
 
 COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS