feat(FILE_ARCHIVER): Improve archival logic and error handling in FILE_ARCHIVER procedures

MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/02_MARS_1409_POSTHOOK_diagnose_workflow_key_status.sql

@@ -46,6 +46,16 @@ DECLARE
     cMaxPrint CONSTANT NUMBER := 1000;
     vPrinted  NUMBER;
 
+    FUNCTION IS_EXTERNAL_TABLE_EMPTY_ERROR(
+        pSqlCode NUMBER,
+        pSqlErrm VARCHAR2
+    ) RETURN BOOLEAN
+    IS
+    BEGIN
+        RETURN pSqlCode IN (-29913, -29400)
+           OR INSTR(pSqlErrm, 'KUP-05002') > 0;
+    END;
+
 BEGIN
 
     FOR config_rec IN (
@@ -79,7 +89,7 @@ BEGIN
             INTO vTableExists;
         EXCEPTION
             WHEN OTHERS THEN
-                IF SQLCODE = -29913 THEN
+                IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
                     vBucketEmpty := TRUE;
                 ELSE
                     RAISE;
@@ -97,6 +107,7 @@ BEGIN
             vInBothNoKey   := 0;
         ELSE
 
+        BEGIN
             -- ----------------------------------------------------------------
             -- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
             -- ----------------------------------------------------------------
@@ -154,6 +165,22 @@ BEGIN
                 '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME)'
             INTO vInBothNoKey
             USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
+        EXCEPTION
+            WHEN OTHERS THEN
+                IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
+                    vBucketEmpty := TRUE;
+                    vOnlyInBucket := 0;
+                    SELECT COUNT(*) INTO vOnlyInDB
+                    FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
+                    WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
+                      AND  sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
+                    vInBothWithKey := 0;
+                    vInBothNoKey   := 0;
+                    DBMS_OUTPUT.PUT_LINE('  NOTE: ODS bucket became empty/inaccessible during diagnostics for ' || vTableName || '. Falling back to DB-only counts for [B].');
+                ELSE
+                    RAISE;
+                END IF;
+        END;
 
         END IF; -- vBucketEmpty
 
@@ -204,6 +231,16 @@ BEGIN
             vConfigsWithIssues := vConfigsWithIssues + 1;
             DBMS_OUTPUT.PUT_LINE('  [B] Registered files not found in bucket:');
             vPrinted := 0;
+            BEGIN
+                IF vBucketEmpty THEN
+                    OPEN vRefCursor FOR
+                        'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
+                        'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
+                        'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
+                        '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
+                        'ORDER BY sfr.SOURCE_FILE_NAME'
+                        USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
+                ELSE
                     OPEN vRefCursor FOR
                         'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
                         'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
@@ -214,6 +251,23 @@ BEGIN
                         '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME) ' ||
                         'ORDER BY sfr.SOURCE_FILE_NAME'
                         USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
+                END IF;
+            EXCEPTION
+                WHEN OTHERS THEN
+                    IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
+                        vBucketEmpty := TRUE;
+                        DBMS_OUTPUT.PUT_LINE('      NOTE: Skipping ODS anti-join details due to empty/inaccessible external table for ' || vTableName || '.');
+                        OPEN vRefCursor FOR
+                            'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
+                            'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
+                            'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
+                            '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
+                            'ORDER BY sfr.SOURCE_FILE_NAME'
+                            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
+                    ELSE
+                        RAISE;
+                    END IF;
+            END;
             LOOP
                 DECLARE
                     vStatus  VARCHAR2(50);

MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/install_mars1409_posthook.sql

@@ -53,6 +53,43 @@ END;
 /
 WHENEVER SQLERROR CONTINUE
 
+PROMPT
+PROMPT ============================================================================
+PROMPT PREREQUISITE CHECK: Verifying MARS-1409 objects
+PROMPT ============================================================================
+WHENEVER SQLERROR EXIT SQL.SQLCODE
+DECLARE
+        vColCount NUMBER;
+        vTableCount NUMBER;
+BEGIN
+        SELECT COUNT(*)
+            INTO vColCount
+            FROM ALL_TAB_COLUMNS
+         WHERE OWNER = 'CT_MRDS'
+             AND TABLE_NAME = 'A_SOURCE_FILE_RECEIVED'
+             AND COLUMN_NAME = 'A_WORKFLOW_HISTORY_KEY';
+
+        IF vColCount = 0 THEN
+                RAISE_APPLICATION_ERROR(-20001,
+                        'Prerequisite failed: CT_MRDS.A_SOURCE_FILE_RECEIVED.A_WORKFLOW_HISTORY_KEY not found. Install MARS-1409 first (or do not run POSTHOOK after rollback).');
+        END IF;
+
+        SELECT COUNT(*)
+            INTO vTableCount
+            FROM ALL_TABLES
+         WHERE OWNER = 'CT_MRDS'
+             AND TABLE_NAME = 'A_WORKFLOW_HISTORY';
+
+        IF vTableCount = 0 THEN
+                RAISE_APPLICATION_ERROR(-20002,
+                        'Prerequisite failed: CT_MRDS.A_WORKFLOW_HISTORY table not found.');
+        END IF;
+
+        DBMS_OUTPUT.PUT_LINE('OK: Prerequisites satisfied (MARS-1409 schema changes detected).');
+END;
+/
+WHENEVER SQLERROR CONTINUE
+
 PROMPT
 PROMPT ============================================================================
 PROMPT STEP 1: Backfill A_WORKFLOW_HISTORY_KEY for existing records

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb

@@ -58,14 +58,15 @@ AS
    BEGIN
       vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
       CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
-      SELECT count(*) , min(SOURCE_FILE_TYPE)
+      -- LEFT JOIN ensures SOURCE_FILE_TYPE is retrieved from config even when no stats exist yet
+      SELECT count(s.A_SOURCE_FILE_CONFIG_KEY), min(c.SOURCE_FILE_TYPE)
         INTO vCount, vSourceFileType
-        FROM CT_MRDS.A_TABLE_STAT s
-        JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
+        FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
+        LEFT JOIN CT_MRDS.A_TABLE_STAT s
           ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
-       WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
+       WHERE c.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
 
-      IF vCount=0 and vSourceFileType='INPUT' THEN
+      IF vCount = 0 AND vSourceFileType = 'INPUT' THEN
          GATHER_TABLE_STAT(pSourceFileConfigKey);
       END IF;
 
@@ -74,9 +75,13 @@ AS
            INTO vTableStat
            FROM CT_MRDS.A_TABLE_STAT
           WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
---      EXCEPTION
---         WHEN NO_DATA_FOUND THEN
---            
+      EXCEPTION
+         WHEN NO_DATA_FOUND THEN
+            CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
+               'No statistics found in A_TABLE_STAT for config key ' || pSourceFileConfigKey 
+               || ' (SOURCE_FILE_TYPE=' || NVL(vSourceFileType, 'NULL') || '). Cannot proceed with archival.',
+               'ERROR', vParameters);
+            RAISE;
       END;
 
       CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
@@ -171,13 +176,14 @@ AS
          ;
 
          -- Get all files that will be archived into  "vfiles" collection ("regular data files")
-         -- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
+         -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
+         -- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
          BEGIN
             execute immediate vQuery bulk collect into vfiles;
          EXCEPTION
             WHEN OTHERS THEN
-               IF SQLCODE = -29913 THEN
-                  CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Nothing to archive.', 'INFO', vParameters);
+               IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
+                  CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Nothing to archive.', 'INFO', vParameters);
                   CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
                   RETURN;
                ELSE
@@ -185,6 +191,18 @@ AS
                END IF;
          END;
 
+         -- Check if any files match archival criteria
+         IF vfiles.COUNT = 0 THEN
+            CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
+               'No files matching archival criteria found (strategy: ' || vSourceFileConfig.ARCHIVAL_STRATEGY 
+               || ', IS_WORKFLOW_SUCCESS_REQUIRED: ' || vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED || '). Nothing to archive.',
+               'INFO', vParameters);
+            CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
+            RETURN;
+         END IF;
+
+         CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Files matching archival criteria: ' || vfiles.COUNT, 'INFO', vParameters);
+
          -- Start EXPORT "regular data files" to parquet and DROP "csv"
          FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
             dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
@@ -538,13 +556,14 @@ AS
       on t.filename = r.object_name'
       ;
       CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery);
-      -- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
+      -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
+      -- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
       BEGIN
          execute immediate vQuery into vStats;
       EXCEPTION
          WHEN OTHERS THEN
-            IF SQLCODE = -29913 THEN
-               CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Saving zero statistics.', 'INFO', vParameters);
+            IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
+               CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Saving zero statistics.', 'INFO', vParameters);
                vStats.A_SOURCE_FILE_CONFIG_KEY        := pSourceFileConfigKey;
                vStats.TABLE_NAME                      := vTableName;
                vStats.FILE_COUNT                      := 0;

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb

@@ -1634,7 +1634,7 @@ AS
       ,pEncoding                    IN    CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE                        DEFAULT NULL               -- MARS-1049
       ,pIsWorkflowSuccessRequired  IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE  DEFAULT 'Y'                -- MARS-1409
       ,pIsArchiveEnabled           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE             DEFAULT 'N'                -- MARS-828
-      ,pIsKeepInTrash              IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
+      ,pIsKeptInTrash              IN    CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
       ,pArchivalStrategy           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE              DEFAULT 'THRESHOLD_BASED'  -- MARS-828
       ,pMinimumAgeMonths           IN    CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE             DEFAULT 0                  -- MARS-828
    ) IS
@@ -1654,13 +1654,13 @@ AS
                                                                         ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||''''  -- MARS-1049: NOWY
                                                             ,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||''''  -- MARS-1409
                                                               ,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||''''  -- MARS-828
-                                                               ,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||''''  -- MARS-828
+                                                               ,'pIsKeptInTrash => '''||nvl(to_char(pIsKeptInTrash), 'NULL')||''''  -- MARS-828
                                                             ,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||''''  -- MARS-828
                                                             ,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||''''  -- MARS-828
                                                             ));
       ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
       INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
-      VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths);
+      VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths);
       COMMIT;
       ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
    EXCEPTION

MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg

@@ -23,7 +23,7 @@ AS
 
    -- Version History (Latest changes first)
    VERSION_HISTORY     CONSTANT VARCHAR2(4000) := 
-      '3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
+      '3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
       '3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) ||
       '3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) ||
       '3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) ||
@@ -446,13 +446,13 @@ AS
       * @desc     Insert a new record to A_SOURCE_FILE_CONFIG table.
       *           MARS-1049: Added pEncoding parameter for CSV character set specification.
       *           MARS-1409: Added pIsWorkflowSuccessRequired parameter.
-      *           MARS-828:  Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths.
+      *           MARS-828:  Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths.
       * @param    pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
       *                      If NULL, no CHARACTERSET clause is added to external table definitions
       * @param    pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow)
       *                                        'N' = archive regardless of workflow status (bypass for manual/non-DBT sources)
       * @param    pIsArchiveEnabled    - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled
-      * @param    pIsKeepInTrash       - 'Y' = move files to trash before purge; 'N' (default) = purge directly
+      * @param    pIsKeptInTrash       - 'Y' = move files to trash before purge; 'N' (default) = purge directly
       * @param    pArchivalStrategy    - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL
       * @param    pMinimumAgeMonths    - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy)
       * @example  CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
@@ -460,7 +460,7 @@ AS
       *               pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
       *               pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
       *               pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y',
-      *               pIsArchiveEnabled => 'Y', pIsKeepInTrash => 'N',
+      *               pIsArchiveEnabled => 'Y', pIsKeptInTrash => 'N',
       *               pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3
       *           );
    **/
@@ -476,7 +476,7 @@ AS
       ,pEncoding                   IN   CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE                        DEFAULT NULL               -- MARS-1049
       ,pIsWorkflowSuccessRequired  IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE  DEFAULT 'Y'                -- MARS-1409
       ,pIsArchiveEnabled           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE             DEFAULT 'N'                -- MARS-828
-      ,pIsKeepInTrash              IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
+      ,pIsKeptInTrash              IN   CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE               DEFAULT 'Y'                -- MARS-828
       ,pArchivalStrategy           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE              DEFAULT 'THRESHOLD_BASED'  -- MARS-828
       ,pMinimumAgeMonths           IN   CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE             DEFAULT 0                  -- MARS-828
    );

confluence/FILE_ARCHIVER_Guide.md

@@ -288,11 +288,11 @@ WHERE SOURCE_FILE_TYPE = 'INPUT'
 ORDER BY SOURCE_FILE_ID, TABLE_ID;
 ```
 
-### IS_KEEP_IN_TRASH Column
+### IS_KEPT_IN_TRASH Column
 
 Controls TRASH folder retention policy for archived files.
 
-**Column**: `A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
+**Column**: `A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
 
 **Values**:
 - `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED)
@@ -312,7 +312,7 @@ Controls TRASH folder retention policy for archived files.
 ```sql
 -- Production: Keep files in TRASH (recommended)
 UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-SET IS_KEEP_IN_TRASH = 'Y'
+SET IS_KEPT_IN_TRASH = 'Y'
 WHERE SOURCE_FILE_TYPE = 'INPUT'
   AND SOURCE_FILE_ID = 'LM'
   AND TABLE_ID LIKE 'LM_%';
@@ -320,14 +320,14 @@ COMMIT;
 
 -- Test environment: Cleanup TRASH to save storage
 UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-SET IS_KEEP_IN_TRASH = 'N'
+SET IS_KEPT_IN_TRASH = 'N'
 WHERE SOURCE_FILE_TYPE = 'INPUT'
   AND SOURCE_FILE_ID = 'TEST_SOURCE';
 COMMIT;
 
 -- Bulk configuration by source
 UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-SET IS_KEEP_IN_TRASH = 'Y'
+SET IS_KEPT_IN_TRASH = 'Y'
 WHERE SOURCE_FILE_TYPE = 'INPUT'
   AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM');
 COMMIT;
@@ -393,7 +393,7 @@ INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
 **Status Descriptions**:
 - **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket
 - **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket)
-- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEEP_IN_TRASH='N')
+- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEPT_IN_TRASH='N')
 
 **Associated Columns Updated During Archival**:
 ```sql
@@ -435,9 +435,9 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
 2.1 TRASH Subfolder (DATA Bucket - File Retention)
    ├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
    ├─ Stores CSV files after archival to Parquet
-   ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEEP_IN_TRASH config)
+   ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEPT_IN_TRASH config)
    ├─ Enables rollback if archival issues occur
-   └─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEEP_IN_TRASH = 'N')
+   └─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEPT_IN_TRASH = 'N')
 
 3. ARCHIVE Bucket (Long-term Storage)
    ├─ Historical data in Parquet format
@@ -447,24 +447,35 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
 
 **Key Procedures**:
 - `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause
-  - TRASH folder retention controlled by `IS_KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG
+  - TRASH folder retention controlled by `IS_KEPT_IN_TRASH` column in A_SOURCE_FILE_CONFIG
 - `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling
   - **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey`
   - **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey`
   - **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll`
   - **Error Handling**: Continues processing other tables on individual failures
   - **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations)
-  - **Individual TRASH Policy**: Each table's `IS_KEEP_IN_TRASH` setting applied independently
+  - **Individual TRASH Policy**: Each table's `IS_KEPT_IN_TRASH` setting applied independently
   - **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables
-- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy
-- `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic
-- `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll)` - Batch statistics with 3-level granularity
-- `RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey, pSourceKey, pRestoreAll)` - Restore archived files from TRASH
-- `PURGE_TRASH_FOLDER(pSourceFileConfigKey, pSourceKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity
+- `GATHER_TABLE_STAT(pSourceFileConfigKey)` - Calculates archival statistics using strategy logic
+- `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll, pOnlyEnabled)` - Batch statistics with 3-level granularity
+  - `pOnlyEnabled` (DEFAULT TRUE): When TRUE, only processes tables with `IS_ARCHIVE_ENABLED='Y'`
+- `RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey, pSourceFileConfigKey, pRestoreAll)` - Restore archived files from TRASH
+- `PURGE_TRASH_FOLDER(pSourceFileReceivedKey, pSourceFileConfigKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity
+- `GET_VERSION` / `GET_BUILD_INFO` / `GET_VERSION_HISTORY` - Package version and metadata
+
+**Function Wrappers (Python Integration)**:
+
+All key procedures have `FN_*` function overloads returning `PLS_INTEGER` (SQLCODE: 0=success, error code on failure) for Python library integration:
+- `FN_ARCHIVE_TABLE_DATA`, `FN_GATHER_TABLE_STAT`, `FN_ARCHIVE_ALL`, `FN_GATHER_TABLE_STAT_ALL`
+- `RESTORE_FILE_FROM_TRASH` and `PURGE_TRASH_FOLDER` also have function overloads returning PLS_INTEGER
+
+**Internal Functions** (not callable externally):
+- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy (private)
+- `GET_TABLE_STAT` - Retrieves or auto-generates table statistics (private)
 
 **Archival Execution**:
 ```sql
--- Single table archival (TRASH retention controlled by IS_KEEP_IN_TRASH config)
+-- Single table archival (TRASH retention controlled by IS_KEPT_IN_TRASH config)
 BEGIN
     CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA(
         pSourceFileConfigKey => vSourceFileConfigKey
@@ -500,7 +511,7 @@ END;
 - Data matching criteria moved from ODS to ARCHIVE bucket
 - CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/)
 - Parquet format with Hive-style partitioning applied to ARCHIVE bucket
-- TRASH retention controlled by IS_KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG
+- TRASH retention controlled by IS_KEPT_IN_TRASH column in A_SOURCE_FILE_CONFIG
 
 ### Automatic Rollback Mechanism
 
@@ -510,7 +521,7 @@ FILE_ARCHIVER implements **automatic rollback** to ensure data integrity if arch
 1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket
 2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED'
 3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket)
-4. **Optional Cleanup**: If IS_KEEP_IN_TRASH='N', files deleted from TRASH
+4. **Optional Cleanup**: If IS_KEPT_IN_TRASH='N', files deleted from TRASH
 
 **Automatic Rollback Trigger**:
 If **any error occurs** during step 3 (Move to TRASH), the system:
@@ -725,7 +736,7 @@ UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
 SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
     MINIMUM_AGE_MONTHS = 6,
     IS_ARCHIVE_ENABLED = 'Y',      -- Enable archival
-    IS_KEEP_IN_TRASH = 'Y'         -- Keep files in TRASH for safety
+    IS_KEPT_IN_TRASH = 'Y'         -- Keep files in TRASH for safety
 WHERE SOURCE_FILE_TYPE = 'INPUT'
   AND SOURCE_FILE_ID = 'CSDB'
   AND TABLE_ID = 'CSDB_DEBT';
@@ -739,7 +750,7 @@ COMMIT;
 
 -- Configure TRASH cleanup for test environment
 UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG 
-SET IS_KEEP_IN_TRASH = 'N'  -- Delete files from TRASH after archival
+SET IS_KEPT_IN_TRASH = 'N'  -- Delete files from TRASH after archival
 WHERE SOURCE_FILE_TYPE = 'INPUT'
   AND SOURCE_FILE_ID = 'TEST_SOURCE';
 COMMIT;
@@ -751,7 +762,7 @@ SELECT
     ARCHIVAL_STRATEGY,
     MINIMUM_AGE_MONTHS,
     IS_ARCHIVE_ENABLED,
-    IS_KEEP_IN_TRASH
+    IS_KEPT_IN_TRASH
 FROM CT_MRDS.A_SOURCE_FILE_CONFIG
 WHERE SOURCE_FILE_TYPE = 'INPUT'
 ORDER BY SOURCE_FILE_ID, TABLE_ID;
@@ -759,12 +770,12 @@ ORDER BY SOURCE_FILE_ID, TABLE_ID;
 -- Summary by archival status
 SELECT 
     IS_ARCHIVE_ENABLED,
-    IS_KEEP_IN_TRASH,
+    IS_KEPT_IN_TRASH,
     COUNT(*) AS TABLE_COUNT
 FROM CT_MRDS.A_SOURCE_FILE_CONFIG
 WHERE SOURCE_FILE_TYPE = 'INPUT'
-GROUP BY IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH
-ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEEP_IN_TRASH DESC;
+GROUP BY IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH
+ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEPT_IN_TRASH DESC;
 ```
 
 ## Release 01 Configuration
@@ -948,7 +959,7 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
 
 **Symptoms**: Files not deleted from TRASH after archival
 
-**Cause**: Configuration has `IS_KEEP_IN_TRASH='Y'` (retain files in TRASH)
+**Cause**: Configuration has `IS_KEPT_IN_TRASH='Y'` (retain files in TRASH)
 
 **Verification**:
 ```sql
@@ -956,8 +967,8 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
 SELECT 
     SOURCE_FILE_ID,
     TABLE_ID,
-    IS_KEEP_IN_TRASH,
-    CASE IS_KEEP_IN_TRASH
+    IS_KEPT_IN_TRASH,
+    CASE IS_KEPT_IN_TRASH
         WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)'
         WHEN 'N' THEN 'Files DELETED immediately after archival'
     END AS TRASH_BEHAVIOR
@@ -969,7 +980,7 @@ WHERE TABLE_ID = 'YOUR_TABLE';
 ```sql
 -- Option A: Change configuration to auto-delete (permanent change)
 UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-SET IS_KEEP_IN_TRASH = 'N'  -- Auto-delete from TRASH after archival
+SET IS_KEPT_IN_TRASH = 'N'  -- Auto-delete from TRASH after archival
 WHERE TABLE_ID = 'YOUR_TABLE';
 COMMIT;
 
@@ -1072,13 +1083,15 @@ BEGIN
     WHERE TABLE_ID = 'YOUR_TABLE'
       AND ROWNUM = 1;
     
-    vWhereClause := CT_MRDS.FILE_ARCHIVER.GET_ARCHIVAL_WHERE_CLAUSE(vConfig);
-    DBMS_OUTPUT.PUT_LINE('WHERE Clause: ' || vWhereClause);
+    -- Note: GET_ARCHIVAL_WHERE_CLAUSE is a private function.
+    -- To test WHERE clause logic, check A_PROCESS_LOG entries from ARCHIVE_TABLE_DATA
+    -- which logs the generated WHERE clause at INFO level.
+    DBMS_OUTPUT.PUT_LINE('Config: ' || vConfig.ARCHIVAL_STRATEGY || ', MIN_AGE=' || vConfig.MINIMUM_AGE_MONTHS);
 END;
 /
 ```
 
-#### Issue 3: Package Compilation Errors After Upgrade
+#### Issue 7: Package Compilation Errors After Upgrade
 
 **Symptoms**: FILE_ARCHIVER package shows INVALID status
 
@@ -1132,7 +1145,7 @@ SELECT
     SFR.FILE_SIZE_BYTES,
     SFR.UPDATED_AT AS ARCHIVED_AT,
     TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH,
-    SFC.IS_KEEP_IN_TRASH AS TRASH_POLICY
+    SFC.IS_KEPT_IN_TRASH AS TRASH_POLICY
 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
 JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
 WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
@@ -1148,17 +1161,17 @@ SELECT
     SFC.TABLE_ID,
     SFC.ARCHIVAL_STRATEGY,
     SFC.IS_ARCHIVE_ENABLED,
-    SFC.IS_KEEP_IN_TRASH,
+    SFC.IS_KEPT_IN_TRASH,
     COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE,
     COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH,
     COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED,
-    MAX(SFR.UPDATED_AT) FILTER (WHERE SFR.PROCESSING_STATUS LIKE 'ARCHIVED%') AS LAST_ARCHIVAL
+    MAX(CASE WHEN SFR.PROCESSING_STATUS LIKE 'ARCHIVED%' THEN SFR.UPDATED_AT END) AS LAST_ARCHIVAL
 FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC
 LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY
 WHERE SFC.SOURCE_FILE_TYPE = 'INPUT'
 GROUP BY 
     SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, 
-    SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEEP_IN_TRASH
+    SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEPT_IN_TRASH
 ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID;
 ```
 
@@ -1218,11 +1231,11 @@ SELECT
     ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB,
     MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH,
     MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH,
-    SFC.IS_KEEP_IN_TRASH AS POLICY
+    SFC.IS_KEPT_IN_TRASH AS POLICY
 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
 JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
 WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
-GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEEP_IN_TRASH
+GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEPT_IN_TRASH
 ORDER BY SIZE_GB DESC;
 ```
 
@@ -1241,7 +1254,7 @@ ORDER BY SIZE_GB DESC;
 ### v3.3.0 (2026-02-11)
 - **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA
 - Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control
-- Added `IS_KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
+- Added `IS_KEPT_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
 - Added batch procedures with 3-level granularity (config/source/all):
   - ARCHIVE_ALL - Batch archival procedure
   - GATHER_TABLE_STAT_ALL - Batch statistics procedure
@@ -1392,7 +1405,7 @@ ORDER BY SIZE_GB DESC;
 
 ### TRASH Folder Retention Best Practices
 
-1. **Default Behavior (IS_KEEP_IN_TRASH = 'Y' - Recommended)**:
+1. **Default Behavior (IS_KEPT_IN_TRASH = 'Y' - Recommended)**:
    - Keeps CSV files in TRASH folder after archival
    - Provides safety net for rollback if archival issues occur
    - Supports compliance and audit requirements
@@ -1401,11 +1414,11 @@ ORDER BY SIZE_GB DESC;
    - Configuration:
      ```sql
      UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-     SET IS_KEEP_IN_TRASH = 'Y'
+     SET IS_KEPT_IN_TRASH = 'Y'
      WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
      ```
 
-2. **TRASH Cleanup (IS_KEEP_IN_TRASH = 'N')**:
+2. **TRASH Cleanup (IS_KEPT_IN_TRASH = 'N')**:
    - Deletes CSV files from TRASH folder after successful archival
    - Reduces storage costs in DATA bucket
    - Status: ARCHIVED_AND_PURGED
@@ -1413,7 +1426,7 @@ ORDER BY SIZE_GB DESC;
    - Configuration:
      ```sql
      UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
-     SET IS_KEEP_IN_TRASH = 'N'
+     SET IS_KEPT_IN_TRASH = 'N'
      WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
      ```
 
@@ -1423,7 +1436,7 @@ ORDER BY SIZE_GB DESC;
    SELECT 
        SOURCE_FILE_NAME,
        PROCESSING_STATUS,
-       ARCH_FILE_NAME,
+       ARCH_PATH,
        PARTITION_YEAR,
        PARTITION_MONTH
    FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
@@ -1446,3 +1459,4 @@ ORDER BY SIZE_GB DESC;
    ```
 
 
+