admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: admin:2ea708a69401845e9e339c0f2c2b25fdfcba34dd
Branches Tags
admin:main admin:develop
...
compare: admin:a35e28042becd202292f23a8e2e83a17b78d9060
Branches Tags
admin:main admin:develop

5 Commits
2ea708a694 ... a35e28042b

Author SHA1 Message Date
Grzegorz Michalski
a35e28042b feat(FILE_ARCHIVER): Improve archival logic and error handling in FILE_ARCHIVER procedures 2026-03-23 11:48:37 +01:00
Grzegorz Michalski
92feb95ae0 feat(FILE_ARCHIVER): Enhance documentation with new function details and clarify private functions 2026-03-20 13:37:05 +01:00
Grzegorz Michalski
74b8857096 feat(FILE_ARCHIVER): Rename IS_KEEP_IN_TRASH to IS_KEPT_IN_TRASH for consistency in configuration 2026-03-20 13:26:37 +01:00
Grzegorz Michalski
24997b1583 feat(MARS-1409): Add prerequisite checks for MARS-1409 objects in installation script 2026-03-20 13:13:26 +01:00
Grzegorz Michalski
eb9b2bc38b feat(FILE_MANAGER): Rename pIsKeepInTrash to pIsKeptInTrash for consistency in parameter naming 2026-03-19 13:29:51 +01:00
6 changed files with 254 additions and 130 deletions
Expand all files Collapse all files

184
MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/02_MARS_1409_POSTHOOK_diagnose_workflow_key_status.sql
View File

@@ -46,6 +46,16 @@ DECLARE
cMaxPrint CONSTANT NUMBER := 1000; cMaxPrint CONSTANT NUMBER := 1000;
vPrinted NUMBER; vPrinted NUMBER;
FUNCTION IS_EXTERNAL_TABLE_EMPTY_ERROR(
pSqlCode NUMBER,
pSqlErrm VARCHAR2
) RETURN BOOLEAN
IS
BEGIN
RETURN pSqlCode IN (-29913, -29400)
OR INSTR(pSqlErrm, 'KUP-05002') > 0;
END;
BEGIN BEGIN
FOR config_rec IN ( FOR config_rec IN (
@@ -79,7 +89,7 @@ BEGIN
INTO vTableExists; INTO vTableExists;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
IF SQLCODE = -29913 THEN IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
vBucketEmpty := TRUE; vBucketEmpty := TRUE;
ELSE ELSE
RAISE; RAISE;
@@ -97,63 +107,80 @@ BEGIN
vInBothNoKey := 0; vInBothNoKey := 0;
ELSE ELSE
-- ---------------------------------------------------------------- BEGIN
-- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED -- ----------------------------------------------------------------
-- ---------------------------------------------------------------- -- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
EXECUTE IMMEDIATE -- ----------------------------------------------------------------
'SELECT COUNT(DISTINCT t.file$name) ' || EXECUTE IMMEDIATE
'FROM ' || vTableName || ' t ' || 'SELECT COUNT(DISTINCT t.file$name) ' ||
'WHERE t.file$name IS NOT NULL ' || 'FROM ' || vTableName || ' t ' ||
' AND NOT EXISTS ( ' || 'WHERE t.file$name IS NOT NULL ' ||
' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' || ' AND NOT EXISTS ( ' ||
' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' || ' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1)' ' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' ||
INTO vOnlyInBucket ' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1)'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY; INTO vOnlyInBucket
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
-- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket -- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
EXECUTE IMMEDIATE EXECUTE IMMEDIATE
'SELECT COUNT(*) ' || 'SELECT COUNT(*) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' || 'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' || 'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' || ' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND NOT EXISTS ( ' || ' AND NOT EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' || ' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)' ' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vOnlyInDB INTO vOnlyInDB
USING config_rec.A_SOURCE_FILE_CONFIG_KEY; USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
-- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL -- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
EXECUTE IMMEDIATE EXECUTE IMMEDIATE
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' || 'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' || 'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' || 'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NOT NULL ' || ' AND sfr.A_WORKFLOW_HISTORY_KEY IS NOT NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' || ' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' || ' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' || ' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)' ' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vInBothWithKey INTO vInBothWithKey
USING config_rec.A_SOURCE_FILE_CONFIG_KEY; USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
-- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL -- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL
-- ---------------------------------------------------------------- -- ----------------------------------------------------------------
EXECUTE IMMEDIATE EXECUTE IMMEDIATE
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' || 'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' || 'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' || 'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' || ' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' || ' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' || ' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' || ' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)' ' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vInBothNoKey INTO vInBothNoKey
USING config_rec.A_SOURCE_FILE_CONFIG_KEY; USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
EXCEPTION
WHEN OTHERS THEN
IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
vBucketEmpty := TRUE;
vOnlyInBucket := 0;
SELECT COUNT(*) INTO vOnlyInDB
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
AND sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
vInBothWithKey := 0;
vInBothNoKey := 0;
DBMS_OUTPUT.PUT_LINE(' NOTE: ODS bucket became empty/inaccessible during diagnostics for ' || vTableName || '. Falling back to DB-only counts for [B].');
ELSE
RAISE;
END IF;
END;
END IF; -- vBucketEmpty END IF; -- vBucketEmpty
@@ -204,16 +231,43 @@ BEGIN
vConfigsWithIssues := vConfigsWithIssues + 1; vConfigsWithIssues := vConfigsWithIssues + 1;
DBMS_OUTPUT.PUT_LINE(' [B] Registered files not found in bucket:'); DBMS_OUTPUT.PUT_LINE(' [B] Registered files not found in bucket:');
vPrinted := 0; vPrinted := 0;
OPEN vRefCursor FOR BEGIN
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' || IF vBucketEmpty THEN
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' || OPEN vRefCursor FOR
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' || 'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' || 'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
' AND NOT EXISTS ( ' || 'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' SELECT 1 FROM ' || vTableName || ' t ' || ' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' || 'ORDER BY sfr.SOURCE_FILE_NAME'
'ORDER BY sfr.SOURCE_FILE_NAME' USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
USING config_rec.A_SOURCE_FILE_CONFIG_KEY; ELSE
OPEN vRefCursor FOR
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND NOT EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' ||
'ORDER BY sfr.SOURCE_FILE_NAME'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
END IF;
EXCEPTION
WHEN OTHERS THEN
IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
vBucketEmpty := TRUE;
DBMS_OUTPUT.PUT_LINE(' NOTE: Skipping ODS anti-join details due to empty/inaccessible external table for ' || vTableName || '.');
OPEN vRefCursor FOR
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
'ORDER BY sfr.SOURCE_FILE_NAME'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
ELSE
RAISE;
END IF;
END;
LOOP LOOP
DECLARE DECLARE
vStatus VARCHAR2(50); vStatus VARCHAR2(50);

37
MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/install_mars1409_posthook.sql
View File

@@ -53,6 +53,43 @@ END;
/ /
WHENEVER SQLERROR CONTINUE WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT ============================================================================
PROMPT PREREQUISITE CHECK: Verifying MARS-1409 objects
PROMPT ============================================================================
WHENEVER SQLERROR EXIT SQL.SQLCODE
DECLARE
vColCount NUMBER;
vTableCount NUMBER;
BEGIN
SELECT COUNT(*)
INTO vColCount
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_SOURCE_FILE_RECEIVED'
AND COLUMN_NAME = 'A_WORKFLOW_HISTORY_KEY';
IF vColCount = 0 THEN
RAISE_APPLICATION_ERROR(-20001,
'Prerequisite failed: CT_MRDS.A_SOURCE_FILE_RECEIVED.A_WORKFLOW_HISTORY_KEY not found. Install MARS-1409 first (or do not run POSTHOOK after rollback).');
END IF;
SELECT COUNT(*)
INTO vTableCount
FROM ALL_TABLES
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_WORKFLOW_HISTORY';
IF vTableCount = 0 THEN
RAISE_APPLICATION_ERROR(-20002,
'Prerequisite failed: CT_MRDS.A_WORKFLOW_HISTORY table not found.');
END IF;
DBMS_OUTPUT.PUT_LINE('OK: Prerequisites satisfied (MARS-1409 schema changes detected).');
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 1: Backfill A_WORKFLOW_HISTORY_KEY for existing records PROMPT STEP 1: Backfill A_WORKFLOW_HISTORY_KEY for existing records

47
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb
View File

@@ -58,14 +58,15 @@ AS
BEGIN BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT count(*) , min(SOURCE_FILE_TYPE) -- LEFT JOIN ensures SOURCE_FILE_TYPE is retrieved from config even when no stats exist yet
SELECT count(s.A_SOURCE_FILE_CONFIG_KEY), min(c.SOURCE_FILE_TYPE)
INTO vCount, vSourceFileType INTO vCount, vSourceFileType
FROM CT_MRDS.A_TABLE_STAT s FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c LEFT JOIN CT_MRDS.A_TABLE_STAT s
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; WHERE c.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
IF vCount=0 and vSourceFileType='INPUT' THEN IF vCount = 0 AND vSourceFileType = 'INPUT' THEN
GATHER_TABLE_STAT(pSourceFileConfigKey); GATHER_TABLE_STAT(pSourceFileConfigKey);
END IF; END IF;
@@ -74,9 +75,13 @@ AS
INTO vTableStat INTO vTableStat
FROM CT_MRDS.A_TABLE_STAT FROM CT_MRDS.A_TABLE_STAT
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- EXCEPTION EXCEPTION
-- WHEN NO_DATA_FOUND THEN WHEN NO_DATA_FOUND THEN
-- CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'No statistics found in A_TABLE_STAT for config key ' || pSourceFileConfigKey
|| ' (SOURCE_FILE_TYPE=' || NVL(vSourceFileType, 'NULL') || '). Cannot proceed with archival.',
'ERROR', vParameters);
RAISE;
END; END;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
@@ -171,13 +176,14 @@ AS
; ;
-- Get all files that will be archived into "vfiles" collection ("regular data files") -- Get all files that will be archived into "vfiles" collection ("regular data files")
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location) -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
BEGIN BEGIN
execute immediate vQuery bulk collect into vfiles; execute immediate vQuery bulk collect into vfiles;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
IF SQLCODE = -29913 THEN IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Nothing to archive.', 'INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Nothing to archive.', 'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN; RETURN;
ELSE ELSE
@@ -185,6 +191,18 @@ AS
END IF; END IF;
END; END;
-- Check if any files match archival criteria
IF vfiles.COUNT = 0 THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'No files matching archival criteria found (strategy: ' || vSourceFileConfig.ARCHIVAL_STRATEGY
|| ', IS_WORKFLOW_SUCCESS_REQUIRED: ' || vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED || '). Nothing to archive.',
'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN;
END IF;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Files matching archival criteria: ' || vfiles.COUNT, 'INFO', vParameters);
-- Start EXPORT "regular data files" to parquet and DROP "csv" -- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
@@ -538,13 +556,14 @@ AS
on t.filename = r.object_name' on t.filename = r.object_name'
; ;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery);
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location) -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
BEGIN BEGIN
execute immediate vQuery into vStats; execute immediate vQuery into vStats;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
IF SQLCODE = -29913 THEN IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Saving zero statistics.', 'INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Saving zero statistics.', 'INFO', vParameters);
vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey; vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey;
vStats.TABLE_NAME := vTableName; vStats.TABLE_NAME := vTableName;
vStats.FILE_COUNT := 0; vStats.FILE_COUNT := 0;

6
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb
View File

@@ -1634,7 +1634,7 @@ AS
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049 ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409 ,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828 ,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828 ,pIsKeptInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828 ,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828 ,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
) IS ) IS
@@ -1654,13 +1654,13 @@ AS
,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY
,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409 ,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409
,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828 ,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828
,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||'''' -- MARS-828 ,'pIsKeptInTrash => '''||nvl(to_char(pIsKeptInTrash), 'NULL')||'''' -- MARS-828
,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828 ,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828
,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828 ,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS) INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths); VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths);
COMMIT; COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION EXCEPTION

10
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg
View File

@@ -23,7 +23,7 @@ AS
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) || '3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
'3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) || '3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) ||
'3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) || '3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) ||
'3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) || '3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) ||
@@ -446,13 +446,13 @@ AS
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table. * @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
* MARS-1049: Added pEncoding parameter for CSV character set specification. * MARS-1049: Added pEncoding parameter for CSV character set specification.
* MARS-1409: Added pIsWorkflowSuccessRequired parameter. * MARS-1409: Added pIsWorkflowSuccessRequired parameter.
* MARS-828: Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths. * MARS-828: Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths.
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2') * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
* If NULL, no CHARACTERSET clause is added to external table definitions * If NULL, no CHARACTERSET clause is added to external table definitions
* @param pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow) * @param pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow)
* 'N' = archive regardless of workflow status (bypass for manual/non-DBT sources) * 'N' = archive regardless of workflow status (bypass for manual/non-DBT sources)
* @param pIsArchiveEnabled - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled * @param pIsArchiveEnabled - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled
* @param pIsKeepInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly * @param pIsKeptInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly
* @param pArchivalStrategy - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL * @param pArchivalStrategy - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL
* @param pMinimumAgeMonths - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy) * @param pMinimumAgeMonths - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy)
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( * @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
@@ -460,7 +460,7 @@ AS
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS', * pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS', * pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
* pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y', * pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y',
* pIsArchiveEnabled => 'Y', pIsKeepInTrash => 'N', * pIsArchiveEnabled => 'Y', pIsKeptInTrash => 'N',
* pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3 * pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3
* ); * );
**/ **/
@@ -476,7 +476,7 @@ AS
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049 ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409 ,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828 ,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828 ,pIsKeptInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828 ,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828 ,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
); );

100
confluence/FILE_ARCHIVER_Guide.md
View File

@@ -288,11 +288,11 @@ WHERE SOURCE_FILE_TYPE = 'INPUT'
ORDER BY SOURCE_FILE_ID, TABLE_ID; ORDER BY SOURCE_FILE_ID, TABLE_ID;
``` ```
### IS_KEEP_IN_TRASH Column ### IS_KEPT_IN_TRASH Column
Controls TRASH folder retention policy for archived files. Controls TRASH folder retention policy for archived files.
**Column**: `A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL) **Column**: `A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
**Values**: **Values**:
- `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED) - `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED)
@@ -312,7 +312,7 @@ Controls TRASH folder retention policy for archived files.
```sql ```sql
-- Production: Keep files in TRASH (recommended) -- Production: Keep files in TRASH (recommended)
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'Y' SET IS_KEPT_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'LM' AND SOURCE_FILE_ID = 'LM'
AND TABLE_ID LIKE 'LM_%'; AND TABLE_ID LIKE 'LM_%';
@@ -320,14 +320,14 @@ COMMIT;
-- Test environment: Cleanup TRASH to save storage -- Test environment: Cleanup TRASH to save storage
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'N' SET IS_KEPT_IN_TRASH = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'TEST_SOURCE'; AND SOURCE_FILE_ID = 'TEST_SOURCE';
COMMIT; COMMIT;
-- Bulk configuration by source -- Bulk configuration by source
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'Y' SET IS_KEPT_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM'); AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM');
COMMIT; COMMIT;
@@ -393,7 +393,7 @@ INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
**Status Descriptions**: **Status Descriptions**:
- **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket - **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket
- **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket) - **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket)
- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEEP_IN_TRASH='N') - **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEPT_IN_TRASH='N')
**Associated Columns Updated During Archival**: **Associated Columns Updated During Archival**:
```sql ```sql
@@ -435,9 +435,9 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
2.1 TRASH Subfolder (DATA Bucket - File Retention) 2.1 TRASH Subfolder (DATA Bucket - File Retention)
├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME) ├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
├─ Stores CSV files after archival to Parquet ├─ Stores CSV files after archival to Parquet
├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEEP_IN_TRASH config) ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEPT_IN_TRASH config)
├─ Enables rollback if archival issues occur ├─ Enables rollback if archival issues occur
└─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEEP_IN_TRASH = 'N') └─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEPT_IN_TRASH = 'N')
3. ARCHIVE Bucket (Long-term Storage) 3. ARCHIVE Bucket (Long-term Storage)
├─ Historical data in Parquet format ├─ Historical data in Parquet format
@@ -447,24 +447,35 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
**Key Procedures**: **Key Procedures**:
- `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause - `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause
- TRASH folder retention controlled by `IS_KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG - TRASH folder retention controlled by `IS_KEPT_IN_TRASH` column in A_SOURCE_FILE_CONFIG
- `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling - `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling
- **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey` - **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey`
- **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey` - **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey`
- **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll` - **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll`
- **Error Handling**: Continues processing other tables on individual failures - **Error Handling**: Continues processing other tables on individual failures
- **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations) - **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations)
- **Individual TRASH Policy**: Each table's `IS_KEEP_IN_TRASH` setting applied independently - **Individual TRASH Policy**: Each table's `IS_KEPT_IN_TRASH` setting applied independently
- **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables - **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables
- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy - `GATHER_TABLE_STAT(pSourceFileConfigKey)` - Calculates archival statistics using strategy logic
- `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic - `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll, pOnlyEnabled)` - Batch statistics with 3-level granularity
- `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll)` - Batch statistics with 3-level granularity - `pOnlyEnabled` (DEFAULT TRUE): When TRUE, only processes tables with `IS_ARCHIVE_ENABLED='Y'`
- `RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey, pSourceKey, pRestoreAll)` - Restore archived files from TRASH - `RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey, pSourceFileConfigKey, pRestoreAll)` - Restore archived files from TRASH
- `PURGE_TRASH_FOLDER(pSourceFileConfigKey, pSourceKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity - `PURGE_TRASH_FOLDER(pSourceFileReceivedKey, pSourceFileConfigKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity
- `GET_VERSION` / `GET_BUILD_INFO` / `GET_VERSION_HISTORY` - Package version and metadata
**Function Wrappers (Python Integration)**:
All key procedures have `FN_*` function overloads returning `PLS_INTEGER` (SQLCODE: 0=success, error code on failure) for Python library integration:
- `FN_ARCHIVE_TABLE_DATA`, `FN_GATHER_TABLE_STAT`, `FN_ARCHIVE_ALL`, `FN_GATHER_TABLE_STAT_ALL`
- `RESTORE_FILE_FROM_TRASH` and `PURGE_TRASH_FOLDER` also have function overloads returning PLS_INTEGER
**Internal Functions** (not callable externally):
- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy (private)
- `GET_TABLE_STAT` - Retrieves or auto-generates table statistics (private)
**Archival Execution**: **Archival Execution**:
```sql ```sql
-- Single table archival (TRASH retention controlled by IS_KEEP_IN_TRASH config) -- Single table archival (TRASH retention controlled by IS_KEPT_IN_TRASH config)
BEGIN BEGIN
CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA( CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA(
pSourceFileConfigKey => vSourceFileConfigKey pSourceFileConfigKey => vSourceFileConfigKey
@@ -500,7 +511,7 @@ END;
- Data matching criteria moved from ODS to ARCHIVE bucket - Data matching criteria moved from ODS to ARCHIVE bucket
- CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/) - CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/)
- Parquet format with Hive-style partitioning applied to ARCHIVE bucket - Parquet format with Hive-style partitioning applied to ARCHIVE bucket
- TRASH retention controlled by IS_KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG - TRASH retention controlled by IS_KEPT_IN_TRASH column in A_SOURCE_FILE_CONFIG
### Automatic Rollback Mechanism ### Automatic Rollback Mechanism
@@ -510,7 +521,7 @@ FILE_ARCHIVER implements **automatic rollback** to ensure data integrity if arch
1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket 1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket
2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED' 2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED'
3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket) 3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket)
4. **Optional Cleanup**: If IS_KEEP_IN_TRASH='N', files deleted from TRASH 4. **Optional Cleanup**: If IS_KEPT_IN_TRASH='N', files deleted from TRASH
**Automatic Rollback Trigger**: **Automatic Rollback Trigger**:
If **any error occurs** during step 3 (Move to TRASH), the system: If **any error occurs** during step 3 (Move to TRASH), the system:
@@ -725,7 +736,7 @@ UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
MINIMUM_AGE_MONTHS = 6, MINIMUM_AGE_MONTHS = 6,
IS_ARCHIVE_ENABLED = 'Y', -- Enable archival IS_ARCHIVE_ENABLED = 'Y', -- Enable archival
IS_KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety IS_KEPT_IN_TRASH = 'Y' -- Keep files in TRASH for safety
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'CSDB' AND SOURCE_FILE_ID = 'CSDB'
AND TABLE_ID = 'CSDB_DEBT'; AND TABLE_ID = 'CSDB_DEBT';
@@ -739,7 +750,7 @@ COMMIT;
-- Configure TRASH cleanup for test environment -- Configure TRASH cleanup for test environment
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival SET IS_KEPT_IN_TRASH = 'N' -- Delete files from TRASH after archival
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'TEST_SOURCE'; AND SOURCE_FILE_ID = 'TEST_SOURCE';
COMMIT; COMMIT;
@@ -751,7 +762,7 @@ SELECT
ARCHIVAL_STRATEGY, ARCHIVAL_STRATEGY,
MINIMUM_AGE_MONTHS, MINIMUM_AGE_MONTHS,
IS_ARCHIVE_ENABLED, IS_ARCHIVE_ENABLED,
IS_KEEP_IN_TRASH IS_KEPT_IN_TRASH
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
ORDER BY SOURCE_FILE_ID, TABLE_ID; ORDER BY SOURCE_FILE_ID, TABLE_ID;
@@ -759,12 +770,12 @@ ORDER BY SOURCE_FILE_ID, TABLE_ID;
-- Summary by archival status -- Summary by archival status
SELECT SELECT
IS_ARCHIVE_ENABLED, IS_ARCHIVE_ENABLED,
IS_KEEP_IN_TRASH, IS_KEPT_IN_TRASH,
COUNT(*) AS TABLE_COUNT COUNT(*) AS TABLE_COUNT
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
GROUP BY IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH GROUP BY IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH
ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEEP_IN_TRASH DESC; ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEPT_IN_TRASH DESC;
``` ```
## Release 01 Configuration ## Release 01 Configuration
@@ -948,7 +959,7 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
**Symptoms**: Files not deleted from TRASH after archival **Symptoms**: Files not deleted from TRASH after archival
**Cause**: Configuration has `IS_KEEP_IN_TRASH='Y'` (retain files in TRASH) **Cause**: Configuration has `IS_KEPT_IN_TRASH='Y'` (retain files in TRASH)
**Verification**: **Verification**:
```sql ```sql
@@ -956,8 +967,8 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
SELECT SELECT
SOURCE_FILE_ID, SOURCE_FILE_ID,
TABLE_ID, TABLE_ID,
IS_KEEP_IN_TRASH, IS_KEPT_IN_TRASH,
CASE IS_KEEP_IN_TRASH CASE IS_KEPT_IN_TRASH
WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)' WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)'
WHEN 'N' THEN 'Files DELETED immediately after archival' WHEN 'N' THEN 'Files DELETED immediately after archival'
END AS TRASH_BEHAVIOR END AS TRASH_BEHAVIOR
@@ -969,7 +980,7 @@ WHERE TABLE_ID = 'YOUR_TABLE';
```sql ```sql
-- Option A: Change configuration to auto-delete (permanent change) -- Option A: Change configuration to auto-delete (permanent change)
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival SET IS_KEPT_IN_TRASH = 'N' -- Auto-delete from TRASH after archival
WHERE TABLE_ID = 'YOUR_TABLE'; WHERE TABLE_ID = 'YOUR_TABLE';
COMMIT; COMMIT;
@@ -1072,13 +1083,15 @@ BEGIN
WHERE TABLE_ID = 'YOUR_TABLE' WHERE TABLE_ID = 'YOUR_TABLE'
AND ROWNUM = 1; AND ROWNUM = 1;
vWhereClause := CT_MRDS.FILE_ARCHIVER.GET_ARCHIVAL_WHERE_CLAUSE(vConfig); -- Note: GET_ARCHIVAL_WHERE_CLAUSE is a private function.
DBMS_OUTPUT.PUT_LINE('WHERE Clause: ' || vWhereClause); -- To test WHERE clause logic, check A_PROCESS_LOG entries from ARCHIVE_TABLE_DATA
-- which logs the generated WHERE clause at INFO level.
DBMS_OUTPUT.PUT_LINE('Config: ' || vConfig.ARCHIVAL_STRATEGY || ', MIN_AGE=' || vConfig.MINIMUM_AGE_MONTHS);
END; END;
/ /
``` ```
#### Issue 3: Package Compilation Errors After Upgrade #### Issue 7: Package Compilation Errors After Upgrade
**Symptoms**: FILE_ARCHIVER package shows INVALID status **Symptoms**: FILE_ARCHIVER package shows INVALID status
@@ -1132,7 +1145,7 @@ SELECT
SFR.FILE_SIZE_BYTES, SFR.FILE_SIZE_BYTES,
SFR.UPDATED_AT AS ARCHIVED_AT, SFR.UPDATED_AT AS ARCHIVED_AT,
TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH, TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH,
SFC.IS_KEEP_IN_TRASH AS TRASH_POLICY SFC.IS_KEPT_IN_TRASH AS TRASH_POLICY
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
@@ -1148,17 +1161,17 @@ SELECT
SFC.TABLE_ID, SFC.TABLE_ID,
SFC.ARCHIVAL_STRATEGY, SFC.ARCHIVAL_STRATEGY,
SFC.IS_ARCHIVE_ENABLED, SFC.IS_ARCHIVE_ENABLED,
SFC.IS_KEEP_IN_TRASH, SFC.IS_KEPT_IN_TRASH,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED,
MAX(SFR.UPDATED_AT) FILTER (WHERE SFR.PROCESSING_STATUS LIKE 'ARCHIVED%') AS LAST_ARCHIVAL MAX(CASE WHEN SFR.PROCESSING_STATUS LIKE 'ARCHIVED%' THEN SFR.UPDATED_AT END) AS LAST_ARCHIVAL
FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC
LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY
WHERE SFC.SOURCE_FILE_TYPE = 'INPUT' WHERE SFC.SOURCE_FILE_TYPE = 'INPUT'
GROUP BY GROUP BY
SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY,
SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEEP_IN_TRASH SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEPT_IN_TRASH
ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID; ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID;
``` ```
@@ -1218,11 +1231,11 @@ SELECT
ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB, ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB,
MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH, MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH,
MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH, MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH,
SFC.IS_KEEP_IN_TRASH AS POLICY SFC.IS_KEPT_IN_TRASH AS POLICY
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEEP_IN_TRASH GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEPT_IN_TRASH
ORDER BY SIZE_GB DESC; ORDER BY SIZE_GB DESC;
``` ```
@@ -1241,7 +1254,7 @@ ORDER BY SIZE_GB DESC;
### v3.3.0 (2026-02-11) ### v3.3.0 (2026-02-11)
- **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA - **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA
- Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control - Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control
- Added `IS_KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter) - Added `IS_KEPT_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
- Added batch procedures with 3-level granularity (config/source/all): - Added batch procedures with 3-level granularity (config/source/all):
- ARCHIVE_ALL - Batch archival procedure - ARCHIVE_ALL - Batch archival procedure
- GATHER_TABLE_STAT_ALL - Batch statistics procedure - GATHER_TABLE_STAT_ALL - Batch statistics procedure
@@ -1392,7 +1405,7 @@ ORDER BY SIZE_GB DESC;
### TRASH Folder Retention Best Practices ### TRASH Folder Retention Best Practices
1. **Default Behavior (IS_KEEP_IN_TRASH = 'Y' - Recommended)**: 1. **Default Behavior (IS_KEPT_IN_TRASH = 'Y' - Recommended)**:
- Keeps CSV files in TRASH folder after archival - Keeps CSV files in TRASH folder after archival
- Provides safety net for rollback if archival issues occur - Provides safety net for rollback if archival issues occur
- Supports compliance and audit requirements - Supports compliance and audit requirements
@@ -1401,11 +1414,11 @@ ORDER BY SIZE_GB DESC;
- Configuration: - Configuration:
```sql ```sql
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'Y' SET IS_KEPT_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
``` ```
2. **TRASH Cleanup (IS_KEEP_IN_TRASH = 'N')**: 2. **TRASH Cleanup (IS_KEPT_IN_TRASH = 'N')**:
- Deletes CSV files from TRASH folder after successful archival - Deletes CSV files from TRASH folder after successful archival
- Reduces storage costs in DATA bucket - Reduces storage costs in DATA bucket
- Status: ARCHIVED_AND_PURGED - Status: ARCHIVED_AND_PURGED
@@ -1413,7 +1426,7 @@ ORDER BY SIZE_GB DESC;
- Configuration: - Configuration:
```sql ```sql
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_KEEP_IN_TRASH = 'N' SET IS_KEPT_IN_TRASH = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
``` ```
@@ -1423,7 +1436,7 @@ ORDER BY SIZE_GB DESC;
SELECT SELECT
SOURCE_FILE_NAME, SOURCE_FILE_NAME,
PROCESSING_STATUS, PROCESSING_STATUS,
ARCH_FILE_NAME, ARCH_PATH,
PARTITION_YEAR, PARTITION_YEAR,
PARTITION_MONTH PARTITION_MONTH
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
@@ -1446,3 +1459,4 @@ ORDER BY SIZE_GB DESC;
``` ```