admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FILE_ARCHIVER): Improve archival logic and error handling in FILE_ARCHIVER procedures

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-23 11:48:37 +01:00
parent 92feb95ae0
commit a35e28042b
1 changed files with 33 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb
View File

@@ -58,14 +58,15 @@ AS
BEGIN BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT count(*) , min(SOURCE_FILE_TYPE) -- LEFT JOIN ensures SOURCE_FILE_TYPE is retrieved from config even when no stats exist yet
SELECT count(s.A_SOURCE_FILE_CONFIG_KEY), min(c.SOURCE_FILE_TYPE)
INTO vCount, vSourceFileType INTO vCount, vSourceFileType
FROM CT_MRDS.A_TABLE_STAT s FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c LEFT JOIN CT_MRDS.A_TABLE_STAT s
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; WHERE c.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
IF vCount=0 and vSourceFileType='INPUT' THEN IF vCount = 0 AND vSourceFileType = 'INPUT' THEN
GATHER_TABLE_STAT(pSourceFileConfigKey); GATHER_TABLE_STAT(pSourceFileConfigKey);
END IF; END IF;
@@ -74,9 +75,13 @@ AS
INTO vTableStat INTO vTableStat
FROM CT_MRDS.A_TABLE_STAT FROM CT_MRDS.A_TABLE_STAT
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- EXCEPTION EXCEPTION
-- WHEN NO_DATA_FOUND THEN WHEN NO_DATA_FOUND THEN
-- CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'No statistics found in A_TABLE_STAT for config key ' || pSourceFileConfigKey
|| ' (SOURCE_FILE_TYPE=' || NVL(vSourceFileType, 'NULL') || '). Cannot proceed with archival.',
'ERROR', vParameters);
RAISE;
END; END;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
@@ -171,13 +176,14 @@ AS
; ;
-- Get all files that will be archived into "vfiles" collection ("regular data files") -- Get all files that will be archived into "vfiles" collection ("regular data files")
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location) -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
BEGIN BEGIN
execute immediate vQuery bulk collect into vfiles; execute immediate vQuery bulk collect into vfiles;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
IF SQLCODE = -29913 THEN IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Nothing to archive.', 'INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Nothing to archive.', 'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN; RETURN;
ELSE ELSE
@@ -185,6 +191,18 @@ AS
END IF; END IF;
END; END;
-- Check if any files match archival criteria
IF vfiles.COUNT = 0 THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'No files matching archival criteria found (strategy: ' || vSourceFileConfig.ARCHIVAL_STRATEGY
|| ', IS_WORKFLOW_SUCCESS_REQUIRED: ' || vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED || '). Nothing to archive.',
'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN;
END IF;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Files matching archival criteria: ' || vfiles.COUNT, 'INFO', vParameters);
-- Start EXPORT "regular data files" to parquet and DROP "csv" -- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
@@ -538,13 +556,14 @@ AS
on t.filename = r.object_name' on t.filename = r.object_name'
; ;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery);
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location) -- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
BEGIN BEGIN
execute immediate vQuery into vStats; execute immediate vQuery into vStats;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
IF SQLCODE = -29913 THEN IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Saving zero statistics.', 'INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Saving zero statistics.', 'INFO', vParameters);
vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey; vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey;
vStats.TABLE_NAME := vTableName; vStats.TABLE_NAME := vTableName;
vStats.FILE_COUNT := 0; vStats.FILE_COUNT := 0;