Compare commits
7 Commits
811df6e8b1
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a35e28042b | ||
|
|
92feb95ae0 | ||
|
|
74b8857096 | ||
|
|
24997b1583 | ||
|
|
eb9b2bc38b | ||
|
|
2ea708a694 | ||
|
|
12c58f32a3 |
@@ -46,6 +46,16 @@ DECLARE
|
||||
cMaxPrint CONSTANT NUMBER := 1000;
|
||||
vPrinted NUMBER;
|
||||
|
||||
FUNCTION IS_EXTERNAL_TABLE_EMPTY_ERROR(
|
||||
pSqlCode NUMBER,
|
||||
pSqlErrm VARCHAR2
|
||||
) RETURN BOOLEAN
|
||||
IS
|
||||
BEGIN
|
||||
RETURN pSqlCode IN (-29913, -29400)
|
||||
OR INSTR(pSqlErrm, 'KUP-05002') > 0;
|
||||
END;
|
||||
|
||||
BEGIN
|
||||
|
||||
FOR config_rec IN (
|
||||
@@ -79,7 +89,7 @@ BEGIN
|
||||
INTO vTableExists;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -29913 THEN
|
||||
IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
|
||||
vBucketEmpty := TRUE;
|
||||
ELSE
|
||||
RAISE;
|
||||
@@ -97,63 +107,80 @@ BEGIN
|
||||
vInBothNoKey := 0;
|
||||
ELSE
|
||||
|
||||
-- ----------------------------------------------------------------
|
||||
-- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT t.file$name) ' ||
|
||||
'FROM ' || vTableName || ' t ' ||
|
||||
'WHERE t.file$name IS NOT NULL ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' ||
|
||||
' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1)'
|
||||
INTO vOnlyInBucket
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
BEGIN
|
||||
-- ----------------------------------------------------------------
|
||||
-- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT t.file$name) ' ||
|
||||
'FROM ' || vTableName || ' t ' ||
|
||||
'WHERE t.file$name IS NOT NULL ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' ||
|
||||
' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1)'
|
||||
INTO vOnlyInBucket
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
|
||||
-- ----------------------------------------------------------------
|
||||
-- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(*) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vOnlyInDB
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
-- ----------------------------------------------------------------
|
||||
-- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(*) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vOnlyInDB
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
|
||||
-- ----------------------------------------------------------------
|
||||
-- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NOT NULL ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vInBothWithKey
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
-- ----------------------------------------------------------------
|
||||
-- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NOT NULL ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vInBothWithKey
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
|
||||
-- ----------------------------------------------------------------
|
||||
-- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vInBothNoKey
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
-- ----------------------------------------------------------------
|
||||
-- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL
|
||||
-- ----------------------------------------------------------------
|
||||
EXECUTE IMMEDIATE
|
||||
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
|
||||
INTO vInBothNoKey
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
|
||||
vBucketEmpty := TRUE;
|
||||
vOnlyInBucket := 0;
|
||||
SELECT COUNT(*) INTO vOnlyInDB
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
|
||||
WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
|
||||
AND sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
|
||||
vInBothWithKey := 0;
|
||||
vInBothNoKey := 0;
|
||||
DBMS_OUTPUT.PUT_LINE(' NOTE: ODS bucket became empty/inaccessible during diagnostics for ' || vTableName || '. Falling back to DB-only counts for [B].');
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
|
||||
END IF; -- vBucketEmpty
|
||||
|
||||
@@ -204,16 +231,43 @@ BEGIN
|
||||
vConfigsWithIssues := vConfigsWithIssues + 1;
|
||||
DBMS_OUTPUT.PUT_LINE(' [B] Registered files not found in bucket:');
|
||||
vPrinted := 0;
|
||||
OPEN vRefCursor FOR
|
||||
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' ||
|
||||
'ORDER BY sfr.SOURCE_FILE_NAME'
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
BEGIN
|
||||
IF vBucketEmpty THEN
|
||||
OPEN vRefCursor FOR
|
||||
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
'ORDER BY sfr.SOURCE_FILE_NAME'
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
ELSE
|
||||
OPEN vRefCursor FOR
|
||||
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
' AND NOT EXISTS ( ' ||
|
||||
' SELECT 1 FROM ' || vTableName || ' t ' ||
|
||||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' ||
|
||||
'ORDER BY sfr.SOURCE_FILE_NAME'
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
|
||||
vBucketEmpty := TRUE;
|
||||
DBMS_OUTPUT.PUT_LINE(' NOTE: Skipping ODS anti-join details due to empty/inaccessible external table for ' || vTableName || '.');
|
||||
OPEN vRefCursor FOR
|
||||
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
|
||||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
|
||||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
|
||||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
|
||||
'ORDER BY sfr.SOURCE_FILE_NAME'
|
||||
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
LOOP
|
||||
DECLARE
|
||||
vStatus VARCHAR2(50);
|
||||
|
||||
@@ -53,6 +53,43 @@ END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT PREREQUISITE CHECK: Verifying MARS-1409 objects
|
||||
PROMPT ============================================================================
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
DECLARE
|
||||
vColCount NUMBER;
|
||||
vTableCount NUMBER;
|
||||
BEGIN
|
||||
SELECT COUNT(*)
|
||||
INTO vColCount
|
||||
FROM ALL_TAB_COLUMNS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND TABLE_NAME = 'A_SOURCE_FILE_RECEIVED'
|
||||
AND COLUMN_NAME = 'A_WORKFLOW_HISTORY_KEY';
|
||||
|
||||
IF vColCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(-20001,
|
||||
'Prerequisite failed: CT_MRDS.A_SOURCE_FILE_RECEIVED.A_WORKFLOW_HISTORY_KEY not found. Install MARS-1409 first (or do not run POSTHOOK after rollback).');
|
||||
END IF;
|
||||
|
||||
SELECT COUNT(*)
|
||||
INTO vTableCount
|
||||
FROM ALL_TABLES
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND TABLE_NAME = 'A_WORKFLOW_HISTORY';
|
||||
|
||||
IF vTableCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(-20002,
|
||||
'Prerequisite failed: CT_MRDS.A_WORKFLOW_HISTORY table not found.');
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('OK: Prerequisites satisfied (MARS-1409 schema changes detected).');
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT STEP 1: Backfill A_WORKFLOW_HISTORY_KEY for existing records
|
||||
|
||||
@@ -35,5 +35,21 @@ END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added to A_SOURCE_FILE_RECEIVED.
|
||||
PROMPT Renaming IS_KEEP_IN_TRASH to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG...
|
||||
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEEP_IN_TRASH TO IS_KEPT_IN_TRASH';
|
||||
DBMS_OUTPUT.PUT_LINE('Column IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG.');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -904 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEEP_IN_TRASH does not exist (already renamed or not present).');
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added and IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH.
|
||||
PROMPT
|
||||
|
||||
@@ -26,5 +26,21 @@ END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY column removed from A_SOURCE_FILE_RECEIVED.
|
||||
PROMPT Renaming IS_KEPT_IN_TRASH back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG...
|
||||
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEPT_IN_TRASH TO IS_KEEP_IN_TRASH';
|
||||
DBMS_OUTPUT.PUT_LINE('Column IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG.');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -904 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEPT_IN_TRASH does not exist (already renamed back or not present).');
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY removed and IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH.
|
||||
PROMPT
|
||||
|
||||
@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
MINIMUM_AGE_MONTHS NUMBER(3,0),
|
||||
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
|
||||
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
|
||||
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
|
||||
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
|
||||
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
|
||||
@@ -97,7 +97,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
|
||||
|
||||
@@ -58,14 +58,15 @@ AS
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
|
||||
SELECT count(*) , min(SOURCE_FILE_TYPE)
|
||||
-- LEFT JOIN ensures SOURCE_FILE_TYPE is retrieved from config even when no stats exist yet
|
||||
SELECT count(s.A_SOURCE_FILE_CONFIG_KEY), min(c.SOURCE_FILE_TYPE)
|
||||
INTO vCount, vSourceFileType
|
||||
FROM CT_MRDS.A_TABLE_STAT s
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
LEFT JOIN CT_MRDS.A_TABLE_STAT s
|
||||
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
WHERE c.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
IF vCount=0 and vSourceFileType='INPUT' THEN
|
||||
IF vCount = 0 AND vSourceFileType = 'INPUT' THEN
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey);
|
||||
END IF;
|
||||
|
||||
@@ -74,9 +75,13 @@ AS
|
||||
INTO vTableStat
|
||||
FROM CT_MRDS.A_TABLE_STAT
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
-- EXCEPTION
|
||||
-- WHEN NO_DATA_FOUND THEN
|
||||
--
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'No statistics found in A_TABLE_STAT for config key ' || pSourceFileConfigKey
|
||||
|| ' (SOURCE_FILE_TYPE=' || NVL(vSourceFileType, 'NULL') || '). Cannot proceed with archival.',
|
||||
'ERROR', vParameters);
|
||||
RAISE;
|
||||
END;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
|
||||
@@ -120,8 +125,8 @@ AS
|
||||
END IF;
|
||||
|
||||
-- Get TRASH policy from configuration
|
||||
vKeepInTrash := (vSourceFileConfig.IS_KEEP_IN_TRASH = 'Y');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEEP_IN_TRASH=' || vSourceFileConfig.IS_KEEP_IN_TRASH, 'INFO', vParameters);
|
||||
vKeepInTrash := (vSourceFileConfig.IS_KEPT_IN_TRASH = 'Y');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEPT_IN_TRASH=' || vSourceFileConfig.IS_KEPT_IN_TRASH, 'INFO', vParameters);
|
||||
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
@@ -171,13 +176,14 @@ AS
|
||||
;
|
||||
|
||||
-- Get all files that will be archived into "vfiles" collection ("regular data files")
|
||||
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
|
||||
-- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
|
||||
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
|
||||
BEGIN
|
||||
execute immediate vQuery bulk collect into vfiles;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -29913 THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Nothing to archive.', 'INFO', vParameters);
|
||||
IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Nothing to archive.', 'INFO', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN;
|
||||
ELSE
|
||||
@@ -185,6 +191,18 @@ AS
|
||||
END IF;
|
||||
END;
|
||||
|
||||
-- Check if any files match archival criteria
|
||||
IF vfiles.COUNT = 0 THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'No files matching archival criteria found (strategy: ' || vSourceFileConfig.ARCHIVAL_STRATEGY
|
||||
|| ', IS_WORKFLOW_SUCCESS_REQUIRED: ' || vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED || '). Nothing to archive.',
|
||||
'INFO', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Files matching archival criteria: ' || vfiles.COUNT, 'INFO', vParameters);
|
||||
|
||||
-- Start EXPORT "regular data files" to parquet and DROP "csv"
|
||||
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
|
||||
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
|
||||
@@ -309,10 +327,10 @@ AS
|
||||
AND r.source_file_name = f.filename
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
END LOOP;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEEP_IN_TRASH=N).','INFO', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEPT_IN_TRASH=N).','INFO', vParameters);
|
||||
ELSE
|
||||
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEPT_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO', vParameters);
|
||||
END IF;
|
||||
|
||||
--ROLLBACK PART
|
||||
@@ -538,13 +556,14 @@ AS
|
||||
on t.filename = r.object_name'
|
||||
;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery:', 'DEBUG', vQuery);
|
||||
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
|
||||
-- MARS-1468: Handle ORA-29913/ORA-12801 - no files in ODS bucket (empty external table location)
|
||||
-- ORA-29913 may come directly or wrapped in ORA-12801 (parallel query) with KUP-05002 root cause
|
||||
BEGIN
|
||||
execute immediate vQuery into vStats;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -29913 THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Saving zero statistics.', 'INFO', vParameters);
|
||||
IF SQLCODE IN (-29913, -12801) AND DBMS_UTILITY.FORMAT_ERROR_STACK LIKE '%KUP-05002%' THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (empty location, SQLCODE=' || SQLCODE || '). Saving zero statistics.', 'INFO', vParameters);
|
||||
vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey;
|
||||
vStats.TABLE_NAME := vTableName;
|
||||
vStats.FILE_COUNT := 0;
|
||||
@@ -1112,7 +1131,7 @@ AS
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
TABLE_ID,
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH,
|
||||
IS_KEPT_IN_TRASH,
|
||||
A_SOURCE_KEY
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
@@ -1137,7 +1156,7 @@ AS
|
||||
ELSE
|
||||
BEGIN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEEP_IN_TRASH=' || config_rec.IS_KEEP_IN_TRASH || ']',
|
||||
'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEPT_IN_TRASH=' || config_rec.IS_KEPT_IN_TRASH || ']',
|
||||
'INFO'
|
||||
);
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ AS
|
||||
'Y=standard DBT flow (WORKFLOW_SUCCESSFUL=Y required), N=bypass for manual/non-DBT sources. ' ||
|
||||
'Flag value stored in A_TABLE_STAT and A_TABLE_STAT_HIST for full audit of statistics basis.' || CHR(13)||CHR(10) ||
|
||||
'3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEPT_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
|
||||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
|
||||
@@ -55,7 +55,7 @@ AS
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
**/
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
@@ -66,7 +66,7 @@ AS
|
||||
* @desc Function wrapper for ARCHIVE_TABLE_DATA procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
* @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
@@ -141,7 +141,7 @@ AS
|
||||
* @name ARCHIVE_ALL
|
||||
* @desc Multi-level batch archival procedure with three granularity levels.
|
||||
* Only processes configurations where IS_ARCHIVE_ENABLED='Y'.
|
||||
* TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column.
|
||||
* TRASH policy for each table is controlled by individual IS_KEPT_IN_TRASH column.
|
||||
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
|
||||
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority)
|
||||
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)
|
||||
|
||||
@@ -1634,7 +1634,7 @@ AS
|
||||
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
|
||||
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
|
||||
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
|
||||
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
|
||||
,pIsKeptInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
|
||||
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
|
||||
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
|
||||
) IS
|
||||
@@ -1654,13 +1654,13 @@ AS
|
||||
,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY
|
||||
,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409
|
||||
,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828
|
||||
,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||'''' -- MARS-828
|
||||
,'pIsKeptInTrash => '''||nvl(to_char(pIsKeptInTrash), 'NULL')||'''' -- MARS-828
|
||||
,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828
|
||||
,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
|
||||
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths);
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
|
||||
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths);
|
||||
COMMIT;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
@@ -1770,7 +1770,7 @@ AS
|
||||
||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
|
||||
||cgBL||pLevel||'ENCODING = '||pSourceFileConfig.ENCODING
|
||||
||cgBL||pLevel||'IS_ARCHIVE_ENABLED = '||pSourceFileConfig.IS_ARCHIVE_ENABLED
|
||||
||cgBL||pLevel||'IS_KEEP_IN_TRASH = '||pSourceFileConfig.IS_KEEP_IN_TRASH
|
||||
||cgBL||pLevel||'IS_KEPT_IN_TRASH = '||pSourceFileConfig.IS_KEPT_IN_TRASH
|
||||
||cgBL||pLevel||'ARCHIVAL_STRATEGY = '||pSourceFileConfig.ARCHIVAL_STRATEGY
|
||||
||cgBL||pLevel||'MINIMUM_AGE_MONTHS = '||pSourceFileConfig.MINIMUM_AGE_MONTHS
|
||||
||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED
|
||||
|
||||
@@ -23,7 +23,7 @@ AS
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
|
||||
'3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
|
||||
'3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) ||
|
||||
'3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) ||
|
||||
'3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) ||
|
||||
@@ -446,13 +446,13 @@ AS
|
||||
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification.
|
||||
* MARS-1409: Added pIsWorkflowSuccessRequired parameter.
|
||||
* MARS-828: Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths.
|
||||
* MARS-828: Added pIsArchiveEnabled, pIsKeptInTrash, pArchivalStrategy, pMinimumAgeMonths.
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
|
||||
* If NULL, no CHARACTERSET clause is added to external table definitions
|
||||
* @param pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow)
|
||||
* 'N' = archive regardless of workflow status (bypass for manual/non-DBT sources)
|
||||
* @param pIsArchiveEnabled - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled
|
||||
* @param pIsKeepInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly
|
||||
* @param pIsKeptInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly
|
||||
* @param pArchivalStrategy - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL
|
||||
* @param pMinimumAgeMonths - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy)
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
|
||||
@@ -460,7 +460,7 @@ AS
|
||||
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
|
||||
* pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y',
|
||||
* pIsArchiveEnabled => 'Y', pIsKeepInTrash => 'N',
|
||||
* pIsArchiveEnabled => 'Y', pIsKeptInTrash => 'N',
|
||||
* pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3
|
||||
* );
|
||||
**/
|
||||
@@ -476,7 +476,7 @@ AS
|
||||
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
|
||||
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
|
||||
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
|
||||
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
|
||||
,pIsKeptInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
|
||||
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
|
||||
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
|
||||
);
|
||||
|
||||
@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
MINIMUM_AGE_MONTHS NUMBER(3,0),
|
||||
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
|
||||
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
|
||||
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
|
||||
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
|
||||
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
|
||||
@@ -107,7 +107,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
|
||||
|
||||
@@ -288,11 +288,11 @@ WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
ORDER BY SOURCE_FILE_ID, TABLE_ID;
|
||||
```
|
||||
|
||||
### IS_KEEP_IN_TRASH Column
|
||||
### IS_KEPT_IN_TRASH Column
|
||||
|
||||
Controls TRASH folder retention policy for archived files.
|
||||
|
||||
**Column**: `A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
|
||||
**Column**: `A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
|
||||
|
||||
**Values**:
|
||||
- `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED)
|
||||
@@ -312,7 +312,7 @@ Controls TRASH folder retention policy for archived files.
|
||||
```sql
|
||||
-- Production: Keep files in TRASH (recommended)
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'Y'
|
||||
SET IS_KEPT_IN_TRASH = 'Y'
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND SOURCE_FILE_ID = 'LM'
|
||||
AND TABLE_ID LIKE 'LM_%';
|
||||
@@ -320,14 +320,14 @@ COMMIT;
|
||||
|
||||
-- Test environment: Cleanup TRASH to save storage
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'N'
|
||||
SET IS_KEPT_IN_TRASH = 'N'
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND SOURCE_FILE_ID = 'TEST_SOURCE';
|
||||
COMMIT;
|
||||
|
||||
-- Bulk configuration by source
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'Y'
|
||||
SET IS_KEPT_IN_TRASH = 'Y'
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM');
|
||||
COMMIT;
|
||||
@@ -393,7 +393,7 @@ INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
|
||||
**Status Descriptions**:
|
||||
- **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket
|
||||
- **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket)
|
||||
- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEEP_IN_TRASH='N')
|
||||
- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEPT_IN_TRASH='N')
|
||||
|
||||
**Associated Columns Updated During Archival**:
|
||||
```sql
|
||||
@@ -435,9 +435,9 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
|
||||
2.1 TRASH Subfolder (DATA Bucket - File Retention)
|
||||
├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
|
||||
├─ Stores CSV files after archival to Parquet
|
||||
├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEEP_IN_TRASH config)
|
||||
├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEPT_IN_TRASH config)
|
||||
├─ Enables rollback if archival issues occur
|
||||
└─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEEP_IN_TRASH = 'N')
|
||||
└─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEPT_IN_TRASH = 'N')
|
||||
|
||||
3. ARCHIVE Bucket (Long-term Storage)
|
||||
├─ Historical data in Parquet format
|
||||
@@ -447,24 +447,35 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
|
||||
|
||||
**Key Procedures**:
|
||||
- `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause
|
||||
- TRASH folder retention controlled by `IS_KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG
|
||||
- TRASH folder retention controlled by `IS_KEPT_IN_TRASH` column in A_SOURCE_FILE_CONFIG
|
||||
- `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling
|
||||
- **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey`
|
||||
- **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey`
|
||||
- **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll`
|
||||
- **Error Handling**: Continues processing other tables on individual failures
|
||||
- **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations)
|
||||
- **Individual TRASH Policy**: Each table's `IS_KEEP_IN_TRASH` setting applied independently
|
||||
- **Individual TRASH Policy**: Each table's `IS_KEPT_IN_TRASH` setting applied independently
|
||||
- **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables
|
||||
- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy
|
||||
- `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic
|
||||
- `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll)` - Batch statistics with 3-level granularity
|
||||
- `RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey, pSourceKey, pRestoreAll)` - Restore archived files from TRASH
|
||||
- `PURGE_TRASH_FOLDER(pSourceFileConfigKey, pSourceKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity
|
||||
- `GATHER_TABLE_STAT(pSourceFileConfigKey)` - Calculates archival statistics using strategy logic
|
||||
- `GATHER_TABLE_STAT_ALL(pSourceFileConfigKey, pSourceKey, pGatherAll, pOnlyEnabled)` - Batch statistics with 3-level granularity
|
||||
- `pOnlyEnabled` (DEFAULT TRUE): When TRUE, only processes tables with `IS_ARCHIVE_ENABLED='Y'`
|
||||
- `RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey, pSourceFileConfigKey, pRestoreAll)` - Restore archived files from TRASH
|
||||
- `PURGE_TRASH_FOLDER(pSourceFileReceivedKey, pSourceFileConfigKey, pPurgeAll)` - Purge TRASH folder with 3-level granularity
|
||||
- `GET_VERSION` / `GET_BUILD_INFO` / `GET_VERSION_HISTORY` - Package version and metadata
|
||||
|
||||
**Function Wrappers (Python Integration)**:
|
||||
|
||||
All key procedures have `FN_*` function overloads returning `PLS_INTEGER` (SQLCODE: 0=success, error code on failure) for Python library integration:
|
||||
- `FN_ARCHIVE_TABLE_DATA`, `FN_GATHER_TABLE_STAT`, `FN_ARCHIVE_ALL`, `FN_GATHER_TABLE_STAT_ALL`
|
||||
- `RESTORE_FILE_FROM_TRASH` and `PURGE_TRASH_FOLDER` also have function overloads returning PLS_INTEGER
|
||||
|
||||
**Internal Functions** (not callable externally):
|
||||
- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy (private)
|
||||
- `GET_TABLE_STAT` - Retrieves or auto-generates table statistics (private)
|
||||
|
||||
**Archival Execution**:
|
||||
```sql
|
||||
-- Single table archival (TRASH retention controlled by IS_KEEP_IN_TRASH config)
|
||||
-- Single table archival (TRASH retention controlled by IS_KEPT_IN_TRASH config)
|
||||
BEGIN
|
||||
CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA(
|
||||
pSourceFileConfigKey => vSourceFileConfigKey
|
||||
@@ -500,7 +511,7 @@ END;
|
||||
- Data matching criteria moved from ODS to ARCHIVE bucket
|
||||
- CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/)
|
||||
- Parquet format with Hive-style partitioning applied to ARCHIVE bucket
|
||||
- TRASH retention controlled by IS_KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG
|
||||
- TRASH retention controlled by IS_KEPT_IN_TRASH column in A_SOURCE_FILE_CONFIG
|
||||
|
||||
### Automatic Rollback Mechanism
|
||||
|
||||
@@ -510,7 +521,7 @@ FILE_ARCHIVER implements **automatic rollback** to ensure data integrity if arch
|
||||
1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket
|
||||
2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED'
|
||||
3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket)
|
||||
4. **Optional Cleanup**: If IS_KEEP_IN_TRASH='N', files deleted from TRASH
|
||||
4. **Optional Cleanup**: If IS_KEPT_IN_TRASH='N', files deleted from TRASH
|
||||
|
||||
**Automatic Rollback Trigger**:
|
||||
If **any error occurs** during step 3 (Move to TRASH), the system:
|
||||
@@ -725,7 +736,7 @@ UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
|
||||
MINIMUM_AGE_MONTHS = 6,
|
||||
IS_ARCHIVE_ENABLED = 'Y', -- Enable archival
|
||||
IS_KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety
|
||||
IS_KEPT_IN_TRASH = 'Y' -- Keep files in TRASH for safety
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND SOURCE_FILE_ID = 'CSDB'
|
||||
AND TABLE_ID = 'CSDB_DEBT';
|
||||
@@ -739,7 +750,7 @@ COMMIT;
|
||||
|
||||
-- Configure TRASH cleanup for test environment
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival
|
||||
SET IS_KEPT_IN_TRASH = 'N' -- Delete files from TRASH after archival
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND SOURCE_FILE_ID = 'TEST_SOURCE';
|
||||
COMMIT;
|
||||
@@ -751,7 +762,7 @@ SELECT
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH
|
||||
IS_KEPT_IN_TRASH
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
ORDER BY SOURCE_FILE_ID, TABLE_ID;
|
||||
@@ -759,12 +770,12 @@ ORDER BY SOURCE_FILE_ID, TABLE_ID;
|
||||
-- Summary by archival status
|
||||
SELECT
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH,
|
||||
IS_KEPT_IN_TRASH,
|
||||
COUNT(*) AS TABLE_COUNT
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
GROUP BY IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH
|
||||
ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEEP_IN_TRASH DESC;
|
||||
GROUP BY IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH
|
||||
ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEPT_IN_TRASH DESC;
|
||||
```
|
||||
|
||||
## Release 01 Configuration
|
||||
@@ -948,7 +959,7 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
|
||||
|
||||
**Symptoms**: Files not deleted from TRASH after archival
|
||||
|
||||
**Cause**: Configuration has `IS_KEEP_IN_TRASH='Y'` (retain files in TRASH)
|
||||
**Cause**: Configuration has `IS_KEPT_IN_TRASH='Y'` (retain files in TRASH)
|
||||
|
||||
**Verification**:
|
||||
```sql
|
||||
@@ -956,8 +967,8 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
|
||||
SELECT
|
||||
SOURCE_FILE_ID,
|
||||
TABLE_ID,
|
||||
IS_KEEP_IN_TRASH,
|
||||
CASE IS_KEEP_IN_TRASH
|
||||
IS_KEPT_IN_TRASH,
|
||||
CASE IS_KEPT_IN_TRASH
|
||||
WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)'
|
||||
WHEN 'N' THEN 'Files DELETED immediately after archival'
|
||||
END AS TRASH_BEHAVIOR
|
||||
@@ -969,7 +980,7 @@ WHERE TABLE_ID = 'YOUR_TABLE';
|
||||
```sql
|
||||
-- Option A: Change configuration to auto-delete (permanent change)
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival
|
||||
SET IS_KEPT_IN_TRASH = 'N' -- Auto-delete from TRASH after archival
|
||||
WHERE TABLE_ID = 'YOUR_TABLE';
|
||||
COMMIT;
|
||||
|
||||
@@ -1072,13 +1083,15 @@ BEGIN
|
||||
WHERE TABLE_ID = 'YOUR_TABLE'
|
||||
AND ROWNUM = 1;
|
||||
|
||||
vWhereClause := CT_MRDS.FILE_ARCHIVER.GET_ARCHIVAL_WHERE_CLAUSE(vConfig);
|
||||
DBMS_OUTPUT.PUT_LINE('WHERE Clause: ' || vWhereClause);
|
||||
-- Note: GET_ARCHIVAL_WHERE_CLAUSE is a private function.
|
||||
-- To test WHERE clause logic, check A_PROCESS_LOG entries from ARCHIVE_TABLE_DATA
|
||||
-- which logs the generated WHERE clause at INFO level.
|
||||
DBMS_OUTPUT.PUT_LINE('Config: ' || vConfig.ARCHIVAL_STRATEGY || ', MIN_AGE=' || vConfig.MINIMUM_AGE_MONTHS);
|
||||
END;
|
||||
/
|
||||
```
|
||||
|
||||
#### Issue 3: Package Compilation Errors After Upgrade
|
||||
#### Issue 7: Package Compilation Errors After Upgrade
|
||||
|
||||
**Symptoms**: FILE_ARCHIVER package shows INVALID status
|
||||
|
||||
@@ -1132,7 +1145,7 @@ SELECT
|
||||
SFR.FILE_SIZE_BYTES,
|
||||
SFR.UPDATED_AT AS ARCHIVED_AT,
|
||||
TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH,
|
||||
SFC.IS_KEEP_IN_TRASH AS TRASH_POLICY
|
||||
SFC.IS_KEPT_IN_TRASH AS TRASH_POLICY
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
@@ -1148,17 +1161,17 @@ SELECT
|
||||
SFC.TABLE_ID,
|
||||
SFC.ARCHIVAL_STRATEGY,
|
||||
SFC.IS_ARCHIVE_ENABLED,
|
||||
SFC.IS_KEEP_IN_TRASH,
|
||||
SFC.IS_KEPT_IN_TRASH,
|
||||
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE,
|
||||
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH,
|
||||
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED,
|
||||
MAX(SFR.UPDATED_AT) FILTER (WHERE SFR.PROCESSING_STATUS LIKE 'ARCHIVED%') AS LAST_ARCHIVAL
|
||||
MAX(CASE WHEN SFR.PROCESSING_STATUS LIKE 'ARCHIVED%' THEN SFR.UPDATED_AT END) AS LAST_ARCHIVAL
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC
|
||||
LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE SFC.SOURCE_FILE_TYPE = 'INPUT'
|
||||
GROUP BY
|
||||
SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY,
|
||||
SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEEP_IN_TRASH
|
||||
SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEPT_IN_TRASH
|
||||
ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID;
|
||||
```
|
||||
|
||||
@@ -1218,11 +1231,11 @@ SELECT
|
||||
ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB,
|
||||
MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH,
|
||||
MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH,
|
||||
SFC.IS_KEEP_IN_TRASH AS POLICY
|
||||
SFC.IS_KEPT_IN_TRASH AS POLICY
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEEP_IN_TRASH
|
||||
GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEPT_IN_TRASH
|
||||
ORDER BY SIZE_GB DESC;
|
||||
```
|
||||
|
||||
@@ -1241,7 +1254,7 @@ ORDER BY SIZE_GB DESC;
|
||||
### v3.3.0 (2026-02-11)
|
||||
- **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA
|
||||
- Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control
|
||||
- Added `IS_KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
|
||||
- Added `IS_KEPT_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
|
||||
- Added batch procedures with 3-level granularity (config/source/all):
|
||||
- ARCHIVE_ALL - Batch archival procedure
|
||||
- GATHER_TABLE_STAT_ALL - Batch statistics procedure
|
||||
@@ -1392,7 +1405,7 @@ ORDER BY SIZE_GB DESC;
|
||||
|
||||
### TRASH Folder Retention Best Practices
|
||||
|
||||
1. **Default Behavior (IS_KEEP_IN_TRASH = 'Y' - Recommended)**:
|
||||
1. **Default Behavior (IS_KEPT_IN_TRASH = 'Y' - Recommended)**:
|
||||
- Keeps CSV files in TRASH folder after archival
|
||||
- Provides safety net for rollback if archival issues occur
|
||||
- Supports compliance and audit requirements
|
||||
@@ -1401,11 +1414,11 @@ ORDER BY SIZE_GB DESC;
|
||||
- Configuration:
|
||||
```sql
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'Y'
|
||||
SET IS_KEPT_IN_TRASH = 'Y'
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
|
||||
```
|
||||
|
||||
2. **TRASH Cleanup (IS_KEEP_IN_TRASH = 'N')**:
|
||||
2. **TRASH Cleanup (IS_KEPT_IN_TRASH = 'N')**:
|
||||
- Deletes CSV files from TRASH folder after successful archival
|
||||
- Reduces storage costs in DATA bucket
|
||||
- Status: ARCHIVED_AND_PURGED
|
||||
@@ -1413,7 +1426,7 @@ ORDER BY SIZE_GB DESC;
|
||||
- Configuration:
|
||||
```sql
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET IS_KEEP_IN_TRASH = 'N'
|
||||
SET IS_KEPT_IN_TRASH = 'N'
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
|
||||
```
|
||||
|
||||
@@ -1423,7 +1436,7 @@ ORDER BY SIZE_GB DESC;
|
||||
SELECT
|
||||
SOURCE_FILE_NAME,
|
||||
PROCESSING_STATUS,
|
||||
ARCH_FILE_NAME,
|
||||
ARCH_PATH,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
@@ -1446,3 +1459,4 @@ ORDER BY SIZE_GB DESC;
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user