admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add diagnostic script for workflow key status in MARS-1409 post-hook

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-13 08:43:14 +01:00
parent 1385bfb9e7
commit 096994d514
3 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
MARS_Packages/REL02_POST/MARS-1409/diagnose_workflow_key_status.sql
View File

@@ -1,319 +0,0 @@
-- ============================================================================
-- MARS-1409 Diagnostic: Workflow key status after step 09
-- ============================================================================
-- Purpose: For each INPUT config with an ODS table, report:
-- [A] Files present in ODS bucket but NOT registered in A_SOURCE_FILE_RECEIVED
-- [B] Files registered in A_SOURCE_FILE_RECEIVED but NOT in ODS bucket
-- [C] Files present in both - with A_WORKFLOW_HISTORY_KEY populated
-- [D] Files present in both - A_WORKFLOW_HISTORY_KEY still NULL
--
-- Can be run at any time, read-only (no DML).
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
SET LINESIZE 200
PROMPT
PROMPT ============================================================================
PROMPT Diagnosing workflow key status (ODS bucket vs A_SOURCE_FILE_RECEIVED)
PROMPT ============================================================================
PROMPT
DECLARE
TYPE tStringList IS TABLE OF VARCHAR2(500);
vTableName VARCHAR2(200);
vTableExists NUMBER;
vBucketEmpty BOOLEAN;
vRefCursor SYS_REFCURSOR;
vFileName VARCHAR2(500);
-- Per-config counters
vOnlyInBucket NUMBER;
vOnlyInDB NUMBER;
vInBothWithKey NUMBER;
vInBothNoKey NUMBER;
-- Grand totals
vConfigsChecked NUMBER := 0;
vConfigsWithIssues NUMBER := 0;
vTotalOnlyInBucket NUMBER := 0;
vTotalOnlyInDB NUMBER := 0;
vTotalInBothWithKey NUMBER := 0;
vTotalInBothNoKey NUMBER := 0;
-- How many individual file names to print per category before summarising
cMaxPrint CONSTANT NUMBER := 1000;
vPrinted NUMBER;
BEGIN
FOR config_rec IN (
SELECT sfc.A_SOURCE_FILE_CONFIG_KEY,
sfc.A_SOURCE_KEY,
sfc.SOURCE_FILE_ID,
sfc.TABLE_ID
FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc
WHERE sfc.SOURCE_FILE_TYPE = 'INPUT'
AND sfc.TABLE_ID IS NOT NULL
ORDER BY sfc.A_SOURCE_KEY, sfc.SOURCE_FILE_ID, sfc.TABLE_ID
) LOOP
vTableName := 'ODS.' || config_rec.TABLE_ID || '_ODS';
SELECT COUNT(*) INTO vTableExists
FROM ALL_TABLES
WHERE OWNER = 'ODS'
AND TABLE_NAME = config_rec.TABLE_ID || '_ODS';
IF vTableExists = 0 THEN
CONTINUE;
END IF;
-- Check if the bucket location has any files at all
-- (empty bucket raises ORA-29913 instead of returning 0 rows)
vBucketEmpty := FALSE;
BEGIN
EXECUTE IMMEDIATE
'SELECT COUNT(*) FROM ' || vTableName || ' t WHERE ROWNUM = 1'
INTO vTableExists;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -29913 THEN
vBucketEmpty := TRUE;
ELSE
RAISE;
END IF;
END;
IF vBucketEmpty THEN
-- Bucket is empty: nothing in ODS, but registered records are all "not in bucket"
vOnlyInBucket := 0;
SELECT COUNT(*) INTO vOnlyInDB
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
AND sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
vInBothWithKey := 0;
vInBothNoKey := 0;
ELSE
-- ----------------------------------------------------------------
-- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
-- ----------------------------------------------------------------
EXECUTE IMMEDIATE
'SELECT COUNT(DISTINCT t.file$name) ' ||
'FROM ' || vTableName || ' t ' ||
'WHERE t.file$name IS NOT NULL ' ||
' AND NOT EXISTS ( ' ||
' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' ||
' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1)'
INTO vOnlyInBucket
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ----------------------------------------------------------------
-- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket
-- ----------------------------------------------------------------
EXECUTE IMMEDIATE
'SELECT COUNT(*) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND NOT EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vOnlyInDB
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ----------------------------------------------------------------
-- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL
-- ----------------------------------------------------------------
EXECUTE IMMEDIATE
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NOT NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vInBothWithKey
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
-- ----------------------------------------------------------------
-- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL
-- ----------------------------------------------------------------
EXECUTE IMMEDIATE
'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME)'
INTO vInBothNoKey
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
END IF; -- vBucketEmpty
-- Skip configs with nothing to report
IF vOnlyInBucket = 0 AND vOnlyInDB = 0 AND vInBothWithKey = 0 AND vInBothNoKey = 0 THEN
CONTINUE;
END IF;
vConfigsChecked := vConfigsChecked + 1;
DBMS_OUTPUT.PUT_LINE('Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY ||
'/' || config_rec.SOURCE_FILE_ID ||
'/' || config_rec.TABLE_ID || ')');
DBMS_OUTPUT.PUT_LINE(' [A] In bucket, not registered: ' || vOnlyInBucket);
DBMS_OUTPUT.PUT_LINE(' [B] Registered, not in bucket: ' || vOnlyInDB);
DBMS_OUTPUT.PUT_LINE(' [C] In both, A_WORKFLOW_HISTORY_KEY set: ' || vInBothWithKey);
DBMS_OUTPUT.PUT_LINE(' [D] In both, A_WORKFLOW_HISTORY_KEY NULL: ' || vInBothNoKey);
-- Print individual file names for categories with problems
IF vOnlyInBucket > 0 THEN
DBMS_OUTPUT.PUT_LINE(' [A] Files in bucket not registered:');
vPrinted := 0;
OPEN vRefCursor FOR
'SELECT DISTINCT t.file$name ' ||
'FROM ' || vTableName || ' t ' ||
'WHERE t.file$name IS NOT NULL ' ||
' AND NOT EXISTS ( ' ||
' SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
' WHERE sfr.SOURCE_FILE_NAME = t.file$name ' ||
' AND sfr.A_SOURCE_FILE_CONFIG_KEY = :1) ' ||
'ORDER BY t.file$name'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
LOOP
FETCH vRefCursor INTO vFileName;
EXIT WHEN vRefCursor%NOTFOUND;
vPrinted := vPrinted + 1;
IF vPrinted <= cMaxPrint THEN
DBMS_OUTPUT.PUT_LINE(' ' || vFileName);
ELSIF vPrinted = cMaxPrint + 1 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vOnlyInBucket - cMaxPrint) || ' more');
END IF;
END LOOP;
CLOSE vRefCursor;
END IF;
IF vOnlyInDB > 0 THEN
vConfigsWithIssues := vConfigsWithIssues + 1;
DBMS_OUTPUT.PUT_LINE(' [B] Registered files not found in bucket:');
vPrinted := 0;
OPEN vRefCursor FOR
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND NOT EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' ||
'ORDER BY sfr.SOURCE_FILE_NAME'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
LOOP
DECLARE
vStatus VARCHAR2(50);
vWfKey NUMBER;
BEGIN
FETCH vRefCursor INTO vFileName, vStatus, vWfKey;
EXIT WHEN vRefCursor%NOTFOUND;
vPrinted := vPrinted + 1;
IF vPrinted <= cMaxPrint THEN
DBMS_OUTPUT.PUT_LINE(' ' || vFileName ||
' status=' || vStatus ||
' wf_key=' || NVL(TO_CHAR(vWfKey), 'NULL'));
ELSIF vPrinted = cMaxPrint + 1 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vOnlyInDB - cMaxPrint) || ' more');
END IF;
END;
END LOOP;
CLOSE vRefCursor;
END IF;
IF vInBothNoKey > 0 THEN
vConfigsWithIssues := vConfigsWithIssues + 1;
DBMS_OUTPUT.PUT_LINE(' [D] Files in both but A_WORKFLOW_HISTORY_KEY still NULL:');
vPrinted := 0;
OPEN vRefCursor FOR
'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS ' ||
'FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME) ' ||
'ORDER BY sfr.SOURCE_FILE_NAME'
USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
LOOP
DECLARE
vStatus VARCHAR2(50);
BEGIN
FETCH vRefCursor INTO vFileName, vStatus;
EXIT WHEN vRefCursor%NOTFOUND;
vPrinted := vPrinted + 1;
IF vPrinted <= cMaxPrint THEN
DBMS_OUTPUT.PUT_LINE(' ' || vFileName || ' status=' || vStatus);
ELSIF vPrinted = cMaxPrint + 1 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vInBothNoKey - cMaxPrint) || ' more');
END IF;
END;
END LOOP;
CLOSE vRefCursor;
END IF;
DBMS_OUTPUT.PUT_LINE('');
-- Accumulate totals
vTotalOnlyInBucket := vTotalOnlyInBucket + vOnlyInBucket;
vTotalOnlyInDB := vTotalOnlyInDB + vOnlyInDB;
vTotalInBothWithKey := vTotalInBothWithKey + vInBothWithKey;
vTotalInBothNoKey := vTotalInBothNoKey + vInBothNoKey;
END LOOP;
DBMS_OUTPUT.PUT_LINE('============================================================================');
DBMS_OUTPUT.PUT_LINE('Grand Summary:');
DBMS_OUTPUT.PUT_LINE(' Configs with data checked: ' || vConfigsChecked);
DBMS_OUTPUT.PUT_LINE(' Configs with issues (B or D): ' || vConfigsWithIssues);
DBMS_OUTPUT.PUT_LINE(' [A] Files in bucket, not registered: ' || vTotalOnlyInBucket);
DBMS_OUTPUT.PUT_LINE(' [B] Registered, not in bucket: ' || vTotalOnlyInDB);
DBMS_OUTPUT.PUT_LINE(' [C] In both - A_WORKFLOW_HISTORY_KEY set: ' || vTotalInBothWithKey);
DBMS_OUTPUT.PUT_LINE(' [D] In both - A_WORKFLOW_HISTORY_KEY NULL: ' || vTotalInBothNoKey);
DBMS_OUTPUT.PUT_LINE('============================================================================');
IF vTotalOnlyInDB > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING [B]: ' || vTotalOnlyInDB || ' registered file(s) not found in ODS bucket.');
DBMS_OUTPUT.PUT_LINE(' These may have been moved to ARCHIVE or deleted from ODS.');
END IF;
IF vTotalInBothNoKey > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING [D]: ' || vTotalInBothNoKey || ' file(s) present in both but A_WORKFLOW_HISTORY_KEY is still NULL.');
DBMS_OUTPUT.PUT_LINE(' ODS table rows for these files may have A_WORKFLOW_HISTORY_KEY = NULL.');
DBMS_OUTPUT.PUT_LINE(' Re-run step 09 after the ODS rows are populated by the pipeline.');
END IF;
IF vConfigsWithIssues = 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('OK: No issues found. All registered files in ODS have A_WORKFLOW_HISTORY_KEY assigned.');
END IF;
EXCEPTION
WHEN OTHERS THEN
IF vRefCursor%ISOPEN THEN
CLOSE vRefCursor;
END IF;
DBMS_OUTPUT.PUT_LINE('ERROR: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Diagnosis complete.
PROMPT

10
MARS_Packages/REL02_POST/MARS-1409/verify_packages_version.sql
View File

@@ -30,8 +30,9 @@ SELECT CT_MRDS.ENV_MANAGER.GET_BUILD_INFO() AS BUILD_INFO FROM DUAL;
PROMPT
PROMPT Package Compilation Status:
SELECT object_name, object_type, status, last_ddl_time
FROM user_objects
WHERE object_name IN ('FILE_MANAGER', 'ENV_MANAGER', 'FILE_ARCHIVER')
FROM all_objects
WHERE owner = 'CT_MRDS'
AND object_name IN ('FILE_MANAGER', 'ENV_MANAGER', 'FILE_ARCHIVER')
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
ORDER BY object_name, object_type;
@@ -39,8 +40,9 @@ ORDER BY object_name, object_type;
PROMPT
PROMPT Compilation Errors (if any):
SELECT name, type, line, position, text
FROM user_errors
WHERE name IN ('FILE_MANAGER', 'ENV_MANAGER', 'FILE_ARCHIVER')
FROM all_errors
WHERE owner = 'CT_MRDS'
AND name IN ('FILE_MANAGER', 'ENV_MANAGER', 'FILE_ARCHIVER')
ORDER BY name, type, line, position;
PROMPT