feat: Implement MARS-1409 post-hook for backfilling A_WORKFLOW_HISTORY_KEY

- Added .gitignore to exclude temporary folders.
- Created SQL script to update existing A_WORKFLOW_HISTORY_KEY in A_SOURCE_FILE_RECEIVED.
- Implemented rollback script to clear backfilled A_WORKFLOW_HISTORY_KEY values.
- Added README.md for installation and usage instructions.
- Developed master installation and rollback scripts for MARS-1409 post-hook.
- Verified installation and rollback processes with detailed checks.
- Updated trigger logic to manage workflow history updates.
- Ensured proper version tracking and verification for related packages.
This commit is contained in:
Grzegorz Michalski
2026-03-13 08:30:32 +01:00
parent 7d2fb34ad9
commit 1385bfb9e7
15 changed files with 346 additions and 47 deletions

View File

@@ -1,12 +1,12 @@
-- ============================================================================
-- MARS-1409 Step 08: Install TRG_A_WORKFLOW_HISTORY trigger
-- MARS-1409 Step 08: Install A_WORKFLOW_HISTORY trigger
-- ============================================================================
-- Purpose: Update trigger to mark A_SOURCE_FILE_RECEIVED as INGESTED
-- when WORKFLOW_SUCCESSFUL is set to 'Y'
-- ============================================================================
PROMPT Installing TRG_A_WORKFLOW_HISTORY (new_version)...
@@new_version/TRG_A_WORKFLOW_HISTORY.sql
PROMPT Installing A_WORKFLOW_HISTORY (new_version)...
@@new_version/A_WORKFLOW_HISTORY.sql
PROMPT
DECLARE
@@ -15,14 +15,14 @@ BEGIN
SELECT status INTO v_status
FROM all_objects
WHERE owner = 'CT_MRDS'
AND object_name = 'TRG_A_WORKFLOW_HISTORY'
AND object_name = 'A_WORKFLOW_HISTORY'
AND object_type = 'TRIGGER';
DBMS_OUTPUT.PUT_LINE('TRG_A_WORKFLOW_HISTORY status: ' || v_status);
DBMS_OUTPUT.PUT_LINE('A_WORKFLOW_HISTORY status: ' || v_status);
IF v_status != 'VALID' THEN
RAISE_APPLICATION_ERROR(-20002, 'ERROR: TRG_A_WORKFLOW_HISTORY compiled with errors (status=' || v_status || '). Installation aborted.');
RAISE_APPLICATION_ERROR(-20002, 'ERROR: A_WORKFLOW_HISTORY compiled with errors (status=' || v_status || '). Installation aborted.');
END IF;
EXCEPTION
WHEN NO_DATA_FOUND THEN
RAISE_APPLICATION_ERROR(-20001, 'ERROR: TRG_A_WORKFLOW_HISTORY not found after installation');
RAISE_APPLICATION_ERROR(-20001, 'ERROR: A_WORKFLOW_HISTORY not found after installation');
END;
/

View File

@@ -1,177 +0,0 @@
-- ============================================================================
-- MARS-1409 Step 09: Update A_WORKFLOW_HISTORY_KEY for existing records
-- ============================================================================
-- Purpose: Populate A_WORKFLOW_HISTORY_KEY for existing A_SOURCE_FILE_RECEIVED records
-- by extracting values from corresponding ODS tables
-- Prerequisites:
-- - A_WORKFLOW_HISTORY_KEY column exists in A_SOURCE_FILE_RECEIVED
-- - ODS tables contain A_WORKFLOW_HISTORY_KEY and file$name columns
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
PROMPT
PROMPT Updating A_WORKFLOW_HISTORY_KEY for existing A_SOURCE_FILE_RECEIVED records...
DECLARE
vUpdatedTotal NUMBER := 0;
vUpdatedCurrent NUMBER := 0;
vFailedConfigs NUMBER := 0;
vTableNotFound NUMBER := 0;
vSkippedConfigs NUMBER := 0;
vTableName VARCHAR2(200);
vSQL VARCHAR2(32767);
vRecordsToUpdate NUMBER := 0;
vRemainingTargeted NUMBER := 0;
vTableExists NUMBER := 0;
BEGIN
-- Count total records to update
SELECT COUNT(*) INTO vRecordsToUpdate
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
AND PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
DBMS_OUTPUT.PUT_LINE('Found ' || vRecordsToUpdate || ' records with NULL A_WORKFLOW_HISTORY_KEY');
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
-- Process each INPUT configuration that has records to update
FOR config_rec IN (
SELECT
sfc.A_SOURCE_FILE_CONFIG_KEY,
sfc.A_SOURCE_KEY,
sfc.SOURCE_FILE_ID,
sfc.TABLE_ID,
sfc.TEMPLATE_TABLE_NAME,
(SELECT COUNT(*)
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = sfc.A_SOURCE_FILE_CONFIG_KEY
AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL
AND sfr.PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')
) AS NULL_COUNT
FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc
WHERE sfc.SOURCE_FILE_TYPE = 'INPUT'
AND sfc.TABLE_ID IS NOT NULL
ORDER BY sfc.A_SOURCE_KEY, sfc.SOURCE_FILE_ID, sfc.TABLE_ID
) LOOP
IF config_rec.NULL_COUNT = 0 THEN
vSkippedConfigs := vSkippedConfigs + 1;
DBMS_OUTPUT.PUT_LINE('SKIP: Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID ||
') - no records to update');
CONTINUE;
END IF;
BEGIN
-- Construct ODS table name from TABLE_ID (ODS tables have _ODS suffix)
vTableName := 'ODS.' || config_rec.TABLE_ID || '_ODS';
-- Check table existence before attempting dynamic SQL
SELECT COUNT(*) INTO vTableExists
FROM ALL_TABLES
WHERE OWNER = 'ODS'
AND TABLE_NAME = config_rec.TABLE_ID || '_ODS';
IF vTableExists = 0 THEN
vTableNotFound := vTableNotFound + 1;
DBMS_OUTPUT.PUT_LINE('SKIP: Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID ||
') - ODS table not found: ' || vTableName);
CONTINUE;
END IF;
DBMS_OUTPUT.PUT_LINE('Processing config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID || ')...');
-- Update using ODS table
vSQL :=
'UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'SET A_WORKFLOW_HISTORY_KEY = ( ' ||
' SELECT t.A_WORKFLOW_HISTORY_KEY ' ||
' FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME ' ||
' AND rownum=1 ' ||
') ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :config_key ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'', ''READY_FOR_INGESTION'', ''INGESTED'', ''ARCHIVED'', ''ARCHIVED_AND_TRASHED'', ''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME ' ||
' AND rownum=1 ' ||
' )';
EXECUTE IMMEDIATE vSQL USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
vUpdatedCurrent := SQL%ROWCOUNT;
vUpdatedTotal := vUpdatedTotal + vUpdatedCurrent;
IF vUpdatedCurrent > 0 THEN
DBMS_OUTPUT.PUT_LINE(' SUCCESS: Updated ' || vUpdatedCurrent || ' record(s)');
ELSE
DBMS_OUTPUT.PUT_LINE(' INFO: No matching records found in ODS table (files may not be ingested yet)');
END IF;
EXCEPTION
WHEN OTHERS THEN
vFailedConfigs := vFailedConfigs + 1;
DBMS_OUTPUT.PUT_LINE(' ERROR: Unexpected failure for config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (table: ' || vTableName || ')');
DBMS_OUTPUT.PUT_LINE(' Reason: ' || SQLERRM);
-- Continue processing other configurations despite this failure
END;
END LOOP;
COMMIT;
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
DBMS_OUTPUT.PUT_LINE('Update Summary:');
DBMS_OUTPUT.PUT_LINE(' Total records updated: ' || vUpdatedTotal);
DBMS_OUTPUT.PUT_LINE(' Configurations skipped (no NULL records): ' || vSkippedConfigs);
DBMS_OUTPUT.PUT_LINE(' Configurations skipped (ODS table not found): ' || vTableNotFound);
DBMS_OUTPUT.PUT_LINE(' Configurations failed (unexpected errors): ' || vFailedConfigs);
-- Check remaining NULL records - targeted statuses only
SELECT COUNT(*) INTO vRemainingTargeted
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
AND PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
-- Check all remaining NULL records (includes RECEIVED, VALIDATION_FAILED)
SELECT COUNT(*) INTO vRecordsToUpdate
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL;
DBMS_OUTPUT.PUT_LINE(' Remaining NULL records (targeted statuses): ' || vRemainingTargeted);
DBMS_OUTPUT.PUT_LINE(' Remaining NULL records (all statuses): ' || vRecordsToUpdate);
IF vRemainingTargeted > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('NOTE: Some records with targeted statuses still have NULL A_WORKFLOW_HISTORY_KEY.');
DBMS_OUTPUT.PUT_LINE(' This is expected for files not yet ingested into ODS tables');
DBMS_OUTPUT.PUT_LINE(' or ODS tables with a different structure.');
DBMS_OUTPUT.PUT_LINE(' These records will be populated when files are re-processed.');
END IF;
IF vFailedConfigs > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vFailedConfigs || ' configuration(s) failed with unexpected errors.');
DBMS_OUTPUT.PUT_LINE(' Review the ERROR lines above and investigate manually.');
END IF;
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
DBMS_OUTPUT.PUT_LINE('FATAL ERROR: ' || SQLERRM);
DBMS_OUTPUT.PUT_LINE('Transaction rolled back');
RAISE;
END;
/
PROMPT
PROMPT Existing workflow keys update completed!
PROMPT

View File

@@ -95,7 +95,7 @@ ORDER BY type, line, position;
-- Check trigger status
PROMPT
PROMPT 5B. Checking TRG_A_WORKFLOW_HISTORY trigger...
PROMPT 5B. Checking A_WORKFLOW_HISTORY trigger...
SELECT
trigger_name,
trigger_type,
@@ -103,7 +103,7 @@ SELECT
status
FROM all_triggers
WHERE owner = 'CT_MRDS'
AND trigger_name = 'TRG_A_WORKFLOW_HISTORY';
AND trigger_name = 'A_WORKFLOW_HISTORY';
-- Verify package versions
PROMPT

View File

@@ -34,19 +34,19 @@ END;
-- Check trigger was restored
PROMPT
PROMPT 1B. Checking TRG_A_WORKFLOW_HISTORY trigger status...
PROMPT 1B. Checking A_WORKFLOW_HISTORY trigger status...
DECLARE
v_status VARCHAR2(20);
BEGIN
SELECT status INTO v_status
FROM all_objects
WHERE owner = 'CT_MRDS'
AND object_name = 'TRG_A_WORKFLOW_HISTORY'
AND object_name = 'A_WORKFLOW_HISTORY'
AND object_type = 'TRIGGER';
DBMS_OUTPUT.PUT_LINE('TRG_A_WORKFLOW_HISTORY status: ' || v_status);
DBMS_OUTPUT.PUT_LINE('A_WORKFLOW_HISTORY status: ' || v_status);
EXCEPTION
WHEN NO_DATA_FOUND THEN
DBMS_OUTPUT.PUT_LINE('WARNING: TRG_A_WORKFLOW_HISTORY not found');
DBMS_OUTPUT.PUT_LINE('WARNING: A_WORKFLOW_HISTORY not found');
END;
/

View File

@@ -1,12 +1,12 @@
-- ============================================================================
-- MARS-1409 Rollback 93C: Restore TRG_A_WORKFLOW_HISTORY trigger
-- MARS-1409 Rollback 93C: Restore A_WORKFLOW_HISTORY trigger
-- ============================================================================
-- Purpose: Restore trigger to pre-MARS-1409 state
-- Removes INGESTED status update logic from A_SOURCE_FILE_RECEIVED
-- ============================================================================
PROMPT Restoring TRG_A_WORKFLOW_HISTORY (rollback_version)...
@@rollback_version/TRG_A_WORKFLOW_HISTORY.sql
PROMPT Restoring trigger A_WORKFLOW_HISTORY (rollback_version)...
@@rollback_version/A_WORKFLOW_HISTORY.sql
PROMPT
DECLARE

View File

@@ -33,7 +33,7 @@ PROMPT =========================================================================
PROMPT Package: CT_MRDS.FILE_MANAGER v3.X.X
PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED
PROMPT Purpose: Direct tracking of workflow history keys in file registration
PROMPT Steps: 11 (DDL, ENV_MANAGER Update, FILE_MANAGER Update, FILE_ARCHIVER Update, Trigger Update, Existing Records Backfill, Verification, Tracking)
PROMPT Steps: 11 (DDL, ENV_MANAGER Update, FILE_MANAGER Update, FILE_ARCHIVER Update, Trigger Update, Verification, Tracking, Version Verification)
PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
PROMPT ============================================================================
@@ -92,28 +92,28 @@ PROMPT =========================================================================
PROMPT
PROMPT ============================================================================
PROMPT STEP 8: Update TRG_A_WORKFLOW_HISTORY trigger
PROMPT STEP 8: Update A_WORKFLOW_HISTORY trigger
PROMPT ============================================================================
@@08_MARS_1409_install_CT_MRDS_TRG_A_WORKFLOW_HISTORY.sql
@@08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 9: Update A_WORKFLOW_HISTORY_KEY for existing records
PROMPT STEP 9: Verify installation
PROMPT ============================================================================
@@09_MARS_1409_update_existing_workflow_keys.sql
@@09_MARS_1409_verify_installation.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 10: Verify installation
PROMPT ============================================================================
@@10_MARS_1409_verify_installation.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 11: Track package versions
PROMPT STEP 10: Track package versions
PROMPT ============================================================================
@@track_package_versions.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 11: Verify package versions
PROMPT ============================================================================
@@verify_packages_version.sql
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409 Installation Complete

View File

@@ -3,16 +3,10 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON ct_ods.a_load_history TO ct_mrds;
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- ============================================================================
-- TRG_A_WORKFLOW_HISTORY Trigger Definition
-- A_WORKFLOW_HISTORY Trigger Definition
-- ============================================================================
-- Drop old trigger (different name) to ensure clean rename; ignore error if not exists
WHENEVER SQLERROR CONTINUE
drop TRIGGER ct_mrds.a_workflow_history;
WHENEVER SQLERROR EXIT SQL.SQLCODE
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."TRG_A_WORKFLOW_HISTORY"
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."A_WORKFLOW_HISTORY"
AFTER INSERT OR UPDATE OF workflow_successful ON ct_mrds.a_workflow_history
REFERENCING NEW AS new OLD AS old
FOR EACH ROW

View File

@@ -0,0 +1,44 @@
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."A_WORKFLOW_HISTORY"
AFTER INSERT OR UPDATE OF workflow_successful ON ct_mrds.a_workflow_history
REFERENCING NEW AS new OLD AS old
FOR EACH ROW
DECLARE
v_workflow_name VARCHAR2(128);
v_wla_id NUMBER;
BEGIN
IF :new.service_name = 'ODS' AND :new.workflow_name IN (
'w_ODS_LM_STANDING_FACILITIES', 'w_ODS_CSDB_DEBT', 'w_ODS_CSDB_DEBT_DAILY', 'w_ODS_CSDB_RATINGS_FULL',
'w_ODS_TMS_LIMIT_ACCESS', 'w_ODS_TMS_PORTFOLIO_ACCESS', 'w_ODS_TMS_PORTFOLIO_TREE',
'w_ODS_TMS_COLLATERAL_INVENTORY', 'w_ODS_TOP_FULLBIDARRAY_COMPILED', 'w_ODS_TOP_ANNOUNCEMENT',
'w_ODS_TOP_ALLOTMENT_MODIFICATIONS', 'w_ODS_TOP_ALLOTMENT', 'w_ODS_CEPH_PRICING', 'w_ODS_C2D_MPEC'
) THEN
IF :new.workflow_successful = 'Y' AND :new.workflow_successful <> NVL(:old.workflow_successful, 'N') THEN
CASE
WHEN :new.workflow_name = 'w_ODS_LM_STANDING_FACILITIES' THEN v_workflow_name := 'w_ODS_LM_STANDING_FACILITY';
WHEN :new.workflow_name = 'w_ODS_TMS_LIMIT_ACCESS' THEN v_workflow_name := 'w_ODS_TMS_RAR_LIMITACCESS';
WHEN :new.workflow_name = 'w_ODS_TMS_PORTFOLIO_ACCESS' THEN v_workflow_name := 'w_ODS_TMS_RAR_PORTFOLIOACCESS';
WHEN :new.workflow_name = 'w_ODS_TMS_PORTFOLIO_TREE' THEN v_workflow_name := 'w_ODS_TMS_RAR_PORTFOLIOTREE';
WHEN :new.workflow_name = 'w_ODS_TMS_COLLATERAL_INVENTORY' THEN v_workflow_name := 'w_ODS_TMS_RAR_RARCOLLATERALINVENTORY';
WHEN :new.workflow_name = 'w_ODS_TOP_FULLBIDARRAY_COMPILED' THEN v_workflow_name := 'w_ODS_TOP_FULLBIDARRAY_COMPILED';
WHEN :new.workflow_name = 'w_ODS_TOP_ANNOUNCEMENT' THEN v_workflow_name := 'w_ODS_TOP_ANNOUNCEMENT';
WHEN :new.workflow_name = 'w_ODS_TOP_ALLOTMENT_MODIFICATIONS' THEN v_workflow_name := 'w_ODS_TOP_ALLOTMENT_MODIFICATIONS';
WHEN :new.workflow_name = 'w_ODS_TOP_ALLOTMENT' THEN v_workflow_name := 'w_ODS_TOP_ALLOTMENT';
WHEN :new.workflow_name = 'w_ODS_CEPH_PRICING' THEN v_workflow_name := 'w_ODS_CEPH_PRICING';
WHEN :new.workflow_name = 'w_ODS_C2D_MPEC' THEN v_workflow_name := 'w_ODS_C2D_MPEC';
ELSE
v_workflow_name := :new.workflow_name;
END CASE;
BEGIN
v_wla_id := TO_NUMBER(:new.orchestration_run_id);
EXCEPTION WHEN OTHERS THEN NULL;
END;
INSERT INTO ct_ods.a_load_history (
a_etl_load_set_key, workflow_name, infa_run_id, load_start, load_end, exdi_appl_req_id, exdi_correlation_id, load_successful, wla_run_id, dq_flag
) VALUES (
:new.a_workflow_history_key, v_workflow_name, NULL, :new.workflow_start, :new.workflow_end, NULL, NULL, :new.workflow_successful, v_wla_id, 'F'
);
END IF;
END IF;
END
;
/

View File

@@ -85,9 +85,9 @@ PROMPT =========================================================================
PROMPT
PROMPT ============================================================================
PROMPT STEP 7: Restore TRG_A_WORKFLOW_HISTORY trigger (previous version)
PROMPT STEP 7: Restore A_WORKFLOW_HISTORY trigger (previous version)
PROMPT ============================================================================
@@97_MARS_1409_rollback_CT_MRDS_TRG_A_WORKFLOW_HISTORY.sql
@@97_MARS_1409_rollback_CT_MRDS_A_WORKFLOW_HISTORY.sql
PROMPT
PROMPT ============================================================================
@@ -101,6 +101,12 @@ PROMPT STEP 9: Verify rollback
PROMPT ============================================================================
@@90_MARS_1409_verify_rollback.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 10: Verify package versions
PROMPT ============================================================================
@@verify_packages_version.sql
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409 Rollback Complete

View File

@@ -2,11 +2,6 @@ WHENEVER SQLERROR CONTINUE
GRANT SELECT, INSERT, UPDATE, DELETE ON ct_ods.a_load_history TO ct_mrds;
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- Drop new trigger (MARS-1409 name) before restoring old trigger name
WHENEVER SQLERROR CONTINUE
drop TRIGGER ct_mrds.TRG_A_WORKFLOW_HISTORY;
WHENEVER SQLERROR EXIT SQL.SQLCODE
create or replace TRIGGER ct_mrds.a_workflow_history
AFTER INSERT OR UPDATE OF workflow_successful ON ct_mrds.a_workflow_history
REFERENCING NEW AS new OLD AS old