feat: Implement MARS-1409 post-hook for backfilling A_WORKFLOW_HISTORY_KEY

- Added .gitignore to exclude temporary folders.
- Created SQL script to update existing A_WORKFLOW_HISTORY_KEY in A_SOURCE_FILE_RECEIVED.
- Implemented rollback script to clear backfilled A_WORKFLOW_HISTORY_KEY values.
- Added README.md for installation and usage instructions.
- Developed master installation and rollback scripts for MARS-1409 post-hook.
- Verified installation and rollback processes with detailed checks.
- Updated trigger logic to manage workflow history updates.
- Ensured proper version tracking and verification for related packages.
This commit is contained in:
Grzegorz Michalski
2026-03-13 08:30:32 +01:00
parent 7d2fb34ad9
commit 1385bfb9e7
15 changed files with 346 additions and 47 deletions

View File

@@ -0,0 +1,5 @@
# Exclude temporary folders from version control
confluence/
log/
test/
mock_data/

View File

@@ -0,0 +1,179 @@
-- ============================================================================
-- MARS-1409-POSTHOOK Step 01: Update A_WORKFLOW_HISTORY_KEY for existing records
-- ============================================================================
-- Purpose: Populate A_WORKFLOW_HISTORY_KEY for existing A_SOURCE_FILE_RECEIVED records
-- by extracting values from corresponding ODS tables
-- Author: Grzegorz Michalski
-- Date: 2026-03-13
-- Prerequisites:
-- - MARS-1409 installed (A_WORKFLOW_HISTORY_KEY column exists in A_SOURCE_FILE_RECEIVED)
-- - ODS tables contain A_WORKFLOW_HISTORY_KEY and file$name columns
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
PROMPT
PROMPT Updating A_WORKFLOW_HISTORY_KEY for existing A_SOURCE_FILE_RECEIVED records...
DECLARE
vUpdatedTotal NUMBER := 0;
vUpdatedCurrent NUMBER := 0;
vFailedConfigs NUMBER := 0;
vTableNotFound NUMBER := 0;
vSkippedConfigs NUMBER := 0;
vTableName VARCHAR2(200);
vSQL VARCHAR2(32767);
vRecordsToUpdate NUMBER := 0;
vRemainingTargeted NUMBER := 0;
vTableExists NUMBER := 0;
BEGIN
-- Count total records to update
SELECT COUNT(*) INTO vRecordsToUpdate
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
AND PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
DBMS_OUTPUT.PUT_LINE('Found ' || vRecordsToUpdate || ' records with NULL A_WORKFLOW_HISTORY_KEY');
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
-- Process each INPUT configuration that has records to update
FOR config_rec IN (
SELECT
sfc.A_SOURCE_FILE_CONFIG_KEY,
sfc.A_SOURCE_KEY,
sfc.SOURCE_FILE_ID,
sfc.TABLE_ID,
sfc.TEMPLATE_TABLE_NAME,
(SELECT COUNT(*)
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = sfc.A_SOURCE_FILE_CONFIG_KEY
AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL
AND sfr.PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')
) AS NULL_COUNT
FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc
WHERE sfc.SOURCE_FILE_TYPE = 'INPUT'
AND sfc.TABLE_ID IS NOT NULL
ORDER BY sfc.A_SOURCE_KEY, sfc.SOURCE_FILE_ID, sfc.TABLE_ID
) LOOP
IF config_rec.NULL_COUNT = 0 THEN
vSkippedConfigs := vSkippedConfigs + 1;
DBMS_OUTPUT.PUT_LINE('SKIP: Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID ||
') - no records to update');
CONTINUE;
END IF;
BEGIN
-- Construct ODS table name from TABLE_ID (ODS tables have _ODS suffix)
vTableName := 'ODS.' || config_rec.TABLE_ID || '_ODS';
-- Check table existence before attempting dynamic SQL
SELECT COUNT(*) INTO vTableExists
FROM ALL_TABLES
WHERE OWNER = 'ODS'
AND TABLE_NAME = config_rec.TABLE_ID || '_ODS';
IF vTableExists = 0 THEN
vTableNotFound := vTableNotFound + 1;
DBMS_OUTPUT.PUT_LINE('SKIP: Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID ||
') - ODS table not found: ' || vTableName);
CONTINUE;
END IF;
DBMS_OUTPUT.PUT_LINE('Processing config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (' || config_rec.A_SOURCE_KEY || '/' || config_rec.SOURCE_FILE_ID || '/' || config_rec.TABLE_ID || ')...');
-- Update using ODS table
vSQL :=
'UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
'SET A_WORKFLOW_HISTORY_KEY = ( ' ||
' SELECT t.A_WORKFLOW_HISTORY_KEY ' ||
' FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME ' ||
' AND rownum=1 ' ||
') ' ||
'WHERE sfr.A_SOURCE_FILE_CONFIG_KEY = :config_key ' ||
' AND sfr.A_WORKFLOW_HISTORY_KEY IS NULL ' ||
' AND sfr.PROCESSING_STATUS IN (''VALIDATED'', ''READY_FOR_INGESTION'', ''INGESTED'', ''ARCHIVED'', ''ARCHIVED_AND_TRASHED'', ''ARCHIVED_AND_PURGED'') ' ||
' AND EXISTS ( ' ||
' SELECT 1 FROM ' || vTableName || ' t ' ||
' WHERE t.file$name = sfr.SOURCE_FILE_NAME ' ||
' AND rownum=1 ' ||
' )';
EXECUTE IMMEDIATE vSQL USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
vUpdatedCurrent := SQL%ROWCOUNT;
vUpdatedTotal := vUpdatedTotal + vUpdatedCurrent;
IF vUpdatedCurrent > 0 THEN
DBMS_OUTPUT.PUT_LINE(' SUCCESS: Updated ' || vUpdatedCurrent || ' record(s)');
ELSE
DBMS_OUTPUT.PUT_LINE(' INFO: No matching records found in ODS table (files may not be ingested yet)');
END IF;
EXCEPTION
WHEN OTHERS THEN
vFailedConfigs := vFailedConfigs + 1;
DBMS_OUTPUT.PUT_LINE(' ERROR: Unexpected failure for config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
' (table: ' || vTableName || ')');
DBMS_OUTPUT.PUT_LINE(' Reason: ' || SQLERRM);
-- Continue processing other configurations despite this failure
END;
END LOOP;
COMMIT;
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
DBMS_OUTPUT.PUT_LINE('Update Summary:');
DBMS_OUTPUT.PUT_LINE(' Total records updated: ' || vUpdatedTotal);
DBMS_OUTPUT.PUT_LINE(' Configurations skipped (no NULL records): ' || vSkippedConfigs);
DBMS_OUTPUT.PUT_LINE(' Configurations skipped (ODS table not found): ' || vTableNotFound);
DBMS_OUTPUT.PUT_LINE(' Configurations failed (unexpected errors): ' || vFailedConfigs);
-- Check remaining NULL records - targeted statuses only
SELECT COUNT(*) INTO vRemainingTargeted
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
AND PROCESSING_STATUS IN ('VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
-- Check all remaining NULL records (includes RECEIVED, VALIDATION_FAILED)
SELECT COUNT(*) INTO vRecordsToUpdate
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_WORKFLOW_HISTORY_KEY IS NULL;
DBMS_OUTPUT.PUT_LINE(' Remaining NULL records (targeted statuses): ' || vRemainingTargeted);
DBMS_OUTPUT.PUT_LINE(' Remaining NULL records (all statuses): ' || vRecordsToUpdate);
IF vRemainingTargeted > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('NOTE: Some records with targeted statuses still have NULL A_WORKFLOW_HISTORY_KEY.');
DBMS_OUTPUT.PUT_LINE(' This is expected for files not yet ingested into ODS tables');
DBMS_OUTPUT.PUT_LINE(' or ODS tables with a different structure.');
DBMS_OUTPUT.PUT_LINE(' These records will be populated when files are re-processed.');
END IF;
IF vFailedConfigs > 0 THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('NOTE: ' || vFailedConfigs || ' configuration(s) failed with unexpected errors.');
DBMS_OUTPUT.PUT_LINE(' Review the ERROR lines above and investigate manually.');
END IF;
DBMS_OUTPUT.PUT_LINE('----------------------------------------');
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
DBMS_OUTPUT.PUT_LINE('FATAL ERROR: ' || SQLERRM);
DBMS_OUTPUT.PUT_LINE('Transaction rolled back');
RAISE;
END;
/
PROMPT
PROMPT Existing workflow keys update completed!
PROMPT

View File

@@ -0,0 +1,43 @@
-- ============================================================================
-- MARS-1409-POSTHOOK Rollback Step 91: Clear backfilled A_WORKFLOW_HISTORY_KEY values
-- ============================================================================
-- Purpose: Reset A_WORKFLOW_HISTORY_KEY to NULL for all records in
-- A_SOURCE_FILE_RECEIVED. Reverts the backfill performed by
-- 01_MARS_1409_POSTHOOK_update_existing_workflow_keys.sql.
-- Author: Grzegorz Michalski
-- Date: 2026-03-13
-- Note: Records populated by the new trigger (after MARS-1409 install) will also
-- be cleared. The trigger will repopulate them on next file processing.
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
PROMPT
PROMPT Clearing backfilled A_WORKFLOW_HISTORY_KEY values...
DECLARE
vCleared NUMBER := 0;
BEGIN
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET A_WORKFLOW_HISTORY_KEY = NULL
WHERE A_WORKFLOW_HISTORY_KEY IS NOT NULL;
vCleared := SQL%ROWCOUNT;
COMMIT;
DBMS_OUTPUT.PUT_LINE('Cleared A_WORKFLOW_HISTORY_KEY for ' || vCleared || ' record(s)');
DBMS_OUTPUT.PUT_LINE('Rollback of backfill completed successfully');
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
DBMS_OUTPUT.PUT_LINE('FATAL ERROR: ' || SQLERRM);
DBMS_OUTPUT.PUT_LINE('Transaction rolled back');
RAISE;
END;
/
PROMPT
PROMPT Workflow keys rollback completed!
PROMPT

View File

@@ -0,0 +1,60 @@
# MARS-1409-POSTHOOK: Backfill A_WORKFLOW_HISTORY_KEY for existing records
## Overview
Post-hook for MARS-1409. Backfills `A_WORKFLOW_HISTORY_KEY` in
`CT_MRDS.A_SOURCE_FILE_RECEIVED` for historical records that existed before
MARS-1409 was installed.
Matches records by `SOURCE_FILE_NAME` against `file$name` in the corresponding
ODS table (`ODS.<TABLE_ID>_ODS`) for each `INPUT` source configuration.
## Contents
| File | Description |
|------|-------------|
| `install_mars1409_posthook.sql` | Master installation script (SPOOL, ACCEPT, quit) |
| `rollback_mars1409_posthook.sql` | Master rollback script (SPOOL, ACCEPT, quit) |
| `01_MARS_1409_POSTHOOK_update_existing_workflow_keys.sql` | Backfill UPDATE script |
| `91_MARS_1409_POSTHOOK_rollback_workflow_keys.sql` | Clear backfilled values |
| `track_package_versions.sql` | Universal version tracking (no packages changed) |
| `verify_packages_version.sql` | Universal package verification |
| `README.md` | This file |
## Prerequisites
- MARS-1409 installed (`A_WORKFLOW_HISTORY_KEY` column must exist in `CT_MRDS.A_SOURCE_FILE_RECEIVED`)
- ODS tables populated with ingested data
- ADMIN user with access to CT_MRDS and ODS schemas
## Installation
```powershell
# Execute as ADMIN user
Get-Content "MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/install_mars1409_posthook.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high"
```
Log file created automatically: `log/INSTALL_MARS_1409_POSTHOOK_<PDB>_<timestamp>.log`
## What it does
- Iterates all `INPUT` source configurations from `CT_MRDS.A_SOURCE_FILE_CONFIG`
- For each config, joins `A_SOURCE_FILE_RECEIVED` with `ODS.<TABLE_ID>_ODS` on `SOURCE_FILE_NAME = file$name`
- Updates `A_WORKFLOW_HISTORY_KEY` for records with statuses:
`VALIDATED`, `READY_FOR_INGESTION`, `INGESTED`, `ARCHIVED`, `ARCHIVED_AND_TRASHED`, `ARCHIVED_AND_PURGED`
- Skips configs with no NULL records or missing ODS tables
- Prints summary with counts per config and overall totals
## Rollback
```powershell
# Execute as ADMIN user
Get-Content "MARS_Packages/REL02_POST/MARS-1409-POSTHOOK/rollback_mars1409_posthook.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high"
```
Rollback clears all non-NULL `A_WORKFLOW_HISTORY_KEY` values from `A_SOURCE_FILE_RECEIVED`.
The trigger installed by MARS-1409 will repopulate new records automatically.
## Related
- MARS-1409: Add A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED (main package)

View File

@@ -0,0 +1,74 @@
-- ============================================================================
-- MARS-1409-POSTHOOK Master Installation Script
-- ============================================================================
-- Purpose: Post-hook for MARS-1409 - Backfill A_WORKFLOW_HISTORY_KEY for
-- existing A_SOURCE_FILE_RECEIVED records by joining with ODS tables.
-- Author: Grzegorz Michalski
-- Date: 2026-03-13
-- Prerequisites: MARS-1409 must be installed first (column must exist)
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
SET VERIFY OFF
SET FEEDBACK ON
SET ECHO OFF
-- Create log directory if it doesn't exist
host mkdir log 2>nul
-- Generate dynamic SPOOL filename with timestamp
var filename VARCHAR2(100)
BEGIN
:filename := 'log/INSTALL_MARS_1409_POSTHOOK_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409-POSTHOOK Installation Starting
PROMPT ============================================================================
PROMPT Purpose: Backfill A_WORKFLOW_HISTORY_KEY for historical records
PROMPT in A_SOURCE_FILE_RECEIVED using matching ODS tables.
PROMPT
PROMPT This script will:
PROMPT - Update A_WORKFLOW_HISTORY_KEY for records with targeted PROCESSING_STATUS
PROMPT - Match records by SOURCE_FILE_NAME against file$name in ODS tables
PROMPT - Skip configs with no NULL records or missing ODS tables
PROMPT
PROMPT Prerequisite: MARS-1409 installed (A_WORKFLOW_HISTORY_KEY column exists)
PROMPT Expected Duration: 1-5 minutes (depends on data volume)
PROMPT ============================================================================
-- Confirm installation with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20000, 'Installation aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT ============================================================================
PROMPT STEP 1: Backfill A_WORKFLOW_HISTORY_KEY for existing records
PROMPT ============================================================================
@@01_MARS_1409_POSTHOOK_update_existing_workflow_keys.sql
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409-POSTHOOK Installation Complete
PROMPT ============================================================================
PROMPT Final Status:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL;
PROMPT
PROMPT Review the log file for detailed results: &_filename
PROMPT ============================================================================
spool off
quit;

View File

@@ -0,0 +1,76 @@
-- ============================================================================
-- MARS-1409-POSTHOOK Master Rollback Script
-- ============================================================================
-- Purpose: Rollback MARS-1409-POSTHOOK - Clear backfilled A_WORKFLOW_HISTORY_KEY
-- values from A_SOURCE_FILE_RECEIVED.
-- Author: Grzegorz Michalski
-- Date: 2026-03-13
-- Note: This clears ALL non-NULL A_WORKFLOW_HISTORY_KEY values. The trigger
-- installed by MARS-1409 will repopulate them on next file processing.
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
SET VERIFY OFF
SET FEEDBACK ON
SET ECHO OFF
-- Create log directory if it doesn't exist
host mkdir log 2>nul
-- Generate dynamic SPOOL filename with timestamp
var filename VARCHAR2(100)
BEGIN
:filename := 'log/ROLLBACK_MARS_1409_POSTHOOK_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409-POSTHOOK Rollback Starting
PROMPT ============================================================================
PROMPT This will reverse all changes from MARS-1409-POSTHOOK installation.
PROMPT
PROMPT Rollback steps:
PROMPT 1. Clear A_WORKFLOW_HISTORY_KEY values from A_SOURCE_FILE_RECEIVED
PROMPT 2. Verify package versions
PROMPT ============================================================================
-- Confirm rollback with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20000, 'Rollback aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT ============================================================================
PROMPT STEP 1: Clear backfilled A_WORKFLOW_HISTORY_KEY values
PROMPT ============================================================================
@@91_MARS_1409_POSTHOOK_rollback_workflow_keys.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 2: Verify Package Versions
PROMPT ============================================================================
@@verify_packages_version.sql
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409-POSTHOOK Rollback Complete
PROMPT ============================================================================
PROMPT Final Status:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_end FROM DUAL;
PROMPT
PROMPT Review the log file for detailed results: &_filename
PROMPT ============================================================================
spool off
quit;