From 73e99b6e76cd3cae64a337792ccfcd56f0c4a549 Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Fri, 6 Mar 2026 12:06:18 +0100 Subject: [PATCH] MARS-1005 --- MARS_Packages/REL03/MARS-1005/.gitignore | 5 + .../01_MARS_1005_export_top_data.sql | 575 ++++++++++++++++++ .../MARS-1005/02_MARS_1005_verify_exports.sql | 190 ++++++ .../03_MARS_1005_verify_data_integrity.sql | 354 +++++++++++ ...90_MARS_1005_rollback_delete_csv_files.sql | 167 +++++ ..._MARS_1005_rollback_file_registrations.sql | 78 +++ .../92_MARS_1005_rollback_process_logs.sql | 77 +++ .../99_MARS_1005_verify_rollback.sql | 207 +++++++ .../REL03/MARS-1005/install_mars1005.sql | 88 +++ .../REL03/MARS-1005/rollback_mars1005.sql | 81 +++ 10 files changed, 1822 insertions(+) create mode 100644 MARS_Packages/REL03/MARS-1005/.gitignore create mode 100644 MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql create mode 100644 MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql create mode 100644 MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql create mode 100644 MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql create mode 100644 MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql create mode 100644 MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql create mode 100644 MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql create mode 100644 MARS_Packages/REL03/MARS-1005/install_mars1005.sql create mode 100644 MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql diff --git a/MARS_Packages/REL03/MARS-1005/.gitignore b/MARS_Packages/REL03/MARS-1005/.gitignore new file mode 100644 index 0000000..754791d --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/.gitignore @@ -0,0 +1,5 @@ +# Exclude temporary folders from version control +confluence/ +log/ +test/ +mock_data/ diff --git a/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql b/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql new file mode 100644 index 0000000..5d83cf6 --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql @@ -0,0 +1,575 @@ +-- ===================================================================================== +-- Script: 01_MARS_1005_export_top_data.sql +-- Purpose: Export OU_TOP historical data to ODS bucket (DATA bucket, CSV format) +-- Author: Grzegorz Michalski +-- Created: 2026-03-06 +-- MARS Issue: MARS-1005 +-- Target: mrds_data_dev/ODS/TOP/ +-- Tables: +-- 1. OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT +-- 2. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER +-- 3. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM +-- 4. OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT +-- 5. OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM +-- 6. OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED +-- ===================================================================================== + +SET SERVEROUTPUT ON SIZE UNLIMITED; +SET TIMING ON; + +PROMPT ===================================================================================== +PROMPT MARS-1005: OU_TOP Historical Data Export +PROMPT ===================================================================================== +PROMPT Export Strategy: +PROMPT - Source: OU_TOP schema tables (operational database) +PROMPT - Target: DATA/ODS bucket as CSV files +PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA +PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED +PROMPT - Path Structure: ODS/TOP/TOP_*/ +PROMPT Tables (6): +PROMPT ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM, +PROMPT ANNOUNCEMENT, FBL_ITEM (->TOP_FULLBIDLIST_ITEM), FULLBID_ARRAY_COMPILED +PROMPT ===================================================================================== + +-- Log export start +INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS) +VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Starting historical OU_TOP data export', + 'Tables: ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM, ANNOUNCEMENT, FBL_ITEM, FULLBID_ARRAY_COMPILED'); + +PROMPT +PROMPT ===================================================================================== +PROMPT PRE-EXPORT CHECK: Verify Existing Files in ODS Bucket +PROMPT ===================================================================================== + +-- Helper procedure (inline) to check one folder +-- Check 1: ALLOTMENT +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT files found - bucket is clean'); + END IF; +END; +/ + +-- Check 2: ALLOTMENT_MODIFICATION_HEADER +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_HEADER files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_HEADER files found - bucket is clean'); + END IF; +END; +/ + +-- Check 3: ALLOTMENT_MODIFICATION_ITEM +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_ITEM files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_ITEM files found - bucket is clean'); + END IF; +END; +/ + +-- Check 4: ANNOUNCEMENT +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ANNOUNCEMENT/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ANNOUNCEMENT files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ANNOUNCEMENT files found - bucket is clean'); + END IF; +END; +/ + +-- Check 5: FBL_ITEM (folder: TOP_FULLBIDLIST_ITEM) +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBIDLIST_ITEM/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBIDLIST_ITEM files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBIDLIST_ITEM files found - bucket is clean'); + END IF; +END; +/ + +-- Check 6: FULLBID_ARRAY_COMPILED +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/'; + SELECT COUNT(*) INTO vFileCount + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) + WHERE object_name NOT LIKE '%/'; + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBID_ARRAY_COMPILED files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); + FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + END LOOP; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); + EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM); + END; + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBID_ARRAY_COMPILED files found - bucket is clean'); + END IF; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT PRE-EXPORT: Verify Source and Target Table Readiness +PROMPT ===================================================================================== + +DECLARE + v1Source NUMBER := 0; v2Source NUMBER := 0; v3Source NUMBER := 0; + v4Source NUMBER := 0; v5Source NUMBER := 0; v6Source NUMBER := 0; + v1Target NUMBER := 0; v2Target NUMBER := 0; v3Target NUMBER := 0; + v4Target NUMBER := 0; v5Target NUMBER := 0; v6Target NUMBER := 0; + vTotalSource NUMBER := 0; + vTotalTarget NUMBER := 0; + + PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS + BEGIN + EXECUTE IMMEDIATE pSql INTO pResult; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN + pResult := 0; + ELSE + pResult := -1; + END IF; + END; +BEGIN + -- Source counts + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Source; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Source; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Source; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Source; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Source; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Source; + vTotalSource := v1Source + v2Source + v3Source + v4Source + v5Source + v6Source; + + DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):'); + DBMS_OUTPUT.PUT_LINE('- ALLOTMENT: ' || v1Source); + DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_HEADER: ' || v2Source); + DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_ITEM: ' || v3Source); + DBMS_OUTPUT.PUT_LINE('- ANNOUNCEMENT: ' || v4Source); + DBMS_OUTPUT.PUT_LINE('- FBL_ITEM: ' || v5Source); + DBMS_OUTPUT.PUT_LINE('- FULLBID_ARRAY_COMPILED: ' || v6Source); + DBMS_OUTPUT.PUT_LINE('- TOTAL SOURCE: ' || vTotalSource); + + -- Target external table counts + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1Target); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2Target); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3Target); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4Target); + safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5Target); + safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6Target); + + vTotalTarget := GREATEST(v1Target,0) + GREATEST(v2Target,0) + GREATEST(v3Target,0) + + GREATEST(v4Target,0) + GREATEST(v5Target,0) + GREATEST(v6Target,0); + + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('Target external table record counts (pre-export):'); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_ODS: ' || CASE WHEN v1Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v1Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_HEADER_ODS: ' || CASE WHEN v2Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v2Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_ITEM_ODS: ' || CASE WHEN v3Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v3Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_ANNOUNCEMENT_ODS: ' || CASE WHEN v4Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v4Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_FULLBIDLIST_ITEM_ODS: ' || CASE WHEN v5Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v5Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_FULLBID_ARRAY_COMPILED_ODS: ' || CASE WHEN v6Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v6Target) END); + + IF vTotalSource > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: No source data found'); + END IF; + + IF vTotalTarget = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export'); + ELSIF vTotalTarget > 0 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access target external tables'); + END IF; + + DBMS_OUTPUT.PUT_LINE('Proceeding with export...'); +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 1/6: OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_ALLOTMENT', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_ALLOTMENT', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 2/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_HEADER', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_HEADER export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_HEADER export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_HEADER', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 3/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_ITEM', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_ITEM export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_ITEM export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 4/6: OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_ANNOUNCEMENT', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_ANNOUNCEMENT', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ANNOUNCEMENT', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ANNOUNCEMENT export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: ANNOUNCEMENT export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_ANNOUNCEMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 5/6: OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_FBL_ITEM', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_FULLBIDLIST_ITEM', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: FBL_ITEM export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: FBL_ITEM export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_FBL_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT TABLE 6/6: OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED +PROMPT ===================================================================================== + +BEGIN + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( + pSchemaName => 'OU_TOP', + pTableName => 'LEGACY_FULLBID_ARRAY_COMPILED', + pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pBucketArea => 'ODS', + pFolderName => 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED', + pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED', + pMaxFileSize => 104857600, + pRegisterExport => TRUE, + pProcessName => 'MARS-1005' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: FULLBID_ARRAY_COMPILED export completed successfully'); +EXCEPTION + WHEN OTHERS THEN + DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN + DBMS_OUTPUT.PUT_LINE('ERROR: FULLBID_ARRAY_COMPILED export failed: ' || vErrorMsg); + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'), + 'MARS-1005', 'MARS-1005', 'EXPORT_FULLBID_ARRAY_COMPILED', NULL, 'ERROR', 'Export failed: ' || vErrorMsg); + COMMIT; + END; +END; +/ + +PROMPT +PROMPT ===================================================================================== +PROMPT Export Summary - Checking Results +PROMPT ===================================================================================== + +-- Log completion +INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) +VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'All OU_TOP historical exports completed successfully'); + +PROMPT +PROMPT ===================================================================================== +PROMPT MARS-1005 OU_TOP Export Completed! +PROMPT ===================================================================================== +PROMPT POST-EXPORT: Source vs Target Record Count Comparison +PROMPT ===================================================================================== + +DECLARE + v1S NUMBER := 0; v2S NUMBER := 0; v3S NUMBER := 0; + v4S NUMBER := 0; v5S NUMBER := 0; v6S NUMBER := 0; + v1T NUMBER := 0; v2T NUMBER := 0; v3T NUMBER := 0; + v4T NUMBER := 0; v5T NUMBER := 0; v6T NUMBER := 0; + vTotalS NUMBER := 0; + vTotalT NUMBER := 0; + vMismatch NUMBER := 0; + + PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS + BEGIN + EXECUTE IMMEDIATE pSql INTO pResult; + EXCEPTION WHEN OTHERS THEN pResult := -1; + END; + + PROCEDURE print_row(pTable VARCHAR2, pSrc NUMBER, pTgt NUMBER) IS + BEGIN + DBMS_OUTPUT.PUT_LINE( + RPAD(pTable, 40) || ' | ' || + RPAD(CASE WHEN pSrc = -1 THEN 'N/A' ELSE TO_CHAR(pSrc) END, 8) || ' | ' || + RPAD(CASE WHEN pTgt = -1 THEN 'ERROR' ELSE TO_CHAR(pTgt) END, 8) || ' | ' || + CASE WHEN pSrc = pTgt THEN 'OK' ELSE 'MISMATCH' END); + END; +BEGIN + -- Source + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1S; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2S; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3S; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4S; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5S; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6S; + vTotalS := v1S + v2S + v3S + v4S + v5S + v6S; + + -- Target + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1T); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2T); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3T); + safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4T); + safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5T); + safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6T); + vTotalT := GREATEST(v1T,0) + GREATEST(v2T,0) + GREATEST(v3T,0) + + GREATEST(v4T,0) + GREATEST(v5T,0) + GREATEST(v6T,0); + + DBMS_OUTPUT.PUT_LINE('POST-EXPORT VERIFICATION SUMMARY'); + DBMS_OUTPUT.PUT_LINE(RPAD('Table', 40) || ' | Source | Target | Match'); + DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-')); + print_row('ALLOTMENT', v1S, v1T); IF v1S != v1T THEN vMismatch := vMismatch + 1; END IF; + print_row('ALLOTMENT_MODIFICATION_HEADER', v2S, v2T); IF v2S != v2T THEN vMismatch := vMismatch + 1; END IF; + print_row('ALLOTMENT_MODIFICATION_ITEM', v3S, v3T); IF v3S != v3T THEN vMismatch := vMismatch + 1; END IF; + print_row('ANNOUNCEMENT', v4S, v4T); IF v4S != v4T THEN vMismatch := vMismatch + 1; END IF; + print_row('FBL_ITEM (->FULLBIDLIST_ITEM)', v5S, v5T); IF v5S != v5T THEN vMismatch := vMismatch + 1; END IF; + print_row('FULLBID_ARRAY_COMPILED', v6S, v6T); IF v6S != v6T THEN vMismatch := vMismatch + 1; END IF; + DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-')); + print_row('TOTAL', vTotalS, vTotalT); + + DBMS_OUTPUT.PUT_LINE(''); + IF vMismatch = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All record counts match - export verified'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vMismatch || ' table(s) have record count mismatches'); + DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions'); + END IF; +END; +/ + +-- Log export completion +INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS) +VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Historical OU_TOP data export completed', + 'Check verification scripts for detailed results'); + +COMMIT; + +PROMPT +PROMPT ===================================================================================== +PROMPT MARS-1005 OU_TOP Historical Data Export - COMPLETED +PROMPT +PROMPT Next steps: +PROMPT 1. Run: @02_MARS_1005_verify_exports.sql (verify file registration) +PROMPT 2. Run: @03_MARS_1005_verify_data_integrity.sql (full data verification) +PROMPT ===================================================================================== \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql b/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql new file mode 100644 index 0000000..89bee24 --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql @@ -0,0 +1,190 @@ +-- =================================================================== +-- MARS-1005 Verify Exports: Check Export Results and File Creation +-- =================================================================== +-- Purpose: Verify that C2D MPEC export completed successfully +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ========================================================================= +PROMPT MARS-1005 Export Verification +PROMPT ========================================================================= + +-- Check 1: Verify files were registered in A_SOURCE_FILE_RECEIVED +PROMPT Checking export file registration... +DECLARE + vFileCount NUMBER := 0; + vTotalBytes NUMBER := 0; +BEGIN + SELECT COUNT(*), NVL(SUM(BYTES), 0) + INTO vFileCount, vTotalBytes + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour + AND (SOURCE_FILE_NAME LIKE '2001_%' -- MPEC_ADMIN ETL keys + OR SOURCE_FILE_NAME LIKE '2002_%' + OR SOURCE_FILE_NAME LIKE '2003_%' + OR SOURCE_FILE_NAME LIKE '2004_%' + OR SOURCE_FILE_NAME LIKE '2005_%' + OR SOURCE_FILE_NAME LIKE '2006_%' -- MPEC_CONTENT ETL keys + OR SOURCE_FILE_NAME LIKE '2007_%' + OR SOURCE_FILE_NAME LIKE '2008_%' + OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys + OR SOURCE_FILE_NAME LIKE '2010_%'); + + DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB'); + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration'); + ELSIF vFileCount < 9 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount); + ELSE + DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found'); + END IF; +END; +/ + +-- Check 2: Show recent export registrations +PROMPT Recent export file registrations: +SELECT + SUBSTR(SOURCE_FILE_NAME, 1, 40) AS FILE_NAME, + A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY, + PROCESSING_STATUS, + ROUND(BYTES/1024, 2) AS SIZE_KB, + TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour + AND (SOURCE_FILE_NAME LIKE '200%') -- ETL keys starting with 200 +ORDER BY RECEPTION_DATE DESC; + +-- Check 3: Verify export process logs +PROMPT Checking export process logs... +DECLARE + vLogCount NUMBER := 0; + vErrorCount NUMBER := 0; +BEGIN + SELECT COUNT(*), SUM(CASE WHEN LOG_LEVEL = 'ERROR' THEN 1 ELSE 0 END) + INTO vLogCount, vErrorCount + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: Process log entries: ' || vLogCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Error entries: ' || vErrorCount); + + IF vErrorCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vErrorCount || ' errors found in process log'); + ELSE + DBMS_OUTPUT.PUT_LINE('SUCCESS: No errors found in process log'); + END IF; +END; +/ + +-- Check 4: Display recent process logs +PROMPT Recent MARS-1005 process logs: +SELECT + TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS TIME, + PROCEDURE_NAME, + LOG_LEVEL, + SUBSTR(LOG_MESSAGE, 1, 60) AS MESSAGE +FROM CT_MRDS.A_PROCESS_LOG +WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR +ORDER BY LOG_TIMESTAMP DESC +FETCH FIRST 10 ROWS ONLY; + +-- Check 5: Cloud bucket file verification (if cloud_wrapper available) +PROMPT Checking cloud bucket files... +DECLARE + vCloudFileCount NUMBER := 0; + vCredentialName VARCHAR2(100); + vDataBucketUri VARCHAR2(500); +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri); + + -- Count files in cloud bucket + BEGIN + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri + )) + WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%' + ) LOOP + vCloudFileCount := vCloudFileCount + 1; + IF vCloudFileCount <= 5 THEN -- Show first 5 files + DBMS_OUTPUT.PUT_LINE('- ' || rec.object_name); + END IF; + END LOOP; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount); + + IF vCloudFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket'); + END IF; + + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM); + END; +END; +/ + +PROMPT +PROMPT ========================================================================= +PROMPT MARS-1005 Export Verification Summary +PROMPT ========================================================================= + +-- Final verification summary +DECLARE + vFileRegCount NUMBER := 0; + vCloudFileCount NUMBER := 0; + vLogErrorCount NUMBER := 0; + vOverallStatus VARCHAR2(20); +BEGIN + -- Count registered files + SELECT COUNT(*) + INTO vFileRegCount + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE RECEPTION_DATE >= SYSDATE - 1/24 + AND SOURCE_FILE_NAME LIKE '200%'; + + -- Count process errors + SELECT COUNT(*) + INTO vLogErrorCount + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_LEVEL = 'ERROR' + AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR; + + -- Determine overall status + IF vFileRegCount >= 9 AND vLogErrorCount = 0 THEN + vOverallStatus := 'SUCCESS'; + ELSIF vFileRegCount > 0 AND vLogErrorCount = 0 THEN + vOverallStatus := 'PARTIAL SUCCESS'; + ELSE + vOverallStatus := 'ISSUES DETECTED'; + END IF; + + DBMS_OUTPUT.PUT_LINE('MARS-1005 Export Verification: ' || vOverallStatus); + DBMS_OUTPUT.PUT_LINE('- Registered files: ' || vFileRegCount || ' (expected: 9)'); + DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount); + + IF vOverallStatus = 'SUCCESS' THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs'); + END IF; +END; +/ + +PROMPT ========================================================================= +PROMPT Export Verification Completed +PROMPT ========================================================================= \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql b/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql new file mode 100644 index 0000000..7b6a412 --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql @@ -0,0 +1,354 @@ +-- =================================================================== +-- MARS-1005 Verify Data Integrity: Source vs Exported Data Validation +-- =================================================================== +-- Purpose: Verify data integrity between source tables and exported files +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ========================================================================= +PROMPT MARS-1005 Data Integrity Verification +PROMPT ========================================================================= + +-- Check 1: Source table record counts vs expected ETL keys +PROMPT Checking source table record counts... +DECLARE + vAdminRows NUMBER := 0; + vContentRows NUMBER := 0; + vCriterionRows NUMBER := 0; + vTotalRows NUMBER := 0; + vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average +BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; + + vTotalRows := vAdminRows + vContentRows + vCriterionRows; + + DBMS_OUTPUT.PUT_LINE('Source table record counts:'); + DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' records'); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT: ' || vContentRows || ' records'); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' records'); + DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records'); + + IF vTotalRows > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables'); + END IF; +END; +/ + +-- Check 2: ETL key distribution analysis +PROMPT Checking ETL key distribution... +DECLARE + vAdminKeys NUMBER := 0; + vContentKeys NUMBER := 0; + vCriterionKeys NUMBER := 0; + vTotalKeys NUMBER := 0; +BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; + + SELECT COUNT(DISTINCT etl_key) + INTO vTotalKeys + FROM ( + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION + ); + + DBMS_OUTPUT.PUT_LINE('ETL key distribution:'); + DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct keys: ' || vAdminKeys); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct keys: ' || vContentKeys); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct keys: ' || vCriterionKeys); + DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys); + + IF vTotalKeys > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data'); + END IF; +END; +/ + +-- Check 3: Template table compatibility verification +PROMPT Checking template table compatibility... +DECLARE + vAdminCols NUMBER := 0; + vContentCols NUMBER := 0; + vCriterionCols NUMBER := 0; +BEGIN + -- Check MPEC_ADMIN template compatibility + SELECT COUNT(*) + INTO vAdminCols + FROM all_tab_columns + WHERE owner = 'CT_ET_TEMPLATES' + AND table_name = 'C2D_MPEC_ADMIN'; + + -- Check MPEC_CONTENT template compatibility + SELECT COUNT(*) + INTO vContentCols + FROM all_tab_columns + WHERE owner = 'CT_ET_TEMPLATES' + AND table_name = 'C2D_MPEC_CONTENT'; + + -- Check MPEC_CONTENT_CRITERION template compatibility + SELECT COUNT(*) + INTO vCriterionCols + FROM all_tab_columns + WHERE owner = 'CT_ET_TEMPLATES' + AND table_name = 'C2D_MPEC_CONTENT_CRITERION'; + + DBMS_OUTPUT.PUT_LINE('Template table column counts:'); + DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_ADMIN: ' || vAdminCols || ' columns'); + DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT: ' || vContentCols || ' columns'); + DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); + + IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns'); + END IF; +END; +/ + +-- Check 4: Verify A_ETL_LOAD_SET_FK values exist in A_LOAD_HISTORY +PROMPT Checking ETL key references in A_LOAD_HISTORY... +DECLARE + vValidKeys NUMBER := 0; + vTotalSourceKeys NUMBER := 0; +BEGIN + -- Count total distinct ETL keys in source tables + SELECT COUNT(DISTINCT etl_key) + INTO vTotalSourceKeys + FROM ( + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION + ); + + -- Count how many exist in A_LOAD_HISTORY + SELECT COUNT(DISTINCT etl_key) + INTO vValidKeys + FROM ( + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT + UNION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION + ) src + WHERE EXISTS ( + SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h + WHERE h.A_ETL_LOAD_SET_KEY = src.etl_key + ); + + DBMS_OUTPUT.PUT_LINE('ETL key validation:'); + DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys in source: ' || vTotalSourceKeys); + DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); + + IF vValidKeys = vTotalSourceKeys THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid'); + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); + END IF; +END; +/ + +PROMPT ===================================================================================== +PROMPT MARS-1005 Record Count Verification +PROMPT ===================================================================================== +PROMPT Comparing source table counts with exported external table counts +PROMPT ===================================================================================== + +DECLARE + TYPE t_table_info IS RECORD ( + source_schema VARCHAR2(50), + source_table VARCHAR2(100), + external_table VARCHAR2(100), + description VARCHAR2(200) + ); + TYPE t_table_list IS TABLE OF t_table_info; + + vTables t_table_list; + vSourceCount NUMBER; + vTargetCount NUMBER; + vTotalSourceCount NUMBER := 0; + vTotalTargetCount NUMBER := 0; + vMismatchCount NUMBER := 0; + vSql VARCHAR2(4000); + vFileCount NUMBER := 0; + vValidationResult VARCHAR2(100); +BEGIN + DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS')); + DBMS_OUTPUT.PUT_LINE(''); + + -- Initialize table list with C2D MPEC configuration + vTables := t_table_list( + t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'), + t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'), + t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)') + ); + + DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status'); + DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); + + FOR i IN 1..vTables.COUNT LOOP + -- Get source table count + vSql := 'SELECT COUNT(*) FROM ' || vTables(i).source_schema || '.' || vTables(i).source_table; + + BEGIN + EXECUTE IMMEDIATE vSql INTO vSourceCount; + vTotalSourceCount := vTotalSourceCount + vSourceCount; + EXCEPTION + WHEN OTHERS THEN + vSourceCount := -1; + DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table'); + CONTINUE; + END; + + -- Get target external table count + vSql := 'SELECT COUNT(*) FROM ' || vTables(i).external_table; + BEGIN + EXECUTE IMMEDIATE vSql INTO vTargetCount; + vTotalTargetCount := vTotalTargetCount + vTargetCount; + EXCEPTION + WHEN OTHERS THEN + -- Handle expected errors for empty external tables + -- ORA-29913: error in executing ODCIEXTTABLEOPEN callout + -- ORA-29400: data cartridge error + -- KUP-13023: nothing matched wildcard query (no files in bucket) + -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty + IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN + vTargetCount := 0; -- Treat as empty (no files exported yet) + ELSE + vTargetCount := -1; -- Real error + END IF; + END; + + -- Display comparison results with thousands separators + DECLARE + vStatus VARCHAR2(20); + vSourceDisplay VARCHAR2(17); + vTargetDisplay VARCHAR2(17); + BEGIN + -- Format source count display + IF vSourceCount = -1 THEN + vSourceDisplay := 'ERROR'; + ELSE + vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999'); + END IF; + + -- Format target count display + IF vTargetCount = -1 THEN + vTargetDisplay := 'ERROR'; + ELSE + vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999'); + END IF; + + -- Determine status + IF vSourceCount = vTargetCount THEN + vStatus := 'PASS'; + ELSIF vTargetCount = -1 THEN + vStatus := 'ERROR'; + vMismatchCount := vMismatchCount + 1; + ELSIF vSourceCount = -1 THEN + vStatus := 'ERROR'; + vMismatchCount := vMismatchCount + 1; + ELSE + vStatus := 'MISMATCH'; + vMismatchCount := vMismatchCount + 1; + END IF; + + DBMS_OUTPUT.PUT_LINE( + RPAD(vTables(i).source_table, 24) || + LPAD(vSourceDisplay, 15) || + LPAD(vTargetDisplay, 15) || ' ' || + vStatus + ); + END; + END LOOP; + + DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE( + RPAD('TOTALS', 24) || + LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) || + LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15) + ); + DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE(''); + + -- Count exported files for additional verification + SELECT COUNT(*) + INTO vFileCount + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE RECEPTION_DATE >= SYSDATE - 1/24 + AND (SOURCE_FILE_NAME LIKE '200_%'); + + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary'); + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999')); + DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)'); + DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE(''); + + IF vMismatchCount = 0 AND vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED'); + DBMS_OUTPUT.PUT_LINE(' All record counts match between source and exported data'); + DBMS_OUTPUT.PUT_LINE(' Export completed successfully'); + ELSIF vMismatchCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES'); + DBMS_OUTPUT.PUT_LINE(' Found ' || vMismatchCount || ' table(s) with count mismatches'); + DBMS_OUTPUT.PUT_LINE(' NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)'); + DBMS_OUTPUT.PUT_LINE(' Review export logs and pre-check results before re-running exports'); + ELSE + DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED'); + DBMS_OUTPUT.PUT_LINE(' No files found in export registration'); + DBMS_OUTPUT.PUT_LINE(' Verify export execution completed successfully'); + END IF; + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE(''); + + DBMS_OUTPUT.PUT_LINE('Legend:'); + DBMS_OUTPUT.PUT_LINE(' PASS - Record counts match (export successful)'); + DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)'); + DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files'); + DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)'); + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + + -- Additional ETL key analysis for C2D MPEC data + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('ETL Key Analysis:'); + + DECLARE + vAdminKeys NUMBER; + vContentKeys NUMBER; + vCriterionKeys NUMBER; + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; + + DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)'); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)'); + DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct ETL keys: ' || vCriterionKeys || ' (expected: 2 for keys 2009-2010)'); + + -- Expected file count = sum of distinct ETL keys per table + DBMS_OUTPUT.PUT_LINE('- Expected export files: ' || (vAdminKeys + vContentKeys + vCriterionKeys)); + DBMS_OUTPUT.PUT_LINE('- Actual export files: ' || vFileCount); + END; +END; +/ + +PROMPT ========================================================================= +PROMPT Data Integrity Verification Completed +PROMPT ========================================================================= \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql b/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql new file mode 100644 index 0000000..a243355 --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql @@ -0,0 +1,167 @@ +--============================================================================================================================= +-- MARS-1005 ROLLBACK: Delete Exported CSV Files from DATA Bucket +--============================================================================================================================= +-- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables +-- WARNING: This will permanently delete exported data files! +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 +-- Related: MARS-1005 - C2D MPEC Data Export Rollback +--============================================================================================================================= + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); + + -- Delete CSV files registered by MARS-1005 process + FOR rec IN ( + SELECT SOURCE_FILE_NAME AS object_name + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + AND SOURCE_FILE_NAME LIKE '%MPEC_ADMIN%' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); + + -- Delete CSV files registered by MARS-1005 process + FOR rec IN ( + SELECT SOURCE_FILE_NAME AS object_name + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT%' + AND SOURCE_FILE_NAME NOT LIKE '%CRITERION%' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); + + -- Delete CSV files registered by MARS-1005 process + FOR rec IN ( + SELECT SOURCE_FILE_NAME AS object_name + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT_CRITERION%' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT SUCCESS: All CSV file deletion operations completed diff --git a/MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql b/MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql new file mode 100644 index 0000000..8ab1891 --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql @@ -0,0 +1,78 @@ +-- =================================================================== +-- MARS-1005 Rollback Step 1: Delete File Registrations +-- =================================================================== +-- Purpose: Remove MARS-1005 export file registrations from A_SOURCE_FILE_RECEIVED +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ========================================================================= +PROMPT MARS-1005 Rollback Step 1: Delete File Registrations +PROMPT ========================================================================= + +DECLARE + vFileCount NUMBER := 0; + vDeletedCount NUMBER := 0; + vErrorMsg VARCHAR2(4000); +BEGIN + -- Count files to be deleted (using PROCESS_NAME) + SELECT COUNT(*) + INTO vFileCount + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005'; + + DBMS_OUTPUT.PUT_LINE('Files to be deleted: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('Using PROCESS_NAME = ''MARS-1005'' filter'); + + IF vFileCount > 0 THEN + -- Show files before deletion + DBMS_OUTPUT.PUT_LINE('Files being removed:'); + FOR rec IN ( + SELECT A_SOURCE_FILE_RECEIVED_KEY, + SUBSTR(SOURCE_FILE_NAME, 1, 60) AS FILE_NAME, + TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME, + PROCESS_NAME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + ORDER BY RECEPTION_DATE DESC + ) LOOP + DBMS_OUTPUT.PUT_LINE('- ' || rec.FILE_NAME || ' (ID: ' || rec.A_SOURCE_FILE_RECEIVED_KEY || ', Process: ' || rec.PROCESS_NAME || ')'); + END LOOP; + + -- Delete the file registrations using PROCESS_NAME + DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005'; + + vDeletedCount := SQL%ROWCOUNT; + COMMIT; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' file registrations'); + + -- Log the rollback action + INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) + VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'INFO', + 'Deleted ' || vDeletedCount || ' file registrations'); + COMMIT; + + ELSE + DBMS_OUTPUT.PUT_LINE('SUCCESS: No file registrations found to delete'); + END IF; + +EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + vErrorMsg := 'Failed to delete file registrations: ' || SQLERRM; + DBMS_OUTPUT.PUT_LINE('ERROR: Error during file registration deletion: ' || SQLERRM); + -- Log the error + INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) + VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', vErrorMsg); + COMMIT; + RAISE; +END; +/ + +PROMPT ========================================================================= +PROMPT File Registration Rollback Completed +PROMPT ========================================================================= \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql b/MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql new file mode 100644 index 0000000..3f0c5fb --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql @@ -0,0 +1,77 @@ +-- =================================================================== +-- MARS-1005 Rollback Step 2: Clean Process Logs +-- =================================================================== +-- Purpose: Remove MARS-1005 process logs from A_PROCESS_LOG +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ========================================================================= +PROMPT MARS-1005 Rollback Step 2: Clean Process Logs +PROMPT ========================================================================= + +DECLARE + vLogCount NUMBER := 0; + vDeletedCount NUMBER := 0; + vErrorMsg VARCHAR2(4000); +BEGIN + -- Count logs to be deleted + SELECT COUNT(*) + INTO vLogCount + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK') + AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week (safety) + + DBMS_OUTPUT.PUT_LINE('Process log entries to be deleted: ' || vLogCount); + + IF vLogCount > 0 THEN + -- Show recent logs before deletion + DBMS_OUTPUT.PUT_LINE('Recent MARS-1005 log entries being removed:'); + FOR rec IN ( + SELECT A_PROCESS_LOG_KEY, + TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME, + PROCEDURE_NAME, + LOG_LEVEL, + SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK') + AND LOG_TIMESTAMP >= SYSDATE - 7 + ORDER BY LOG_TIMESTAMP DESC + FETCH FIRST 10 ROWS ONLY + ) LOOP + DBMS_OUTPUT.PUT_LINE('- ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' || + rec.PROCEDURE_NAME || ': ' || rec.MESSAGE); + END LOOP; + + -- Delete the process logs + DELETE FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK') + AND LOG_TIMESTAMP >= SYSDATE - 7; + + vDeletedCount := SQL%ROWCOUNT; + COMMIT; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' process log entries'); + + ELSE + DBMS_OUTPUT.PUT_LINE('SUCCESS: No process log entries found to delete'); + END IF; + +EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + vErrorMsg := 'Failed to clean process logs: ' || SQLERRM; + DBMS_OUTPUT.PUT_LINE('ERROR: Error during process log cleanup: ' || SQLERRM); + -- Log the error (will remain after rollback for debugging) + INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) + VALUES ('MARS-1005-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', vErrorMsg); + COMMIT; + RAISE; +END; +/ + +PROMPT ========================================================================= +PROMPT Process Log Cleanup Completed +PROMPT ========================================================================= \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql b/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql new file mode 100644 index 0000000..30124bf --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql @@ -0,0 +1,207 @@ +-- =================================================================== +-- MARS-1005 Rollback Verification: Confirm Rollback Completion +-- =================================================================== +-- Purpose: Verify that MARS-1005 rollback completed successfully +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ========================================================================= +PROMPT MARS-1005 Rollback Verification +PROMPT ========================================================================= + +-- Check 1: Verify file registrations were removed +PROMPT Checking file registration cleanup... +DECLARE + vRemainingFiles NUMBER := 0; +BEGIN + SELECT COUNT(*) + INTO vRemainingFiles + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE SOURCE_FILE_NAME LIKE '200%' -- ETL keys 2001-2010 + AND RECEPTION_DATE >= SYSDATE - 7; -- Last week + + IF vRemainingFiles = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 file registrations successfully removed'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingFiles || ' file registrations still exist'); + + -- Show remaining files + FOR rec IN ( + SELECT SUBSTR(SOURCE_FILE_NAME, 1, 50) AS FILE_NAME, + TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE SOURCE_FILE_NAME LIKE '200%' + AND RECEPTION_DATE >= SYSDATE - 7 + ) LOOP + DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.FILE_NAME); + END LOOP; + END IF; +END; +/ + +-- Check 2: Verify process logs were cleaned +PROMPT Checking process log cleanup... +DECLARE + vRemainingLogs NUMBER := 0; +BEGIN + SELECT COUNT(*) + INTO vRemainingLogs + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week + + IF vRemainingLogs = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 process logs successfully removed'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingLogs || ' process log entries still exist'); + + -- Show remaining logs (first few) + FOR rec IN ( + SELECT TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME, + PROCEDURE_NAME, + SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_TIMESTAMP >= SYSDATE - 7 + ORDER BY LOG_TIMESTAMP DESC + FETCH FIRST 3 ROWS ONLY + ) LOOP + DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.LOG_TIME || ' ' || rec.PROCEDURE_NAME); + END LOOP; + END IF; +END; +/ + +-- Check 3: Verify cloud bucket cleanup (informational only) +PROMPT Checking cloud bucket status... +DECLARE + vCloudFileCount NUMBER := 0; + vCredentialName VARCHAR2(100); + vDataBucketUri VARCHAR2(500); +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri); + + -- Count remaining files in cloud bucket + BEGIN + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri + )) + WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%' + ) LOOP + vCloudFileCount := vCloudFileCount + 1; + IF vCloudFileCount <= 3 THEN -- Show first 3 files + DBMS_OUTPUT.PUT_LINE(' Cloud file: ' || rec.object_name); + END IF; + END LOOP; + + IF vCloudFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket'); + ELSE + DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket'); + DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback'); + DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed'); + END IF; + + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot check cloud bucket: ' || SQLERRM); + END; +END; +/ + +-- Check 4: Verify rollback logs were created +PROMPT Checking rollback operation logs... +DECLARE + vRollbackLogs NUMBER := 0; +BEGIN + SELECT COUNT(*) + INTO vRollbackLogs + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK' + AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour + + IF vRollbackLogs > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback operation logs found: ' || vRollbackLogs); + + -- Show recent rollback logs + FOR rec IN ( + SELECT TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS LOG_TIME, + PROCEDURE_NAME, + LOG_LEVEL, + SUBSTR(LOG_MESSAGE, 1, 50) AS MESSAGE + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK' + AND LOG_TIMESTAMP >= SYSDATE - 1/24 + ORDER BY LOG_TIMESTAMP DESC + ) LOOP + DBMS_OUTPUT.PUT_LINE(' ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' || + rec.PROCEDURE_NAME || ': ' || rec.MESSAGE); + END LOOP; + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: Warning: No rollback operation logs found'); + END IF; +END; +/ + +PROMPT +PROMPT ========================================================================= +PROMPT MARS-1005 Rollback Verification Summary +PROMPT ========================================================================= + +DECLARE + vRemainingFiles NUMBER := 0; + vRemainingLogs NUMBER := 0; + vRollbackStatus VARCHAR2(20); +BEGIN + -- Count remaining registrations + SELECT COUNT(*) + INTO vRemainingFiles + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE SOURCE_FILE_NAME LIKE '200%' + AND RECEPTION_DATE >= SYSDATE - 7; + + -- Count remaining process logs + SELECT COUNT(*) + INTO vRemainingLogs + FROM CT_MRDS.A_PROCESS_LOG + WHERE PROCESS_NAME = 'MARS-1005' + AND LOG_TIMESTAMP >= SYSDATE - 7; + + -- Determine rollback status + IF vRemainingFiles = 0 AND vRemainingLogs = 0 THEN + vRollbackStatus := 'COMPLETE'; + ELSIF vRemainingFiles = 0 OR vRemainingLogs = 0 THEN + vRollbackStatus := 'PARTIAL'; + ELSE + vRollbackStatus := 'INCOMPLETE'; + END IF; + + DBMS_OUTPUT.PUT_LINE('MARS-1005 Rollback Status: ' || vRollbackStatus); + DBMS_OUTPUT.PUT_LINE('- Remaining file registrations: ' || vRemainingFiles); + DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs); + + IF vRollbackStatus = 'COMPLETE' THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback completed successfully - system clean'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: Rollback incomplete - manual cleanup may be required'); + END IF; + + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('Note: Cloud bucket files (OCI) are not automatically removed'); + DBMS_OUTPUT.PUT_LINE(' Use OCI console or DBMS_CLOUD commands for file deletion if needed'); +END; +/ + +PROMPT ========================================================================= +PROMPT Rollback Verification Completed +PROMPT ========================================================================= \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/install_mars1005.sql b/MARS_Packages/REL03/MARS-1005/install_mars1005.sql new file mode 100644 index 0000000..742fd6d --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/install_mars1005.sql @@ -0,0 +1,88 @@ +-- =================================================================== +-- MARS-1005 INSTALL SCRIPT: C2D MPEC Data Export to External Tables +-- =================================================================== +-- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema +-- to OCI buckets (ODS bucket CSV format) +-- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) +-- Log files are automatically created in log/ subdirectory +-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir) +host mkdir log 2>nul + +var filename VARCHAR2(100) +BEGIN + :filename := 'log/INSTALL_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +SET ECHO OFF +SET TIMING ON +SET SERVEROUTPUT ON SIZE UNLIMITED +SET PAUSE OFF + +-- Set current schema context (optional - use when modifying packages in specific schema) +-- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS; + +PROMPT ========================================================================= +PROMPT MARS-1005: C2D MPEC Data Export to External Tables (One-Time Migration) +PROMPT ========================================================================= +PROMPT +PROMPT This script will export 3 C2D MPEC tables to OCI buckets: +PROMPT +PROMPT TARGET: ODS Bucket (CSV format): +PROMPT - MPEC_ADMIN +PROMPT - MPEC_CONTENT +PROMPT - MPEC_CONTENT_CRITERION +PROMPT +PROMPT Key Features: +PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking +PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*) +PROMPT - ODS/C2D bucket path structure +PROMPT ========================================================================= + +-- Confirm installation with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20001, 'Installation aborted by user'); + END IF; +END; +/ +WHENEVER SQLERROR CONTINUE + +PROMPT +PROMPT ========================================================================= +PROMPT Step 1: Export OU_TOP Data to ODS Bucket +PROMPT ========================================================================= +@@01_MARS_1005_export_top_data.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 2: Verify Exports (File Registration Check) +PROMPT ========================================================================= +@@02_MARS_1005_verify_exports.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 3: Verify Data Integrity (Source vs Exported) +PROMPT ========================================================================= +@@03_MARS_1005_verify_data_integrity.sql + +PROMPT +PROMPT ========================================================================= +PROMPT MARS-1005 Installation - COMPLETED +PROMPT ========================================================================= +PROMPT Check the log file for complete installation details. +PROMPT For rollback, use: rollback_mars1005.sql +PROMPT ========================================================================= + +spool off + +quit; \ No newline at end of file diff --git a/MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql b/MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql new file mode 100644 index 0000000..11d46af --- /dev/null +++ b/MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql @@ -0,0 +1,81 @@ +-- =================================================================== +-- MARS-1005 ROLLBACK SCRIPT: C2D MPEC Data Export Rollback +-- =================================================================== +-- Purpose: Rollback MARS-1005 - Delete exported CSV files and file registrations +-- WARNING: This will DELETE all exported data files and registrations! +-- Author: Grzegorz Michalski +-- Date: 2026-02-12 + +-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) +-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir) +host mkdir log 2>nul + +var filename VARCHAR2(100) +BEGIN + :filename := 'log/ROLLBACK_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +SET ECHO OFF +SET TIMING ON +SET SERVEROUTPUT ON SIZE UNLIMITED +SET PAUSE OFF + +PROMPT ========================================================================= +PROMPT MARS-1005: Rollback C2D MPEC Data Export +PROMPT ========================================================================= +PROMPT WARNING: This will DELETE exported CSV files and file registrations! +PROMPT - ODS bucket: mrds_data_dev/ODS/C2D/ +PROMPT - File registrations: A_SOURCE_FILE_RECEIVED entries +PROMPT +PROMPT Only proceed if export failed and needs to be restarted! +PROMPT ========================================================================= + +-- Confirm rollback with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user'); + END IF; +END; +/ +WHENEVER SQLERROR CONTINUE + +PROMPT +PROMPT ========================================================================= +PROMPT Step 1: Delete Exported CSV Files from DATA Bucket +PROMPT ========================================================================= +@@90_MARS_1005_rollback_delete_csv_files.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 2: Delete File Registrations +PROMPT ========================================================================= +@@91_MARS_1005_rollback_file_registrations.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 3: Clean Process Logs +PROMPT ========================================================================= +@@92_MARS_1005_rollback_process_logs.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 4: Verify Rollback Completion +PROMPT ========================================================================= +@@99_MARS_1005_verify_rollback.sql + +PROMPT +PROMPT ========================================================================= +PROMPT MARS-1005 Rollback - COMPLETED +PROMPT ========================================================================= +PROMPT Check the log file for complete rollback details. +PROMPT ========================================================================= + +spool off + +quit; \ No newline at end of file