From b81e524351d793c0e3d4f8cc93dfaead1df33564 Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Fri, 6 Mar 2026 14:34:12 +0100 Subject: [PATCH] Refactor MARS-1005 scripts for OU_TOP legacy data export and rollback - Updated SQL scripts to verify data integrity for 6 OU_TOP.LEGACY_* tables instead of 3 C2D MPEC tables. - Modified rollback script to delete exported CSV files from ODS/TOP/ bucket paths. - Enhanced verification script to check for remaining files and cloud bucket contents specific to MARS-1005. - Adjusted install script to reflect changes in target tables and their corresponding paths in the ODS bucket. - Updated README to include instructions for the new MARS-1005 installation and rollback processes. --- .../MARS-1005/02_MARS_1005_verify_exports.sql | 197 +++++++------ .../03_MARS_1005_verify_data_integrity.sql | 275 +++++++++--------- ...90_MARS_1005_rollback_delete_csv_files.sql | 244 +++++++++------- .../99_MARS_1005_verify_rollback.sql | 14 +- .../REL03/MARS-1005/install_mars1005.sql | 31 +- README.md | 5 + 6 files changed, 423 insertions(+), 343 deletions(-) diff --git a/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql b/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql index 89bee24..d97f77e 100644 --- a/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql +++ b/MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql @@ -1,9 +1,11 @@ -- =================================================================== -- MARS-1005 Verify Exports: Check Export Results and File Creation -- =================================================================== --- Purpose: Verify that C2D MPEC export completed successfully +-- Purpose: Verify that OU_TOP historical data export completed successfully -- Author: Grzegorz Michalski --- Date: 2026-02-12 +-- Date: 2026-03-06 +-- MARS Issue: MARS-1005 +-- Tables: 6 OU_TOP.LEGACY_* tables exported to ODS/TOP/ bucket paths SET SERVEROUTPUT ON SIZE UNLIMITED SET TIMING ON @@ -13,51 +15,57 @@ PROMPT MARS-1005 Export Verification PROMPT ========================================================================= -- Check 1: Verify files were registered in A_SOURCE_FILE_RECEIVED -PROMPT Checking export file registration... +PROMPT Checking export file registration (PROCESS_NAME = MARS-1005)... DECLARE - vFileCount NUMBER := 0; - vTotalBytes NUMBER := 0; + vFileCount NUMBER := 0; + vTotalBytes NUMBER := 0; BEGIN SELECT COUNT(*), NVL(SUM(BYTES), 0) INTO vFileCount, vTotalBytes - FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour - AND (SOURCE_FILE_NAME LIKE '2001_%' -- MPEC_ADMIN ETL keys - OR SOURCE_FILE_NAME LIKE '2002_%' - OR SOURCE_FILE_NAME LIKE '2003_%' - OR SOURCE_FILE_NAME LIKE '2004_%' - OR SOURCE_FILE_NAME LIKE '2005_%' - OR SOURCE_FILE_NAME LIKE '2006_%' -- MPEC_CONTENT ETL keys - OR SOURCE_FILE_NAME LIKE '2007_%' - OR SOURCE_FILE_NAME LIKE '2008_%' - OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys - OR SOURCE_FILE_NAME LIKE '2010_%'); - - DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount); - DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB'); - + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + AND RECEPTION_DATE >= SYSDATE - 1/24; -- Last hour + + DBMS_OUTPUT.PUT_LINE('Registered export files (last hour): ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('Total file size: ' || ROUND(vTotalBytes / 1024, 2) || ' KB'); + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration'); - ELSIF vFileCount < 9 THEN - DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount); + ELSIF vFileCount < 6 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Expected at least 6 files (1 per table), found: ' || vFileCount); ELSE - DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files registered (>= 6)'); END IF; END; / --- Check 2: Show recent export registrations -PROMPT Recent export file registrations: -SELECT - SUBSTR(SOURCE_FILE_NAME, 1, 40) AS FILE_NAME, - A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY, +-- Check 2: Show recent export registrations by table +PROMPT Recent export file registrations per table: +SELECT + A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY, + SUBSTR(SOURCE_FILE_NAME, 1, 55) AS FILE_NAME, PROCESSING_STATUS, - ROUND(BYTES/1024, 2) AS SIZE_KB, - TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED -FROM CT_MRDS.A_SOURCE_FILE_RECEIVED -WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour - AND (SOURCE_FILE_NAME LIKE '200%') -- ETL keys starting with 200 -ORDER BY RECEPTION_DATE DESC; + ROUND(NVL(BYTES, 0) / 1024, 2) AS SIZE_KB, + TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +WHERE PROCESS_NAME = 'MARS-1005' + AND RECEPTION_DATE >= SYSDATE - 1/24 +ORDER BY A_SOURCE_FILE_CONFIG_KEY, RECEPTION_DATE DESC; + +-- Check 2b: File count per A_SOURCE_FILE_CONFIG_KEY +PROMPT Export file count per source config key: +SELECT + r.A_SOURCE_FILE_CONFIG_KEY, + c.TABLE_ID, + COUNT(*) AS FILE_COUNT, + ROUND(NVL(SUM(r.BYTES), 0) / 1024, 2) AS TOTAL_KB +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r +JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY +WHERE r.PROCESS_NAME = 'MARS-1005' + AND r.RECEPTION_DATE >= SYSDATE - 1/24 +GROUP BY r.A_SOURCE_FILE_CONFIG_KEY, c.TABLE_ID +ORDER BY r.A_SOURCE_FILE_CONFIG_KEY; -- Check 3: Verify export process logs PROMPT Checking export process logs... @@ -95,92 +103,109 @@ WHERE PROCESS_NAME = 'MARS-1005' ORDER BY LOG_TIMESTAMP DESC FETCH FIRST 10 ROWS ONLY; --- Check 5: Cloud bucket file verification (if cloud_wrapper available) -PROMPT Checking cloud bucket files... +-- Check 5: Cloud bucket file verification across all 6 TOP folders +PROMPT Checking cloud bucket files in ODS/TOP/ paths... DECLARE - vCloudFileCount NUMBER := 0; - vCredentialName VARCHAR2(100); - vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL'; + vDataBucketUri VARCHAR2(500); + vTotalFiles NUMBER := 0; + + TYPE t_folder IS TABLE OF VARCHAR2(200); + vFolders t_folder := t_folder( + 'ODS/TOP/TOP_ALLOTMENT/', + 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/', + 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/', + 'ODS/TOP/TOP_ANNOUNCEMENT/', + 'ODS/TOP/TOP_FULLBIDLIST_ITEM/', + 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/' + ); + + vFolderFiles NUMBER; + vFolderSize NUMBER; BEGIN - -- Get bucket URI and credential - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS'); - vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - - DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri); - - -- Count files in cloud bucket - BEGIN - FOR rec IN ( - SELECT object_name + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + DBMS_OUTPUT.PUT_LINE('Bucket URI: ' || vDataBucketUri); + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE(RPAD('Folder', 55) || RPAD('Files', 8) || 'Total KB'); + DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-')); + + FOR i IN 1..vFolders.COUNT LOOP + vFolderFiles := 0; + vFolderSize := 0; + BEGIN + SELECT COUNT(*), NVL(SUM(bytes), 0) + INTO vFolderFiles, vFolderSize FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => vCredentialName, - location_uri => vDataBucketUri + location_uri => vDataBucketUri || vFolders(i) )) - WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%' - ) LOOP - vCloudFileCount := vCloudFileCount + 1; - IF vCloudFileCount <= 5 THEN -- Show first 5 files - DBMS_OUTPUT.PUT_LINE('- ' || rec.object_name); - END IF; - END LOOP; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount); - - IF vCloudFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket'); - END IF; - - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM); - END; + WHERE object_name NOT LIKE '%/'; + EXCEPTION WHEN OTHERS THEN NULL; + END; + + DBMS_OUTPUT.PUT_LINE( + RPAD(vFolders(i), 55) || + RPAD(TO_CHAR(vFolderFiles), 8) || + ROUND(vFolderSize / 1024, 2) || ' KB' + ); + vTotalFiles := vTotalFiles + vFolderFiles; + END LOOP; + + DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-')); + DBMS_OUTPUT.PUT_LINE('Total files across all TOP folders: ' || vTotalFiles); + + IF vTotalFiles = 0 THEN + DBMS_OUTPUT.PUT_LINE('WARNING: No files found in any TOP folder'); + ELSE + DBMS_OUTPUT.PUT_LINE('SUCCESS: Files present in ODS/TOP/ bucket paths'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM); END; / -PROMPT +PROMPT PROMPT ========================================================================= PROMPT MARS-1005 Export Verification Summary PROMPT ========================================================================= --- Final verification summary DECLARE - vFileRegCount NUMBER := 0; - vCloudFileCount NUMBER := 0; + vFileRegCount NUMBER := 0; vLogErrorCount NUMBER := 0; vOverallStatus VARCHAR2(20); BEGIN - -- Count registered files SELECT COUNT(*) INTO vFileRegCount - FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE RECEPTION_DATE >= SYSDATE - 1/24 - AND SOURCE_FILE_NAME LIKE '200%'; - - -- Count process errors + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-1005' + AND RECEPTION_DATE >= SYSDATE - 1/24; + SELECT COUNT(*) INTO vLogErrorCount FROM CT_MRDS.A_PROCESS_LOG WHERE PROCESS_NAME = 'MARS-1005' AND LOG_LEVEL = 'ERROR' AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR; - - -- Determine overall status - IF vFileRegCount >= 9 AND vLogErrorCount = 0 THEN + + IF vFileRegCount >= 6 AND vLogErrorCount = 0 THEN vOverallStatus := 'SUCCESS'; ELSIF vFileRegCount > 0 AND vLogErrorCount = 0 THEN vOverallStatus := 'PARTIAL SUCCESS'; ELSE vOverallStatus := 'ISSUES DETECTED'; END IF; - + DBMS_OUTPUT.PUT_LINE('MARS-1005 Export Verification: ' || vOverallStatus); - DBMS_OUTPUT.PUT_LINE('- Registered files: ' || vFileRegCount || ' (expected: 9)'); + DBMS_OUTPUT.PUT_LINE('- Registered files (last hour): ' || vFileRegCount || ' (expected: >= 6, one per table)'); DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount); - + IF vOverallStatus = 'SUCCESS' THEN DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful'); + ELSIF vOverallStatus = 'PARTIAL SUCCESS' THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Some tables may have incomplete exports - review registrations above'); ELSE - DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs'); + DBMS_OUTPUT.PUT_LINE('ISSUES DETECTED: Review process logs and bucket contents above'); END IF; END; / diff --git a/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql b/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql index 7b6a412..cf38a36 100644 --- a/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql +++ b/MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql @@ -1,9 +1,11 @@ -- =================================================================== -- MARS-1005 Verify Data Integrity: Source vs Exported Data Validation -- =================================================================== --- Purpose: Verify data integrity between source tables and exported files +-- Purpose: Verify data integrity between 6 OU_TOP.LEGACY_* source tables +-- and corresponding ODS external tables after export -- Author: Grzegorz Michalski --- Date: 2026-02-12 +-- Date: 2026-03-06 +-- MARS Issue: MARS-1005 SET SERVEROUTPUT ON SIZE UNLIMITED SET TIMING ON @@ -12,156 +14,151 @@ PROMPT ========================================================================= PROMPT MARS-1005 Data Integrity Verification PROMPT ========================================================================= --- Check 1: Source table record counts vs expected ETL keys -PROMPT Checking source table record counts... +-- Check 1: Source table record counts +PROMPT Checking source table record counts (OU_TOP.LEGACY_* tables)... DECLARE - vAdminRows NUMBER := 0; - vContentRows NUMBER := 0; - vCriterionRows NUMBER := 0; + v1Rows NUMBER := 0; v2Rows NUMBER := 0; v3Rows NUMBER := 0; + v4Rows NUMBER := 0; v5Rows NUMBER := 0; v6Rows NUMBER := 0; vTotalRows NUMBER := 0; - vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; - - vTotalRows := vAdminRows + vContentRows + vCriterionRows; - + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Rows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Rows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Rows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Rows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Rows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Rows; + + vTotalRows := v1Rows + v2Rows + v3Rows + v4Rows + v5Rows + v6Rows; + DBMS_OUTPUT.PUT_LINE('Source table record counts:'); - DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' records'); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT: ' || vContentRows || ' records'); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' records'); - DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records'); - + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Rows || ' records'); + DBMS_OUTPUT.PUT_LINE('- TOTAL : ' || vTotalRows || ' records'); + IF vTotalRows > 0 THEN DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data'); ELSE DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables'); END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access source tables: ' || SQLERRM); + DBMS_OUTPUT.PUT_LINE('NOTE: Ensure SELECT privilege on OU_TOP.LEGACY_* is granted to CT_MRDS'); END; / --- Check 2: ETL key distribution analysis -PROMPT Checking ETL key distribution... +-- Check 2: A_WORKFLOW_HISTORY_KEY distribution across source tables +PROMPT Checking A_WORKFLOW_HISTORY_KEY distribution... DECLARE - vAdminKeys NUMBER := 0; - vContentKeys NUMBER := 0; - vCriterionKeys NUMBER := 0; - vTotalKeys NUMBER := 0; + v1Keys NUMBER := 0; v2Keys NUMBER := 0; v3Keys NUMBER := 0; + v4Keys NUMBER := 0; v5Keys NUMBER := 0; v6Keys NUMBER := 0; + vDistinctAllKeys NUMBER := 0; BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; - - SELECT COUNT(DISTINCT etl_key) - INTO vTotalKeys + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys; + + SELECT COUNT(DISTINCT wk) + INTO vDistinctAllKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION + SELECT A_WORKFLOW_HISTORY_KEY AS wk FROM OU_TOP.LEGACY_ALLOTMENT UNION ALL + SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER UNION ALL + SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM UNION ALL + SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ANNOUNCEMENT UNION ALL + SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_FBL_ITEM UNION ALL + SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED ); - - DBMS_OUTPUT.PUT_LINE('ETL key distribution:'); - DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct keys: ' || vAdminKeys); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct keys: ' || vContentKeys); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct keys: ' || vCriterionKeys); - DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys); - - IF vTotalKeys > 0 THEN - DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal'); + + DBMS_OUTPUT.PUT_LINE('Distinct A_WORKFLOW_HISTORY_KEY values per table:'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys); + DBMS_OUTPUT.PUT_LINE('- Total distinct workflow keys (all tables): ' || vDistinctAllKeys); + + IF vDistinctAllKeys > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Workflow key distribution looks normal'); ELSE - DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data'); + DBMS_OUTPUT.PUT_LINE('ERROR: No workflow keys found in source data'); END IF; END; / -- Check 3: Template table compatibility verification -PROMPT Checking template table compatibility... +PROMPT Checking template table compatibility (CT_ET_TEMPLATES.TOP_*)... DECLARE - vAdminCols NUMBER := 0; - vContentCols NUMBER := 0; - vCriterionCols NUMBER := 0; + vCols1 NUMBER := 0; vCols2 NUMBER := 0; vCols3 NUMBER := 0; + vCols4 NUMBER := 0; vCols5 NUMBER := 0; vCols6 NUMBER := 0; BEGIN - -- Check MPEC_ADMIN template compatibility - SELECT COUNT(*) - INTO vAdminCols - FROM all_tab_columns - WHERE owner = 'CT_ET_TEMPLATES' - AND table_name = 'C2D_MPEC_ADMIN'; - - -- Check MPEC_CONTENT template compatibility - SELECT COUNT(*) - INTO vContentCols - FROM all_tab_columns - WHERE owner = 'CT_ET_TEMPLATES' - AND table_name = 'C2D_MPEC_CONTENT'; - - -- Check MPEC_CONTENT_CRITERION template compatibility - SELECT COUNT(*) - INTO vCriterionCols - FROM all_tab_columns - WHERE owner = 'CT_ET_TEMPLATES' - AND table_name = 'C2D_MPEC_CONTENT_CRITERION'; + SELECT COUNT(*) INTO vCols1 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT'; + SELECT COUNT(*) INTO vCols2 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_HEADER'; + SELECT COUNT(*) INTO vCols3 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_ITEM'; + SELECT COUNT(*) INTO vCols4 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ANNOUNCEMENT'; + SELECT COUNT(*) INTO vCols5 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBIDLIST_ITEM'; + SELECT COUNT(*) INTO vCols6 + FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBID_ARRAY_COMPILED'; DBMS_OUTPUT.PUT_LINE('Template table column counts:'); - DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_ADMIN: ' || vAdminCols || ' columns'); - DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT: ' || vContentCols || ' columns'); - DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT : ' || vCols1 || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER: ' || vCols2 || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM : ' || vCols3 || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ANNOUNCEMENT : ' || vCols4 || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM : ' || vCols5 || ' columns'); + DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED : ' || vCols6 || ' columns'); - IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN - DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure'); + IF vCols1 > 0 AND vCols2 > 0 AND vCols3 > 0 AND vCols4 > 0 AND vCols5 > 0 AND vCols6 > 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 template tables have defined structure'); ELSE DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns'); END IF; END; / --- Check 4: Verify A_ETL_LOAD_SET_FK values exist in A_LOAD_HISTORY -PROMPT Checking ETL key references in A_LOAD_HISTORY... +-- Check 4: Verify A_SOURCE_FILE_CONFIG entries for 6 TOP tables +PROMPT Checking A_SOURCE_FILE_CONFIG registration for TOP tables... DECLARE - vValidKeys NUMBER := 0; - vTotalSourceKeys NUMBER := 0; + vConfigCount NUMBER := 0; BEGIN - -- Count total distinct ETL keys in source tables - SELECT COUNT(DISTINCT etl_key) - INTO vTotalSourceKeys - FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION - ); + SELECT COUNT(*) + INTO vConfigCount + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697); -- MARS-1005 config keys - -- Count how many exist in A_LOAD_HISTORY - SELECT COUNT(DISTINCT etl_key) - INTO vValidKeys - FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT - UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION - ) src - WHERE EXISTS ( - SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h - WHERE h.A_ETL_LOAD_SET_KEY = src.etl_key - ); + DBMS_OUTPUT.PUT_LINE('A_SOURCE_FILE_CONFIG entries for MARS-1005 tables: ' || vConfigCount || ' (expected: 6)'); + DBMS_OUTPUT.PUT_LINE('Config keys: 705(ALLOTMENT), 683(MOD_HDR), 684(MOD_ITEM), 689(ANNOUNCEMENT), 696(FBL_ITEM), 697(FBA_COMPILED)'); - DBMS_OUTPUT.PUT_LINE('ETL key validation:'); - DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys in source: ' || vTotalSourceKeys); - DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); - - IF vValidKeys = vTotalSourceKeys THEN - DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid'); + IF vConfigCount = 6 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 source file config entries confirmed'); ELSE - DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); + DBMS_OUTPUT.PUT_LINE('ERROR: Missing config entries (' || (6 - vConfigCount) || ' missing)'); END IF; END; / +PROMPT A_SOURCE_FILE_CONFIG details for TOP tables: +SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + TEMPLATE_TABLE_NAME, + SUBSTR(SOURCE_FILE_NAME_PATTERN, 1, 40) AS FILE_PATTERN +FROM CT_MRDS.A_SOURCE_FILE_CONFIG +WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697) +ORDER BY A_SOURCE_FILE_CONFIG_KEY; + PROMPT ===================================================================================== PROMPT MARS-1005 Record Count Verification PROMPT ===================================================================================== @@ -190,11 +187,14 @@ BEGIN DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS')); DBMS_OUTPUT.PUT_LINE(''); - -- Initialize table list with C2D MPEC configuration + -- Initialize table list with 6 OU_TOP LEGACY table configuration vTables := t_table_list( - t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'), - t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'), - t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)') + t_table_info('OU_TOP', 'LEGACY_ALLOTMENT', 'ODS.TOP_ALLOTMENT_ODS', 'ALLOTMENT data (A_SOURCE_FILE_CONFIG_KEY=705)'), + t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_HEADER', 'ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', 'MOD HEADER data (A_SOURCE_FILE_CONFIG_KEY=683)'), + t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_ITEM', 'ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', 'MOD ITEM data (A_SOURCE_FILE_CONFIG_KEY=684)'), + t_table_info('OU_TOP', 'LEGACY_ANNOUNCEMENT', 'ODS.TOP_ANNOUNCEMENT_ODS', 'ANNOUNCEMENT data (A_SOURCE_FILE_CONFIG_KEY=689)'), + t_table_info('OU_TOP', 'LEGACY_FBL_ITEM', 'ODS.TOP_FULLBIDLIST_ITEM_ODS', 'FBL ITEM data (A_SOURCE_FILE_CONFIG_KEY=696)'), + t_table_info('OU_TOP', 'LEGACY_FULLBID_ARRAY_COMPILED', 'ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', 'FBA COMPILED data (A_SOURCE_FILE_CONFIG_KEY=697)') ); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); @@ -286,19 +286,19 @@ BEGIN DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE(''); - -- Count exported files for additional verification + -- Count MARS-1005 registered export files SELECT COUNT(*) INTO vFileCount FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE RECEPTION_DATE >= SYSDATE - 1/24 - AND (SOURCE_FILE_NAME LIKE '200_%'); + WHERE PROCESS_NAME = 'MARS-1005' + AND RECEPTION_DATE >= SYSDATE - 1/24; DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999')); DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)'); - DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('Export files registered (PROCESS_NAME=MARS-1005): ' || vFileCount); DBMS_OUTPUT.PUT_LINE(''); IF vMismatchCount = 0 AND vFileCount > 0 THEN @@ -325,26 +325,31 @@ BEGIN DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - -- Additional ETL key analysis for C2D MPEC data + -- Workflow Key Analysis DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('ETL Key Analysis:'); - + DBMS_OUTPUT.PUT_LINE('Workflow Key Analysis (distinct A_WORKFLOW_HISTORY_KEY per table):'); + DECLARE - vAdminKeys NUMBER; - vContentKeys NUMBER; - vCriterionKeys NUMBER; + v1Keys NUMBER; v2Keys NUMBER; v3Keys NUMBER; + v4Keys NUMBER; v5Keys NUMBER; v6Keys NUMBER; BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; - - DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)'); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)'); - DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct ETL keys: ' || vCriterionKeys || ' (expected: 2 for keys 2009-2010)'); - - -- Expected file count = sum of distinct ETL keys per table - DBMS_OUTPUT.PUT_LINE('- Expected export files: ' || (vAdminKeys + vContentKeys + vCriterionKeys)); - DBMS_OUTPUT.PUT_LINE('- Actual export files: ' || vFileCount); + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys; + + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys || ' distinct keys'); + DBMS_OUTPUT.PUT_LINE('- Actual export files registered: ' || vFileCount); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot query workflow keys: ' || SQLERRM); END; END; / diff --git a/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql b/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql index a243355..544243f 100644 --- a/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql +++ b/MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql @@ -1,45 +1,47 @@ --============================================================================================================================= -- MARS-1005 ROLLBACK: Delete Exported CSV Files from DATA Bucket --============================================================================================================================= --- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables +-- Purpose: Delete exported CSV files from ODS/TOP/ bucket folders for 6 OU_TOP LEGACY tables -- WARNING: This will permanently delete exported data files! -- Author: Grzegorz Michalski --- Date: 2026-02-12 --- Related: MARS-1005 - C2D MPEC Data Export Rollback +-- Date: 2026-03-06 +-- Related: MARS-1005 - OU_TOP Historical Data Export Rollback --============================================================================================================================= SET SERVEROUTPUT ON SIZE UNLIMITED PROMPT ======================================================================== -PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files +PROMPT ROLLBACK: Deleting OU_TOP LEGACY exported files from DATA Bucket PROMPT ======================================================================== -PROMPT WARNING: This will delete files from: -PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/ +PROMPT WARNING: This will delete files registered with PROCESS_NAME = 'MARS-1005' +PROMPT from ODS/TOP/* paths in the DATA bucket. PROMPT ======================================================================== +-- Helper: generic delete procedure for one TOP table folder +-- Deletes objects whose SOURCE_FILE_NAME matches the folder prefix pattern + +-- ROLLBACK TABLE 1/6: LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT/ +PROMPT ROLLBACK: Deleting TOP_ALLOTMENT files... DECLARE vDataBucketUri VARCHAR2(500); - vCredentialName VARCHAR2(100); - vFileCount NUMBER := 0; + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT/'; + vFileCount NUMBER := 0; BEGIN - -- Get bucket URI and credential vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); - vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - - DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); - - -- Delete CSV files registered by MARS-1005 process + DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT files from: ' || vDataBucketUri || vFolderPath); + FOR rec IN ( - SELECT SOURCE_FILE_NAME AS object_name - FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE PROCESS_NAME = 'MARS-1005' - AND SOURCE_FILE_NAME LIKE '%MPEC_ADMIN%' + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) + WHERE object_name NOT LIKE '%/' ) LOOP BEGIN DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name + credential_name => 'OCI$RESOURCE_PRINCIPAL', + object_uri => vDataBucketUri || vFolderPath || rec.object_name ); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); vFileCount := vFileCount + 1; @@ -47,120 +49,160 @@ BEGIN WHEN OTHERS THEN IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - RAISE; + ELSE RAISE; END IF; END; END LOOP; - + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT files deleted (' || vFileCount || ' file(s))'); END; / -PROMPT ======================================================================== -PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files -PROMPT ======================================================================== -PROMPT WARNING: This will delete files from: -PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/ -PROMPT ======================================================================== - +-- ROLLBACK TABLE 2/6: LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/ +PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files... DECLARE vDataBucketUri VARCHAR2(500); - vCredentialName VARCHAR2(100); - vFileCount NUMBER := 0; + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/'; + vFileCount NUMBER := 0; BEGIN - -- Get bucket URI and credential vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); - vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - - DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); - - -- Delete CSV files registered by MARS-1005 process + DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files...'); FOR rec IN ( - SELECT SOURCE_FILE_NAME AS object_name - FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE PROCESS_NAME = 'MARS-1005' - AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT%' - AND SOURCE_FILE_NAME NOT LIKE '%CRITERION%' + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) WHERE object_name NOT LIKE '%/' ) LOOP BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name - ); + DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - RAISE; - END IF; + EXCEPTION WHEN OTHERS THEN + IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE RAISE; END IF; END; END LOOP; - - IF vFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); - END IF; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))'); + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_HEADER files deleted (' || vFileCount || ' file(s))'); END; / -PROMPT ======================================================================== -PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files -PROMPT ======================================================================== -PROMPT WARNING: This will delete files from: -PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/ -PROMPT ======================================================================== - +-- ROLLBACK TABLE 3/6: LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/ +PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files... DECLARE vDataBucketUri VARCHAR2(500); - vCredentialName VARCHAR2(100); - vFileCount NUMBER := 0; + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/'; + vFileCount NUMBER := 0; BEGIN - -- Get bucket URI and credential vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); - vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - - DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005'''); - - -- Delete CSV files registered by MARS-1005 process + DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files...'); FOR rec IN ( - SELECT SOURCE_FILE_NAME AS object_name - FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE PROCESS_NAME = 'MARS-1005' - AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT_CRITERION%' + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) WHERE object_name NOT LIKE '%/' ) LOOP BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name - ); + DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - RAISE; - END IF; + EXCEPTION WHEN OTHERS THEN + IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE RAISE; END IF; END; END LOOP; - - IF vFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); - END IF; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))'); + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_ITEM files deleted (' || vFileCount || ' file(s))'); +END; +/ + +-- ROLLBACK TABLE 4/6: LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT/ +PROMPT ROLLBACK: Deleting TOP_ANNOUNCEMENT files... +DECLARE + vDataBucketUri VARCHAR2(500); + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ANNOUNCEMENT/'; + vFileCount NUMBER := 0; +BEGIN + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + DBMS_OUTPUT.PUT_LINE('Deleting TOP_ANNOUNCEMENT files...'); + FOR rec IN ( + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) WHERE object_name NOT LIKE '%/' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION WHEN OTHERS THEN + IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE RAISE; END IF; + END; + END LOOP; + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ANNOUNCEMENT files deleted (' || vFileCount || ' file(s))'); +END; +/ + +-- ROLLBACK TABLE 5/6: LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM/ +PROMPT ROLLBACK: Deleting TOP_FULLBIDLIST_ITEM files... +DECLARE + vDataBucketUri VARCHAR2(500); + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBIDLIST_ITEM/'; + vFileCount NUMBER := 0; +BEGIN + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBIDLIST_ITEM files...'); + FOR rec IN ( + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) WHERE object_name NOT LIKE '%/' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION WHEN OTHERS THEN + IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE RAISE; END IF; + END; + END LOOP; + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBIDLIST_ITEM files deleted (' || vFileCount || ' file(s))'); +END; +/ + +-- ROLLBACK TABLE 6/6: LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/ +PROMPT ROLLBACK: Deleting TOP_FULLBID_ARRAY_COMPILED files... +DECLARE + vDataBucketUri VARCHAR2(500); + vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/'; + vFileCount NUMBER := 0; +BEGIN + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBID_ARRAY_COMPILED files...'); + FOR rec IN ( + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vDataBucketUri || vFolderPath + )) WHERE object_name NOT LIKE '%/' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION WHEN OTHERS THEN + IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE RAISE; END IF; + END; + END LOOP; + IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF; + DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBID_ARRAY_COMPILED files deleted (' || vFileCount || ' file(s))'); END; / diff --git a/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql b/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql index 30124bf..0c4918e 100644 --- a/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql +++ b/MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql @@ -20,7 +20,7 @@ BEGIN SELECT COUNT(*) INTO vRemainingFiles FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE SOURCE_FILE_NAME LIKE '200%' -- ETL keys 2001-2010 + WHERE PROCESS_NAME = 'MARS-1005' AND RECEPTION_DATE >= SYSDATE - 7; -- Last week IF vRemainingFiles = 0 THEN @@ -33,7 +33,7 @@ BEGIN SELECT SUBSTR(SOURCE_FILE_NAME, 1, 50) AS FILE_NAME, TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE SOURCE_FILE_NAME LIKE '200%' + WHERE PROCESS_NAME = 'MARS-1005' AND RECEPTION_DATE >= SYSDATE - 7 ) LOOP DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.FILE_NAME); @@ -96,7 +96,7 @@ BEGIN credential_name => vCredentialName, location_uri => vDataBucketUri )) - WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%' + WHERE object_name LIKE 'ODS/TOP/%' ) LOOP vCloudFileCount := vCloudFileCount + 1; IF vCloudFileCount <= 3 THEN -- Show first 3 files @@ -105,11 +105,11 @@ BEGIN END LOOP; IF vCloudFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No TOP files found in cloud bucket - clean'); ELSE - DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket'); + DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' TOP file(s) still in cloud bucket'); DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback'); - DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed'); + DBMS_OUTPUT.PUT_LINE(' Run 90_MARS_1005_rollback_delete_csv_files.sql to remove them'); END IF; EXCEPTION @@ -167,7 +167,7 @@ BEGIN SELECT COUNT(*) INTO vRemainingFiles FROM CT_MRDS.A_SOURCE_FILE_RECEIVED - WHERE SOURCE_FILE_NAME LIKE '200%' + WHERE PROCESS_NAME = 'MARS-1005' AND RECEPTION_DATE >= SYSDATE - 7; -- Count remaining process logs diff --git a/MARS_Packages/REL03/MARS-1005/install_mars1005.sql b/MARS_Packages/REL03/MARS-1005/install_mars1005.sql index 742fd6d..bf6dc4d 100644 --- a/MARS_Packages/REL03/MARS-1005/install_mars1005.sql +++ b/MARS_Packages/REL03/MARS-1005/install_mars1005.sql @@ -1,11 +1,11 @@ -- =================================================================== --- MARS-1005 INSTALL SCRIPT: C2D MPEC Data Export to External Tables +-- MARS-1005 INSTALL SCRIPT: OU_TOP Historical Data Export to ODS Bucket -- =================================================================== --- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema --- to OCI buckets (ODS bucket CSV format) --- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration +-- Purpose: One-time bulk export of 6 OU_TOP LEGACY tables to OCI DATA bucket +-- (ODS bucket area, CSV format) +-- Uses DATA_EXPORTER EXPORT_TABLE_DATA with pRegisterExport for file tracking -- Author: Grzegorz Michalski --- Date: 2026-02-12 +-- Date: 2026-03-06 -- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) -- Log files are automatically created in log/ subdirectory @@ -30,20 +30,23 @@ SET PAUSE OFF -- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS; PROMPT ========================================================================= -PROMPT MARS-1005: C2D MPEC Data Export to External Tables (One-Time Migration) +PROMPT MARS-1005: OU_TOP Historical Data Export to ODS Bucket (One-Time Migration) PROMPT ========================================================================= PROMPT -PROMPT This script will export 3 C2D MPEC tables to OCI buckets: +PROMPT This script will export 6 OU_TOP LEGACY tables to OCI DATA bucket: PROMPT -PROMPT TARGET: ODS Bucket (CSV format): -PROMPT - MPEC_ADMIN -PROMPT - MPEC_CONTENT -PROMPT - MPEC_CONTENT_CRITERION +PROMPT TARGET: DATA Bucket / ODS area (CSV format): +PROMPT - OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT +PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER +PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM +PROMPT - OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT +PROMPT - OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM +PROMPT - OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED PROMPT PROMPT Key Features: -PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking -PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*) -PROMPT - ODS/C2D bucket path structure +PROMPT - Files registered in A_SOURCE_FILE_RECEIVED with PROCESS_NAME = 'MARS-1005' +PROMPT - Template table column order matching (CT_ET_TEMPLATES.TOP_*) +PROMPT - ODS/TOP bucket path structure PROMPT ========================================================================= -- Confirm installation with user diff --git a/README.md b/README.md index da8b5cb..b1a188a 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,8 @@ cd .\MARS_Packages\REL02_POST\MARS-1409 echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1409.sql" echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1409.sql" 7z a -pMojeSuperHaslo#123 -mhe=on M1409_arch.7z MARS-1409 + +cd .\MARS_Packages\REL03\MARS-1005 +echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1005.sql" +echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1005.sql" +7z a -pMojeSuperHaslo#123 -mhe=on M1005_arch.7z MARS-1005 \ No newline at end of file