diff --git a/MARS_Packages/REL02_POST/MARS-956/00_MARS_956_pre_check_prerequisites.sql b/MARS_Packages/REL02_POST/MARS-956/00_MARS_956_pre_check_prerequisites.sql index d393a82..81b1bdd 100644 --- a/MARS_Packages/REL02_POST/MARS-956/00_MARS_956_pre_check_prerequisites.sql +++ b/MARS_Packages/REL02_POST/MARS-956/00_MARS_956_pre_check_prerequisites.sql @@ -44,22 +44,22 @@ BEGIN BEGIN DBMS_OUTPUT.PUT_LINE('DATA_EXPORTER Version: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_VERSION); DBMS_OUTPUT.PUT_LINE('Build Date: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_BUILD_DATE); - DBMS_OUTPUT.PUT_LINE('✓ DATA_EXPORTER package is available'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: DATA_EXPORTER package is available'); EXCEPTION WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('✗ ERROR: DATA_EXPORTER package not available: ' || SQLERRM); + DBMS_OUTPUT.PUT_LINE('ERROR: ERROR: DATA_EXPORTER package not available: ' || SQLERRM); RAISE; END; DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('Check 2: Verify Source Tables in OU_C2D Schema'); + DBMS_OUTPUT.PUT_LINE('Check 2: Verify Source Tables in OU_LEGACY_C2D Schema'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); -- Check source table row counts - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; DBMS_OUTPUT.PUT_LINE('Source table row counts:'); DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' rows'); @@ -67,9 +67,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' rows'); IF vAdminRows > 0 AND vContentRows > 0 AND vCriterionRows > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All source tables have data'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables have data'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ One or more source tables are empty'); + DBMS_OUTPUT.PUT_LINE('ERROR: One or more source tables are empty'); END IF; DBMS_OUTPUT.PUT_LINE(''); @@ -102,9 +102,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All template tables have defined structure'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ One or more template tables missing columns'); + DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns'); END IF; DBMS_OUTPUT.PUT_LINE(''); @@ -121,22 +121,22 @@ BEGIN SELECT COUNT(DISTINCT etl_key) INTO vTotalSourceKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION ); -- Count how many exist in A_LOAD_HISTORY SELECT COUNT(DISTINCT etl_key) INTO vValidKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION ) src WHERE EXISTS ( SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h @@ -148,9 +148,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); IF vValidKeys = vTotalSourceKeys THEN - DBMS_OUTPUT.PUT_LINE('✓ All source ETL keys are valid'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); + DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); END IF; END; @@ -219,14 +219,14 @@ BEGIN IF vAdminExtCount >= 0 AND vContentExtCount >= 0 AND vCriterionExtCount >= 0 THEN IF vAdminExtCount = 0 AND vContentExtCount = 0 AND vCriterionExtCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All target external tables are clean (ready for fresh export)'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All target external tables are clean (ready for fresh export)'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Target external tables contain data (' || + DBMS_OUTPUT.PUT_LINE('WARNING: Target external tables contain data (' || (vAdminExtCount + vContentExtCount + vCriterionExtCount) || ' total records)'); DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run'); END IF; ELSE - DBMS_OUTPUT.PUT_LINE('✗ Some external tables are inaccessible - check table definitions'); + DBMS_OUTPUT.PUT_LINE('ERROR: Some external tables are inaccessible - check table definitions'); END IF; END; @@ -253,30 +253,30 @@ BEGIN END LOOP; IF vFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ No existing C2D MPEC files found - ready for clean export'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No existing C2D MPEC files found - ready for clean export'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vFileCount || ' existing C2D MPEC files found'); + DBMS_OUTPUT.PUT_LINE('WARNING: Warning: ' || vFileCount || ' existing C2D MPEC files found'); DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run'); END IF; EXCEPTION WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('✗ ERROR checking existing files: ' || SQLERRM); + DBMS_OUTPUT.PUT_LINE('ERROR: ERROR checking existing files: ' || SQLERRM); END; DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('MARS-956 Pre-Check Summary'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('✓ DATA_EXPORTER package available'); - DBMS_OUTPUT.PUT_LINE('✓ Source tables: ' || (vAdminRows + vContentRows + vCriterionRows) || ' total rows'); - DBMS_OUTPUT.PUT_LINE('✓ Template tables: All structures defined'); - DBMS_OUTPUT.PUT_LINE('✓ ETL keys: All validated in A_LOAD_HISTORY'); - DBMS_OUTPUT.PUT_LINE('✓ External tables: Accessible and ready'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: DATA_EXPORTER package available'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables: ' || (vAdminRows + vContentRows + vCriterionRows) || ' total rows'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Template tables: All structures defined'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL keys: All validated in A_LOAD_HISTORY'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: External tables: Accessible and ready'); IF vFileCount > 0 THEN - DBMS_OUTPUT.PUT_LINE('⚠ Existing files: ' || vFileCount || ' (consider rollback)'); + DBMS_OUTPUT.PUT_LINE('WARNING: Existing files: ' || vFileCount || ' (consider rollback)'); ELSE - DBMS_OUTPUT.PUT_LINE('✓ Target bucket: Clean for export'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Target bucket: Clean for export'); END IF; DBMS_OUTPUT.PUT_LINE(''); diff --git a/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql b/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql index a24af63..a3b4c9d 100644 --- a/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql +++ b/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql @@ -14,7 +14,7 @@ PROMPT ========================================================================= PROMPT MARS-956: C2D MPEC Historical Data Export PROMPT ===================================================================================== PROMPT Export Strategy: -PROMPT - Source: OU_C2D schema tables (operational database) +PROMPT - Source: OU_LEGACY_C2D schema tables (operational database) PROMPT - Target: ODS bucket as CSV files PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED @@ -26,6 +26,198 @@ INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_ VALUES ('MARS-956', 'EXPORT_C2D_MPEC_DATA', 'INFO', 'Starting historical C2D MPEC data export', 'Tables: MPEC_ADMIN, MPEC_CONTENT, MPEC_CONTENT_CRITERION'); +PROMPT +PROMPT ===================================================================================== +PROMPT PRE-EXPORT CHECK: Verify Existing Files in ODS Bucket +PROMPT ===================================================================================== + +-- Check 1: MPEC_ADMIN files +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + -- Get bucket URI for DATA bucket + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_ADMIN/'; + + -- Count existing files + SELECT COUNT(*) + INTO vFileCount + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/'; -- Exclude directories + + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_ADMIN files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri); + DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE(''); + + -- List existing files + DBMS_OUTPUT.PUT_LINE('Existing files:'); + FOR rec IN ( + SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/' + ORDER BY object_name + ) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')'); + END LOOP; + + -- Count records in external table + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_ADMIN_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table'); + DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); + END; + + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE(''); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_ADMIN files found - bucket is clean'); + DBMS_OUTPUT.PUT_LINE(''); + END IF; +END; +/ + +-- Check 2: MPEC_CONTENT files +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_CONTENT/'; + + SELECT COUNT(*) + INTO vFileCount + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/'; + + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_CONTENT files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri); + DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE(''); + + DBMS_OUTPUT.PUT_LINE('Existing files:'); + FOR rec IN ( + SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/' + ORDER BY object_name + ) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')'); + END LOOP; + + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_CONTENT_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table'); + DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); + END; + + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE(''); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_CONTENT files found - bucket is clean'); + DBMS_OUTPUT.PUT_LINE(''); + END IF; +END; +/ + +-- Check 3: MPEC_CONTENT_CRITERION files +DECLARE + vFileCount NUMBER := 0; + vRecordCount NUMBER := 0; + vLocationUri VARCHAR2(1000); +BEGIN + vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/'; + + SELECT COUNT(*) + INTO vFileCount + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/'; + + IF vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_CONTENT_CRITERION files already exist in DATA bucket'); + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri); + DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE(''); + + DBMS_OUTPUT.PUT_LINE('Existing files:'); + FOR rec IN ( + SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified + FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vLocationUri + )) + WHERE object_name NOT LIKE '%/' + ORDER BY object_name + ) LOOP + DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')'); + END LOOP; + + BEGIN + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_CONTENT_CRITERION_ODS' INTO vRecordCount; + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount); + DBMS_OUTPUT.PUT_LINE('>>>'); + DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table'); + DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); + END; + + DBMS_OUTPUT.PUT_LINE('==============================================================================='); + DBMS_OUTPUT.PUT_LINE(''); + ELSE + DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_CONTENT_CRITERION files found - bucket is clean'); + DBMS_OUTPUT.PUT_LINE(''); + END IF; +END; +/ + PROMPT PROMPT ===================================================================================== PROMPT PRE-EXPORT: Verify Source and Target Table Readiness @@ -43,9 +235,9 @@ DECLARE vTotalTarget NUMBER := 0; BEGIN -- Source table counts - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; vTotalSource := vAdminRows + vContentRows + vCriterionRows; DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):'); @@ -106,17 +298,17 @@ BEGIN CASE WHEN vTotalTarget = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(vTotalTarget) END); IF vTotalSource > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ Source tables contain data - ready for export'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ WARNING: No source data found'); + DBMS_OUTPUT.PUT_LINE('ERROR: WARNING: No source data found'); END IF; IF vTotalTarget = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ Target external tables are clean - ready for fresh export'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export'); ELSIF vTotalTarget > 0 THEN - DBMS_OUTPUT.PUT_LINE('⚠ WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); + DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ ERROR: Cannot access target external tables'); + DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access target external tables'); END IF; DBMS_OUTPUT.PUT_LINE(''); @@ -126,14 +318,12 @@ END; PROMPT PROMPT ===================================================================================== -PROMPT TABLE 1/3: OU_C2D.MPEC_ADMIN -> ODS/C2D/C2D_MPEC_ADMIN +PROMPT TABLE 1/3: OU_LEGACY_C2D.MPEC_ADMIN -> ODS/C2D/C2D_MPEC_ADMIN PROMPT ===================================================================================== -PROMPT Expected: 5 records with ETL keys 2001-2005 -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/*.csv BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( - pSchemaName => 'OU_C2D', + pSchemaName => 'OU_LEGACY_C2D', pTableName => 'MPEC_ADMIN', pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', @@ -143,13 +333,13 @@ BEGIN pRegisterExport => TRUE -- Register files in A_SOURCE_FILE_RECEIVED ); - DBMS_OUTPUT.PUT_LINE('✓ MPEC_ADMIN export completed successfully'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_ADMIN export completed successfully'); EXCEPTION WHEN OTHERS THEN DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN - DBMS_OUTPUT.PUT_LINE('✗ MPEC_ADMIN export failed: ' || vErrorMsg); + DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_ADMIN export failed: ' || vErrorMsg); -- Log error using proper ENV_MANAGER pattern INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) @@ -164,14 +354,12 @@ END; PROMPT PROMPT ===================================================================================== -PROMPT TABLE 2/3: OU_C2D.MPEC_CONTENT -> ODS/C2D/C2D_MPEC_CONTENT +PROMPT TABLE 2/3: OU_LEGACY_C2D.MPEC_CONTENT -> ODS/C2D/C2D_MPEC_CONTENT PROMPT ===================================================================================== -PROMPT Expected: 5 records with ETL keys 2006-2008 -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( - pSchemaName => 'OU_C2D', + pSchemaName => 'OU_LEGACY_C2D', pTableName => 'MPEC_CONTENT', pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ODS', @@ -181,13 +369,13 @@ BEGIN pRegisterExport => TRUE ); - DBMS_OUTPUT.PUT_LINE('✓ MPEC_CONTENT export completed successfully'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_CONTENT export completed successfully'); EXCEPTION WHEN OTHERS THEN DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN - DBMS_OUTPUT.PUT_LINE('✗ MPEC_CONTENT export failed: ' || vErrorMsg); + DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_CONTENT export failed: ' || vErrorMsg); -- Log error using proper ENV_MANAGER pattern INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) @@ -202,14 +390,12 @@ END; PROMPT PROMPT ===================================================================================== -PROMPT TABLE 3/3: OU_C2D.MPEC_CONTENT_CRITERION -> ODS/C2D/C2D_MPEC_CONTENT_CRITERION +PROMPT TABLE 3/3: OU_LEGACY_C2D.MPEC_CONTENT_CRITERION -> ODS/C2D/C2D_MPEC_CONTENT_CRITERION PROMPT ===================================================================================== -PROMPT Expected: 9 records with ETL keys 2009-2010 -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( - pSchemaName => 'OU_C2D', + pSchemaName => 'OU_LEGACY_C2D', pTableName => 'MPEC_CONTENT_CRITERION', pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ODS', @@ -219,13 +405,13 @@ BEGIN pRegisterExport => TRUE ); - DBMS_OUTPUT.PUT_LINE('✓ MPEC_CONTENT_CRITERION export completed successfully'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_CONTENT_CRITERION export completed successfully'); EXCEPTION WHEN OTHERS THEN DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN - DBMS_OUTPUT.PUT_LINE('✗ MPEC_CONTENT_CRITERION export failed: ' || vErrorMsg); + DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_CONTENT_CRITERION export failed: ' || vErrorMsg); -- Log error using proper ENV_MANAGER pattern INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) @@ -281,9 +467,9 @@ DECLARE vMismatchCount NUMBER := 0; BEGIN -- Source table counts - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminSource; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentSource; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionSource; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminSource; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentSource; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionSource; vTotalSource := vAdminSource + vContentSource + vCriterionSource; -- Target external table counts @@ -302,40 +488,40 @@ BEGIN DBMS_OUTPUT.PUT_LINE('MPEC_ADMIN | ' || RPAD(vAdminSource, 8) || ' | ' || RPAD(vAdminTarget, 8) || ' | ' || - CASE WHEN vAdminSource = vAdminTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); + CASE WHEN vAdminSource = vAdminTarget THEN 'OK' ELSE 'MISMATCH' END); IF vAdminSource != vAdminTarget THEN vMismatchCount := vMismatchCount + 1; END IF; -- MPEC_CONTENT comparison DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT | ' || RPAD(vContentSource, 8) || ' | ' || RPAD(vContentTarget, 8) || ' | ' || - CASE WHEN vContentSource = vContentTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); + CASE WHEN vContentSource = vContentTarget THEN 'OK' ELSE 'MISMATCH' END); IF vContentSource != vContentTarget THEN vMismatchCount := vMismatchCount + 1; END IF; -- MPEC_CONTENT_CRITERION comparison DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT_CRITERION | ' || RPAD(vCriterionSource, 8) || ' | ' || RPAD(vCriterionTarget, 8) || ' | ' || - CASE WHEN vCriterionSource = vCriterionTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); + CASE WHEN vCriterionSource = vCriterionTarget THEN 'OK' ELSE 'MISMATCH' END); IF vCriterionSource != vCriterionTarget THEN vMismatchCount := vMismatchCount + 1; END IF; DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE('TOTAL | ' || RPAD(vTotalSource, 8) || ' | ' || RPAD(vTotalTarget, 8) || ' | ' || - CASE WHEN vTotalSource = vTotalTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); + CASE WHEN vTotalSource = vTotalTarget THEN 'OK' ELSE 'MISMATCH' END); DBMS_OUTPUT.PUT_LINE(''); IF vMismatchCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ SUCCESS: All record counts match - export verified'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All record counts match - export verified'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ WARNING: ' || vMismatchCount || ' table(s) have record count mismatches'); + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vMismatchCount || ' table(s) have record count mismatches'); DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions'); END IF; EXCEPTION WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('✗ ERROR: Cannot verify target external tables post-export'); + DBMS_OUTPUT.PUT_LINE('ERROR: Cannot verify target external tables post-export'); DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('Please check external table configuration and ODS bucket access'); END; diff --git a/MARS_Packages/REL02_POST/MARS-956/02_MARS_956_verify_exports.sql b/MARS_Packages/REL02_POST/MARS-956/02_MARS_956_verify_exports.sql index 09cff7d..db8c607 100644 --- a/MARS_Packages/REL02_POST/MARS-956/02_MARS_956_verify_exports.sql +++ b/MARS_Packages/REL02_POST/MARS-956/02_MARS_956_verify_exports.sql @@ -33,15 +33,15 @@ BEGIN OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys OR SOURCE_FILE_NAME LIKE '2010_%'); - DBMS_OUTPUT.PUT_LINE('✓ Registered export files: ' || vFileCount); - DBMS_OUTPUT.PUT_LINE('✓ Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB'); IF vFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: No export files found in registration'); + DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration'); ELSIF vFileCount < 9 THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount); + DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount); ELSE - DBMS_OUTPUT.PUT_LINE('✓ All expected export files found'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found'); END IF; END; / @@ -71,13 +71,13 @@ BEGIN WHERE PROCESS_NAME = 'MARS-956' AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR; - DBMS_OUTPUT.PUT_LINE('✓ Process log entries: ' || vLogCount); - DBMS_OUTPUT.PUT_LINE('✓ Error entries: ' || vErrorCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Process log entries: ' || vLogCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Error entries: ' || vErrorCount); IF vErrorCount > 0 THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vErrorCount || ' errors found in process log'); + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vErrorCount || ' errors found in process log'); ELSE - DBMS_OUTPUT.PUT_LINE('✓ No errors found in process log'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No errors found in process log'); END IF; END; / @@ -124,15 +124,15 @@ BEGIN END IF; END LOOP; - DBMS_OUTPUT.PUT_LINE('✓ Cloud bucket files found: ' || vCloudFileCount); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount); IF vCloudFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: No files found in cloud bucket'); + DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket'); END IF; EXCEPTION WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: Cannot access cloud bucket: ' || SQLERRM); + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM); END; END; / @@ -178,9 +178,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount); IF vOverallStatus = 'SUCCESS' THEN - DBMS_OUTPUT.PUT_LINE('✓ All validations passed - export successful'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Some issues detected - review logs'); + DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs'); END IF; END; / diff --git a/MARS_Packages/REL02_POST/MARS-956/03_MARS_956_verify_data_integrity.sql b/MARS_Packages/REL02_POST/MARS-956/03_MARS_956_verify_data_integrity.sql index 8507cad..0c7af39 100644 --- a/MARS_Packages/REL02_POST/MARS-956/03_MARS_956_verify_data_integrity.sql +++ b/MARS_Packages/REL02_POST/MARS-956/03_MARS_956_verify_data_integrity.sql @@ -21,9 +21,9 @@ DECLARE vTotalRows NUMBER := 0; vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows; + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; vTotalRows := vAdminRows + vContentRows + vCriterionRows; @@ -34,9 +34,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records'); IF vTotalRows > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All source tables contain data'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ No data found in source tables'); + DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables'); END IF; END; / @@ -49,18 +49,18 @@ DECLARE vCriterionKeys NUMBER := 0; vTotalKeys NUMBER := 0; BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_ADMIN' INTO vAdminKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT' INTO vContentKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; SELECT COUNT(DISTINCT etl_key) INTO vTotalKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION ); DBMS_OUTPUT.PUT_LINE('ETL key distribution:'); @@ -70,9 +70,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys); IF vTotalKeys > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ ETL key distribution looks normal'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ No ETL keys found in source data'); + DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data'); END IF; END; / @@ -111,9 +111,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All template tables have defined structure'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ One or more template tables missing columns'); + DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns'); END IF; END; / @@ -128,22 +128,22 @@ BEGIN SELECT COUNT(DISTINCT etl_key) INTO vTotalSourceKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION ); -- Count how many exist in A_LOAD_HISTORY SELECT COUNT(DISTINCT etl_key) INTO vValidKeys FROM ( - SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN + SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT UNION - SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION + SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION ) src WHERE EXISTS ( SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h @@ -155,16 +155,18 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); IF vValidKeys = vTotalSourceKeys THEN - DBMS_OUTPUT.PUT_LINE('✓ All source ETL keys are valid'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid'); ELSE - DBMS_OUTPUT.PUT_LINE('✗ Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); + DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); END IF; END; / -PROMPT ========================================================================= -PROMPT Data Integrity Verification Summary: Source vs Target Record Counts -PROMPT ========================================================================= +PROMPT ===================================================================================== +PROMPT MARS-956 Record Count Verification +PROMPT ===================================================================================== +PROMPT Comparing source table counts with exported external table counts +PROMPT ===================================================================================== DECLARE TYPE t_table_info IS RECORD ( @@ -190,13 +192,13 @@ BEGIN -- Initialize table list with C2D MPEC configuration vTables := t_table_list( - t_table_info('OU_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'), - t_table_info('OU_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'), - t_table_info('OU_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)') + t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'), + t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'), + t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)') ); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status Difference'); + DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); FOR i IN 1..vTables.COUNT LOOP @@ -209,7 +211,7 @@ BEGIN EXCEPTION WHEN OTHERS THEN vSourceCount := -1; - DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || 'ERROR: Cannot access source table'); + DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table'); CONTINUE; END; @@ -224,39 +226,65 @@ BEGIN -- ORA-29913: error in executing ODCIEXTTABLEOPEN callout -- ORA-29400: data cartridge error -- KUP-13023: nothing matched wildcard query (no files in bucket) - IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' OR SQLERRM LIKE '%does not exist%' THEN + -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty + IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN vTargetCount := 0; -- Treat as empty (no files exported yet) ELSE vTargetCount := -1; -- Real error END IF; END; - -- Display comparison results - IF vSourceCount = -1 OR vTargetCount = -1 THEN - DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || - RPAD(CASE WHEN vSourceCount = -1 THEN 'ERROR' ELSE TO_CHAR(vSourceCount) END, 14) || - RPAD(CASE WHEN vTargetCount = -1 THEN 'ERROR' ELSE TO_CHAR(vTargetCount) END, 14) || - 'ERROR ' || 'N/A'); - ELSIF vSourceCount = vTargetCount THEN - DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || - RPAD(TO_CHAR(vSourceCount), 14) || - RPAD(TO_CHAR(vTargetCount), 14) || - 'MATCH ' || '0'); - ELSE - vMismatchCount := vMismatchCount + 1; - DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || - RPAD(TO_CHAR(vSourceCount), 14) || - RPAD(TO_CHAR(vTargetCount), 14) || - 'MISMATCH ' || TO_CHAR(vTargetCount - vSourceCount)); - END IF; + -- Display comparison results with thousands separators + DECLARE + vStatus VARCHAR2(20); + vSourceDisplay VARCHAR2(17); + vTargetDisplay VARCHAR2(17); + BEGIN + -- Format source count display + IF vSourceCount = -1 THEN + vSourceDisplay := 'ERROR'; + ELSE + vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999'); + END IF; + + -- Format target count display + IF vTargetCount = -1 THEN + vTargetDisplay := 'ERROR'; + ELSE + vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999'); + END IF; + + -- Determine status + IF vSourceCount = vTargetCount THEN + vStatus := 'PASS'; + ELSIF vTargetCount = -1 THEN + vStatus := 'ERROR'; + vMismatchCount := vMismatchCount + 1; + ELSIF vSourceCount = -1 THEN + vStatus := 'ERROR'; + vMismatchCount := vMismatchCount + 1; + ELSE + vStatus := 'MISMATCH'; + vMismatchCount := vMismatchCount + 1; + END IF; + + DBMS_OUTPUT.PUT_LINE( + RPAD(vTables(i).source_table, 24) || + LPAD(vSourceDisplay, 15) || + LPAD(vTargetDisplay, 15) || ' ' || + vStatus + ); + END; END LOOP; DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE(RPAD('TOTALS:', 28) || - RPAD(TO_CHAR(vTotalSourceCount), 14) || - RPAD(TO_CHAR(vTotalTargetCount), 14) || - CASE WHEN vTotalSourceCount = vTotalTargetCount THEN 'MATCH' ELSE 'MISMATCH' END || ' ' || - TO_CHAR(vTotalTargetCount - vTotalSourceCount)); + DBMS_OUTPUT.PUT_LINE( + RPAD('TOTALS', 24) || + LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) || + LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15) + ); + DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE(''); -- Count exported files for additional verification SELECT COUNT(*) @@ -265,33 +293,37 @@ BEGIN WHERE RECEPTION_DATE >= SYSDATE - 1/24 AND (SOURCE_FILE_NAME LIKE '200_%'); - -- Determine overall validation result - IF vTotalSourceCount = vTotalTargetCount AND vMismatchCount = 0 AND vFileCount > 0 THEN - vValidationResult := 'SUCCESS'; - ELSIF vTotalTargetCount = 0 AND vFileCount = 0 THEN - vValidationResult := 'NO EXPORT DETECTED'; - ELSIF vMismatchCount > 0 THEN - vValidationResult := 'DATA MISMATCH'; - ELSE - vValidationResult := 'PARTIAL SUCCESS'; - END IF; - + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary'); + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999')); + DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)'); + DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount); DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('MARS-956 Record Count Verification: ' || vValidationResult); - DBMS_OUTPUT.PUT_LINE('- Source records: ' || vTotalSourceCount); - DBMS_OUTPUT.PUT_LINE('- Target records: ' || vTotalTargetCount); - DBMS_OUTPUT.PUT_LINE('- Export files: ' || vFileCount); - DBMS_OUTPUT.PUT_LINE('- Mismatched tables: ' || vMismatchCount); - IF vValidationResult = 'SUCCESS' THEN - DBMS_OUTPUT.PUT_LINE('✓ All record counts match - export successful'); - ELSIF vValidationResult = 'NO EXPORT DETECTED' THEN - DBMS_OUTPUT.PUT_LINE('⚠ No export detected - check export execution'); - ELSIF vValidationResult = 'DATA MISMATCH' THEN - DBMS_OUTPUT.PUT_LINE('✗ Record count mismatches found - investigate data integrity'); + IF vMismatchCount = 0 AND vFileCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED'); + DBMS_OUTPUT.PUT_LINE(' All record counts match between source and exported data'); + DBMS_OUTPUT.PUT_LINE(' Export completed successfully'); + ELSIF vMismatchCount > 0 THEN + DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES'); + DBMS_OUTPUT.PUT_LINE(' Found ' || vMismatchCount || ' table(s) with count mismatches'); + DBMS_OUTPUT.PUT_LINE(' NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)'); + DBMS_OUTPUT.PUT_LINE(' Review export logs and pre-check results before re-running exports'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Partial success - review individual table results'); + DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED'); + DBMS_OUTPUT.PUT_LINE(' No files found in export registration'); + DBMS_OUTPUT.PUT_LINE(' Verify export execution completed successfully'); END IF; + DBMS_OUTPUT.PUT_LINE('====================================================================================='); + DBMS_OUTPUT.PUT_LINE(''); + + DBMS_OUTPUT.PUT_LINE('Legend:'); + DBMS_OUTPUT.PUT_LINE(' PASS - Record counts match (export successful)'); + DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)'); + DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files'); + DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)'); + DBMS_OUTPUT.PUT_LINE('====================================================================================='); -- Additional ETL key analysis for C2D MPEC data DBMS_OUTPUT.PUT_LINE(''); @@ -302,9 +334,9 @@ BEGIN vContentKeys NUMBER; vCriterionKeys NUMBER; BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_ADMIN' INTO vAdminKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT' INTO vContentKeys; - EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys; + EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)'); DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)'); diff --git a/MARS_Packages/REL02_POST/MARS-956/90_MARS_956_rollback_delete_csv_files.sql b/MARS_Packages/REL02_POST/MARS-956/90_MARS_956_rollback_delete_csv_files.sql new file mode 100644 index 0000000..649dc2e --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-956/90_MARS_956_rollback_delete_csv_files.sql @@ -0,0 +1,169 @@ +--============================================================================================================================= +-- MARS-956 ROLLBACK: Delete Exported CSV Files from DATA Bucket +--============================================================================================================================= +-- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables +-- WARNING: This will permanently delete exported data files! +-- Author: System Generated +-- Date: 2026-02-12 +-- Related: MARS-956 - C2D MPEC Data Export Rollback +--============================================================================================================================= + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...'); + + -- Delete CSV files from DATA bucket (only files matching export pattern) + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' + )) + WHERE object_name LIKE '%.csv' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...'); + + -- Delete CSV files from DATA bucket (only files matching export pattern) + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' + )) + WHERE object_name LIKE '%.csv' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files +PROMPT ======================================================================== +PROMPT WARNING: This will delete files from: +PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/ +PROMPT ======================================================================== + +DECLARE + vDataBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URI and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...'); + + -- Delete CSV files from DATA bucket (only files matching export pattern) + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' + )) + WHERE object_name LIKE '%.csv' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + RAISE; + END IF; + END; + END LOOP; + + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))'); +END; +/ + +PROMPT SUCCESS: All CSV file deletion operations completed diff --git a/MARS_Packages/REL02_POST/MARS-956/91_MARS_956_rollback_file_registrations.sql b/MARS_Packages/REL02_POST/MARS-956/91_MARS_956_rollback_file_registrations.sql index 98eba67..28ad299 100644 --- a/MARS_Packages/REL02_POST/MARS-956/91_MARS_956_rollback_file_registrations.sql +++ b/MARS_Packages/REL02_POST/MARS-956/91_MARS_956_rollback_file_registrations.sql @@ -15,6 +15,7 @@ PROMPT ========================================================================= DECLARE vFileCount NUMBER := 0; vDeletedCount NUMBER := 0; + vErrorMsg VARCHAR2(4000); BEGIN -- Count files to be deleted SELECT COUNT(*) @@ -48,7 +49,7 @@ BEGIN vDeletedCount := SQL%ROWCOUNT; COMMIT; - DBMS_OUTPUT.PUT_LINE('✓ Successfully deleted ' || vDeletedCount || ' file registrations'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' file registrations'); -- Log the rollback action INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) @@ -57,17 +58,17 @@ BEGIN COMMIT; ELSE - DBMS_OUTPUT.PUT_LINE('✓ No file registrations found to delete'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No file registrations found to delete'); END IF; EXCEPTION WHEN OTHERS THEN ROLLBACK; - DBMS_OUTPUT.PUT_LINE('✗ Error during file registration deletion: ' || SQLERRM); + vErrorMsg := 'Failed to delete file registrations: ' || SQLERRM; + DBMS_OUTPUT.PUT_LINE('ERROR: Error during file registration deletion: ' || SQLERRM); -- Log the error INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) - VALUES ('MARS-956-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', - 'Failed to delete file registrations: ' || SQLERRM); + VALUES ('MARS-956-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', vErrorMsg); COMMIT; RAISE; END; diff --git a/MARS_Packages/REL02_POST/MARS-956/92_MARS_956_rollback_process_logs.sql b/MARS_Packages/REL02_POST/MARS-956/92_MARS_956_rollback_process_logs.sql index fdef3ac..9f3047e 100644 --- a/MARS_Packages/REL02_POST/MARS-956/92_MARS_956_rollback_process_logs.sql +++ b/MARS_Packages/REL02_POST/MARS-956/92_MARS_956_rollback_process_logs.sql @@ -15,6 +15,7 @@ PROMPT ========================================================================= DECLARE vLogCount NUMBER := 0; vDeletedCount NUMBER := 0; + vErrorMsg VARCHAR2(4000); BEGIN -- Count logs to be deleted SELECT COUNT(*) @@ -52,20 +53,20 @@ BEGIN vDeletedCount := SQL%ROWCOUNT; COMMIT; - DBMS_OUTPUT.PUT_LINE('✓ Successfully deleted ' || vDeletedCount || ' process log entries'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' process log entries'); ELSE - DBMS_OUTPUT.PUT_LINE('✓ No process log entries found to delete'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No process log entries found to delete'); END IF; EXCEPTION WHEN OTHERS THEN ROLLBACK; - DBMS_OUTPUT.PUT_LINE('✗ Error during process log cleanup: ' || SQLERRM); + vErrorMsg := 'Failed to clean process logs: ' || SQLERRM; + DBMS_OUTPUT.PUT_LINE('ERROR: Error during process log cleanup: ' || SQLERRM); -- Log the error (will remain after rollback for debugging) INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) - VALUES ('MARS-956-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', - 'Failed to clean process logs: ' || SQLERRM); + VALUES ('MARS-956-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', vErrorMsg); COMMIT; RAISE; END; diff --git a/MARS_Packages/REL02_POST/MARS-956/99_MARS_956_verify_rollback.sql b/MARS_Packages/REL02_POST/MARS-956/99_MARS_956_verify_rollback.sql index 159c642..07f1b35 100644 --- a/MARS_Packages/REL02_POST/MARS-956/99_MARS_956_verify_rollback.sql +++ b/MARS_Packages/REL02_POST/MARS-956/99_MARS_956_verify_rollback.sql @@ -24,9 +24,9 @@ BEGIN AND RECEPTION_DATE >= SYSDATE - 7; -- Last week IF vRemainingFiles = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All MARS-956 file registrations successfully removed'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-956 file registrations successfully removed'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vRemainingFiles || ' file registrations still exist'); + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingFiles || ' file registrations still exist'); -- Show remaining files FOR rec IN ( @@ -54,9 +54,9 @@ BEGIN AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week IF vRemainingLogs = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ All MARS-956 process logs successfully removed'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-956 process logs successfully removed'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vRemainingLogs || ' process log entries still exist'); + DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingLogs || ' process log entries still exist'); -- Show remaining logs (first few) FOR rec IN ( @@ -105,16 +105,16 @@ BEGIN END LOOP; IF vCloudFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ No C2D MPEC files found in cloud bucket'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket'); ELSE - DBMS_OUTPUT.PUT_LINE('ℹ Info: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket'); + DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket'); DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback'); DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed'); END IF; EXCEPTION WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('⚠ Warning: Cannot check cloud bucket: ' || SQLERRM); + DBMS_OUTPUT.PUT_LINE('WARNING: Cannot check cloud bucket: ' || SQLERRM); END; END; / @@ -131,7 +131,7 @@ BEGIN AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour IF vRollbackLogs > 0 THEN - DBMS_OUTPUT.PUT_LINE('✓ Rollback operation logs found: ' || vRollbackLogs); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback operation logs found: ' || vRollbackLogs); -- Show recent rollback logs FOR rec IN ( @@ -148,7 +148,7 @@ BEGIN rec.PROCEDURE_NAME || ': ' || rec.MESSAGE); END LOOP; ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Warning: No rollback operation logs found'); + DBMS_OUTPUT.PUT_LINE('WARNING: Warning: No rollback operation logs found'); END IF; END; / @@ -191,9 +191,9 @@ BEGIN DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs); IF vRollbackStatus = 'COMPLETE' THEN - DBMS_OUTPUT.PUT_LINE('✓ Rollback completed successfully - system clean'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback completed successfully - system clean'); ELSE - DBMS_OUTPUT.PUT_LINE('⚠ Rollback incomplete - manual cleanup may be required'); + DBMS_OUTPUT.PUT_LINE('WARNING: Rollback incomplete - manual cleanup may be required'); END IF; DBMS_OUTPUT.PUT_LINE(''); diff --git a/MARS_Packages/REL02_POST/MARS-956/README.md b/MARS_Packages/REL02_POST/MARS-956/README.md deleted file mode 100644 index fef7aef..0000000 --- a/MARS_Packages/REL02_POST/MARS-956/README.md +++ /dev/null @@ -1,296 +0,0 @@ -# MARS-956: One-Time C2D MPEC Data Export from Operational Database to External Tables - -## Overview -This package performs a one-time bulk export of C2D MPEC data from operational database tables (OU_C2D schema) to new external tables in OCI buckets. The export uses DATA_EXPORTER v2.7.5 with pRegisterExport parameter to move historical data to ODS bucket in CSV format with automatic file registration. - -**Migration Strategy:** -- **Source**: OU_C2D schema tables (MPEC_ADMIN, MPEC_CONTENT, MPEC_CONTENT_CRITERION) -- **Target**: ODS bucket (CSV format) -- **Registration**: Files automatically registered in A_SOURCE_FILE_RECEIVED -- **Template Compatibility**: Uses CT_ET_TEMPLATES.C2D_MPEC_* for column ordering - -**Key Features:** -- Automatic file registration with pRegisterExport=TRUE -- Template table column order compliance -- Comprehensive verification and rollback capabilities -- ODS/C2D bucket path structure - -## Contents -- `install_mars956.sql` - Master installation script with SPOOL logging -- `rollback_mars956.sql` - Master rollback script -- `00_MARS_956_*.sql` - Prerequisites check -- `01_MARS_956_*.sql` - Data export script -- `02_MARS_956_*.sql` - Export verification -- `03_MARS_956_*.sql` - Data integrity validation -- `91_MARS_956_*.sql` - Rollback scripts -- `99_MARS_956_*.sql` - Rollback verification - -## Prerequisites -- Oracle Database 23ai with DBMS_CLOUD -- ADMIN user access (required for all MARS installations) -- DATA_EXPORTER package v2.7.5 or higher -- OU_C2D schema with populated MPEC tables -- CT_ET_TEMPLATES schema with C2D_MPEC_* template tables -- OCI bucket access credentials configured - -## Data Structure - -### Source Tables (OU_C2D schema) -- **MPEC_ADMIN**: 5 records with ETL keys 2001-2005 -- **MPEC_CONTENT**: 5 records with ETL keys 2006-2008 -- **MPEC_CONTENT_CRITERION**: 9 records with ETL keys 2009-2010 - -### Template Tables (CT_ET_TEMPLATES schema) -- **C2D_MPEC_ADMIN**: 10 columns -- **C2D_MPEC_CONTENT**: 87 columns -- **C2D_MPEC_CONTENT_CRITERION**: 4 columns - -### Target Structure (OCI ODS bucket) -``` -mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/*.csv -mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv -mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv -``` - -## Enhanced Validation Framework - -### Pre-Export Validation (Check 1-5) -1. **Source Table Analysis**: Comprehensive source table record counting with business rule validation -2. **Template Table Verification**: Ensures external table templates exist and are accessible -3. **ETL Key Validation**: Verifies integrity of load history tracking keys (expected: 2001-2010) -4. **Schema Permissions**: Validates all required schema access permissions -5. **External Table Readiness**: Checks accessibility of target external tables (ODS.C2D_MPEC_*_ODS) with proper handling of empty states - -### In-Export Validation -- **Pre-Export Record Count**: Source and target table counts before export begins -- **Live Progress Monitoring**: Real-time feedback during export operations -- **Post-Export Verification**: Immediate source vs target COUNT(*) comparison table -- **Error Handling**: Comprehensive error capture with detailed diagnostic information - -### Post-Export Validation -- **Record Count Verification**: Detailed source vs target COUNT(*) comparison for all exported tables -- **File Registration Validation**: Confirms all exported files are properly registered in A_SOURCE_FILE_RECEIVED -- **Data Integrity Analysis**: Validates data structure and completeness using external table access -- **Delta Query Readiness**: Confirms external tables ready for operational delta queries -mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv -mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv -``` - -## Installation - -## Installation - -### Prerequisites -**IMPORTANT**: External tables must already exist on the database (created via FILE_MANAGER procedures) - -Required external tables in ODS schema: -- `ODS.C2D_MPEC_ADMIN_ODS` -- `ODS.C2D_MPEC_CONTENT_ODS` -- `ODS.C2D_MPEC_CONTENT_CRITERION_ODS` - -If external tables don't exist, use `FILE_MANAGER.CREATE_EXTERNAL_TABLE` or `ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE` procedures to create them. - -### Installation Command -```powershell -# Run as ADMIN user -cd c:\_git\MARS\wkcopy02\MARS_Packages\REL02_POST\MARS-956-NEW -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars956.sql" - -# Log file created: log/INSTALL_MARS_956__.log -``` - -### Alternative: Individual Scripts -```powershell -# Execute individual scripts for step-by-step validation -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@00_MARS_956_pre_check_prerequisites.sql" -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@01_MARS_956_export_c2d_mpec_data.sql" -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@02_MARS_956_verify_exports.sql" -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@03_MARS_956_verify_data_integrity.sql" -``` - -## Expected Results - -### File Registration -- **9 CSV files** registered in A_SOURCE_FILE_RECEIVED -- **File naming pattern**: {ETL_KEY}_1_{TIMESTAMP}Z.csv -- **Processing status**: INGESTED -- **Total size**: ~0.33 KB - -### Export Distribution -- **MPEC_ADMIN**: 3 files (ETL keys 2001-2005, 3 distinct keys) -- **MPEC_CONTENT**: 3 files (ETL keys 2006-2008, 3 distinct keys) -- **MPEC_CONTENT_CRITERION**: 3 files (ETL keys 2009-2010, 3 distinct keys) - -### Duration -- **Expected**: 2-5 minutes (small datasets) -- **Actual**: ~16 seconds (based on testing) - -## Verification - -### Manual Verification -```sql --- Check export results -SELECT COUNT(*) AS TOTAL_FILES FROM CT_MRDS.A_SOURCE_FILE_RECEIVED -WHERE RECEPTION_DATE >= SYSDATE - 1/24 AND SOURCE_FILE_NAME LIKE '200%'; - --- Check process logs -SELECT PROCESS_NAME, LOG_LEVEL, LOG_MESSAGE FROM CT_MRDS.A_PROCESS_LOG -WHERE PROCESS_NAME = 'MARS-956' ORDER BY LOG_TIMESTAMP DESC; -``` - -### Cloud Bucket Verification -```sql --- List exported files in OCI bucket -SELECT object_name FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects( - credential_name => 'DEF_CRED_ARN', - location_uri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/data/' -)) WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%'; -``` - -## Rollback - -### Full Rollback (if needed) -```powershell -cd c:\_git\MARS\wkcopy02\MARS_Packages\REL02_POST\MARS-956-NEW -sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars956.sql" -``` - -### Manual Rollback -```sql --- Remove file registrations -DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED -WHERE SOURCE_FILE_NAME LIKE '200%' AND RECEPTION_DATE >= SYSDATE - 7; - --- Remove process logs -DELETE FROM CT_MRDS.A_PROCESS_LOG -WHERE PROCESS_NAME IN ('MARS-956', 'MARS-956-ROLLBACK') AND LOG_TIMESTAMP >= SYSDATE - 7; -``` - -## Troubleshooting - -### Common Issues - -1. **ORA-17008: Closed connection** - - Use MCP SQLcl server instead of traditional SQLcl - - Connection: `CT_MRDS@ggmichalski_high` - -2. **No files registered** - - Check DATA_EXPORTER package version (requires v2.7.5+) - - Verify pRegisterExport=TRUE parameter - -3. **Template table errors** - - Ensure CT_ET_TEMPLATES.C2D_MPEC_* tables exist - - Check column compatibility with source tables - -4. **ETL key validation errors** - - Verify A_ETL_LOAD_SET_FK values exist in CT_ODS.A_LOAD_HISTORY - - Check ETL key range: 2001-2010 - -### Error Resolution -- Check `log/INSTALL_MARS_956_*.log` for detailed error information -- Review A_PROCESS_LOG table for runtime errors -- Use verification scripts to diagnose specific issues - -### Common Validation Issues - -#### External Table Access Errors -**Problem**: "✗ ERROR: Cannot access C2D_MPEC_*_ODS tables" -- **Cause**: Missing external table definitions or incorrect bucket paths -- **Solution**: Verify external tables exist and point to correct OCI bucket paths -- **Check**: `SELECT * FROM ODS.C2D_MPEC_ADMIN_ODS WHERE ROWNUM <= 1;` -- **Create**: Use `ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE` if tables don't exist - -#### Record Count Mismatches -**Problem**: Source vs target COUNT(*) differences in post-export validation -- **Cause**: Export failure, external table configuration, or bucket synchronization lag -- **Solution**: Wait 30-60 seconds and re-run verification script -- **Check**: Run `@03_MARS_956_verify_data_integrity.sql` again - -#### Empty Source Tables -**Problem**: "⚠ WARNING: No source data found" -- **Cause**: Source tables not populated or incorrect schema permissions -- **Solution**: Verify OU_C2D schema contains expected MPEC data -- **Check**: `SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN;` (should return 5) - -#### ETL Key Validation Failures -**Problem**: ETL keys outside expected range 2001-2010 -- **Cause**: Data inconsistency or load history corruption -- **Solution**: Review source data quality and CT_ODS.A_LOAD_HISTORY entries -- **Check**: Verify ETL key distribution in source tables - -#### External Table Empty State Handling -**Problem**: KUP-13023 or ORA-29913 errors during validation -- **Expected**: These errors are normal for empty external tables -- **Handling**: Pre-checks properly handle empty states as normal condition -- **Action**: No action required if source tables contain data - -## Dependencies - -### External Tables Setup (If Not Already Created) - -If external tables don't exist, create them using FILE_MANAGER procedures: - -```sql --- Option 1: Using ODS.FILE_MANAGER_ODS (recommended - AUTHID DEFINER) -BEGIN - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => 'C2D_MPEC_ADMIN_ODS', - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_ADMIN', - pPrefix => 'ODS/C2D/C2D_MPEC_ADMIN', - pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri - ); - - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => 'C2D_MPEC_CONTENT_ODS', - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT', - pPrefix => 'ODS/C2D/C2D_MPEC_CONTENT', - pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri - ); - - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => 'C2D_MPEC_CONTENT_CRITERION_ODS', - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT_CRITERION', - pPrefix => 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION', - pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri - ); -END; -/ - --- Option 2: Using CT_MRDS.FILE_MANAGER (AUTHID CURRENT_USER - requires ODS context) --- Connect as ODS user first --- Then call CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE with same parameters -``` - -### Package Dependencies -- **CT_MRDS.DATA_EXPORTER** v2.7.5+ (with pRegisterExport support) -- **CT_MRDS.ENV_MANAGER** v3.1.0+ -- **MRDS_LOADER.cloud_wrapper** (for OCI bucket access) - -### Schema Dependencies -- **OU_C2D**: Source data schema -- **CT_ET_TEMPLATES**: Template table definitions -- **CT_MRDS**: Process logging and configuration -- **CT_ODS**: ETL load history validation - -### Infrastructure Dependencies -- **OCI Object Storage**: ODS bucket access -- **Database credentials**: DEF_CRED_ARN configured -- **Network connectivity**: Frankfurt region access - -## Related MARS Issues -- **MARS-956**: C2D MPEC Historical Data Export (this package) -- **MARS-835**: CSDB Data Export (reference implementation) -- **Related**: DATA_EXPORTER package enhancements - -## Support -For technical support or issues: -1. Check installation log files in `log/` directory -2. Review verification script outputs -3. Contact Database Team with specific error messages -4. Include A_PROCESS_LOG entries for debugging - ---- -**Package Version**: 1.0.0 -**Created**: 2026-02-12 -**Author**: System Generated (based on MARS-835 structure) -**Status**: Ready for deployment \ No newline at end of file diff --git a/MARS_Packages/REL02_POST/MARS-956/install_mars956.sql b/MARS_Packages/REL02_POST/MARS-956/install_mars956.sql index cda29ef..11c6e8a 100644 --- a/MARS_Packages/REL02_POST/MARS-956/install_mars956.sql +++ b/MARS_Packages/REL02_POST/MARS-956/install_mars956.sql @@ -1,7 +1,7 @@ -- =================================================================== -- MARS-956 INSTALL SCRIPT: C2D MPEC Data Export to External Tables -- =================================================================== --- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_C2D schema +-- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema -- to OCI buckets (ODS bucket CSV format) -- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration -- Author: System Generated @@ -37,16 +37,14 @@ PROMPT PROMPT This script will export 3 C2D MPEC tables to OCI buckets: PROMPT PROMPT TARGET: ODS Bucket (CSV format): -PROMPT - MPEC_ADMIN: 5 records with ETL keys 2001-2005 -PROMPT - MPEC_CONTENT: 5 records with ETL keys 2006-2008 -PROMPT - MPEC_CONTENT_CRITERION: 9 records with ETL keys 2009-2010 +PROMPT - MPEC_ADMIN +PROMPT - MPEC_CONTENT +PROMPT - MPEC_CONTENT_CRITERION PROMPT PROMPT Key Features: PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*) PROMPT - ODS/C2D bucket path structure -PROMPT -PROMPT Expected Duration: 2-5 minutes (small datasets) PROMPT ========================================================================= -- Confirm installation with user diff --git a/MARS_Packages/REL02_POST/MARS-956/rollback_mars956.sql b/MARS_Packages/REL02_POST/MARS-956/rollback_mars956.sql index dbbf7ee..32b70f4 100644 --- a/MARS_Packages/REL02_POST/MARS-956/rollback_mars956.sql +++ b/MARS_Packages/REL02_POST/MARS-956/rollback_mars956.sql @@ -47,19 +47,25 @@ WHENEVER SQLERROR CONTINUE PROMPT PROMPT ========================================================================= -PROMPT Step 1: Delete File Registrations +PROMPT Step 1: Delete Exported CSV Files from DATA Bucket +PROMPT ========================================================================= +@@90_MARS_956_rollback_delete_csv_files.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 2: Delete File Registrations PROMPT ========================================================================= @@91_MARS_956_rollback_file_registrations.sql PROMPT PROMPT ========================================================================= -PROMPT Step 2: Clean Process Logs +PROMPT Step 3: Clean Process Logs PROMPT ========================================================================= @@92_MARS_956_rollback_process_logs.sql PROMPT PROMPT ========================================================================= -PROMPT Step 3: Verify Rollback Completion +PROMPT Step 4: Verify Rollback Completion PROMPT ========================================================================= @@99_MARS_956_verify_rollback.sql