diff --git a/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql b/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql index 5d83cf6..1967749 100644 --- a/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql +++ b/MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql @@ -44,9 +44,10 @@ PROMPT ========================================================================= -- Helper procedure (inline) to check one folder -- Check 1: ALLOTMENT DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT/'; SELECT COUNT(*) INTO vFileCount @@ -58,7 +59,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -73,9 +79,10 @@ END; -- Check 2: ALLOTMENT_MODIFICATION_HEADER DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/'; SELECT COUNT(*) INTO vFileCount @@ -87,7 +94,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -102,9 +114,10 @@ END; -- Check 3: ALLOTMENT_MODIFICATION_ITEM DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/'; SELECT COUNT(*) INTO vFileCount @@ -116,7 +129,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -131,9 +149,10 @@ END; -- Check 4: ANNOUNCEMENT DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ANNOUNCEMENT/'; SELECT COUNT(*) INTO vFileCount @@ -145,7 +164,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -160,9 +184,10 @@ END; -- Check 5: FBL_ITEM (folder: TOP_FULLBIDLIST_ITEM) DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBIDLIST_ITEM/'; SELECT COUNT(*) INTO vFileCount @@ -174,7 +199,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -189,9 +219,10 @@ END; -- Check 6: FULLBID_ARRAY_COMPILED DECLARE - vFileCount NUMBER := 0; + vFileCount NUMBER := 0; vRecordCount NUMBER := 0; vLocationUri VARCHAR2(1000); + vPrintCount NUMBER := 0; BEGIN vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/'; SELECT COUNT(*) INTO vFileCount @@ -203,7 +234,12 @@ BEGIN DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount); FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)'); + vPrintCount := vPrintCount + 1; + EXIT WHEN vPrintCount >= 10; END LOOP; + IF vFileCount > 10 THEN + DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)'); + END IF; BEGIN EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS' INTO vRecordCount; DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount); @@ -229,6 +265,8 @@ DECLARE vTotalSource NUMBER := 0; vTotalTarget NUMBER := 0; + -- safe_count: ONLY for ODS external tables + -- Returns 0 when no data file (ORA-29913, ORA-29400, KUP-13023); re-raises all other errors PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS BEGIN EXECUTE IMMEDIATE pSql INTO pResult; @@ -237,11 +275,11 @@ DECLARE IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN pResult := 0; ELSE - pResult := -1; + RAISE; END IF; END; BEGIN - -- Source counts + -- Source counts (direct - if table does not exist, error propagates) EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Source; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Source; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Source; @@ -272,25 +310,23 @@ BEGIN DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('Target external table record counts (pre-export):'); - DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_ODS: ' || CASE WHEN v1Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v1Target) END); - DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_HEADER_ODS: ' || CASE WHEN v2Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v2Target) END); - DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_ITEM_ODS: ' || CASE WHEN v3Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v3Target) END); - DBMS_OUTPUT.PUT_LINE('- TOP_ANNOUNCEMENT_ODS: ' || CASE WHEN v4Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v4Target) END); - DBMS_OUTPUT.PUT_LINE('- TOP_FULLBIDLIST_ITEM_ODS: ' || CASE WHEN v5Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v5Target) END); - DBMS_OUTPUT.PUT_LINE('- TOP_FULLBID_ARRAY_COMPILED_ODS: ' || CASE WHEN v6Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v6Target) END); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_ODS: ' || v1Target); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_HEADER_ODS: ' || v2Target); + DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_ITEM_ODS: ' || v3Target); + DBMS_OUTPUT.PUT_LINE('- TOP_ANNOUNCEMENT_ODS: ' || v4Target); + DBMS_OUTPUT.PUT_LINE('- TOP_FULLBIDLIST_ITEM_ODS: ' || v5Target); + DBMS_OUTPUT.PUT_LINE('- TOP_FULLBID_ARRAY_COMPILED_ODS: ' || v6Target); IF vTotalSource > 0 THEN DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export'); ELSE - DBMS_OUTPUT.PUT_LINE('ERROR: No source data found'); + DBMS_OUTPUT.PUT_LINE('WARNING: Source tables exist but contain no data - export will produce empty files'); END IF; IF vTotalTarget = 0 THEN DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export'); - ELSIF vTotalTarget > 0 THEN - DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); ELSE - DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access target external tables'); + DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); END IF; DBMS_OUTPUT.PUT_LINE('Proceeding with export...'); @@ -306,7 +342,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_ALLOTMENT', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_ALLOTMENT', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT', @@ -336,7 +372,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_HEADER', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER', @@ -366,7 +402,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_ITEM', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM', @@ -396,7 +432,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_ANNOUNCEMENT', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_ANNOUNCEMENT', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ANNOUNCEMENT', @@ -426,7 +462,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_FBL_ITEM', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_FULLBIDLIST_ITEM', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM', @@ -456,7 +492,7 @@ BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_TOP', pTableName => 'LEGACY_FULLBID_ARRAY_COMPILED', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'ODS', pFolderName => 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED', pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED', @@ -502,22 +538,30 @@ DECLARE vTotalT NUMBER := 0; vMismatch NUMBER := 0; + -- safe_count: ONLY for ODS external tables + -- Returns 0 when no data file (ORA-29913, ORA-29400, KUP-13023); re-raises all other errors PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS BEGIN EXECUTE IMMEDIATE pSql INTO pResult; - EXCEPTION WHEN OTHERS THEN pResult := -1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN + pResult := 0; + ELSE + RAISE; + END IF; END; PROCEDURE print_row(pTable VARCHAR2, pSrc NUMBER, pTgt NUMBER) IS BEGIN DBMS_OUTPUT.PUT_LINE( RPAD(pTable, 40) || ' | ' || - RPAD(CASE WHEN pSrc = -1 THEN 'N/A' ELSE TO_CHAR(pSrc) END, 8) || ' | ' || - RPAD(CASE WHEN pTgt = -1 THEN 'ERROR' ELSE TO_CHAR(pTgt) END, 8) || ' | ' || + RPAD(TO_CHAR(pSrc), 8) || ' | ' || + RPAD(TO_CHAR(pTgt), 8) || ' | ' || CASE WHEN pSrc = pTgt THEN 'OK' ELSE 'MISMATCH' END); END; BEGIN - -- Source + -- Source (direct - if table does not exist, error propagates) EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1S; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2S; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3S; @@ -533,8 +577,7 @@ BEGIN safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4T); safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5T); safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6T); - vTotalT := GREATEST(v1T,0) + GREATEST(v2T,0) + GREATEST(v3T,0) - + GREATEST(v4T,0) + GREATEST(v5T,0) + GREATEST(v6T,0); + vTotalT := v1T + v2T + v3T + v4T + v5T + v6T; DBMS_OUTPUT.PUT_LINE('POST-EXPORT VERIFICATION SUMMARY'); DBMS_OUTPUT.PUT_LINE(RPAD('Table', 40) || ' | Source | Target | Match');