admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement rollback functionality for MARS-956:

Browse Source 
- Added script to delete exported CSV files from DATA bucket.
- Enhanced rollback file registrations script with improved error handling and success messages.
- Updated process logs cleanup script to include error handling and success messages.
- Modified rollback verification script for clearer success and warning messages.
- Removed outdated README.md file and updated installation script to reflect schema changes.
- Adjusted rollback master script to streamline execution steps and improve clarity.
This commit is contained in:
Grzegorz Michalski
2026-02-13 07:50:54 +01:00
parent a3ec31341b
commit 3b7ce7de97
11 changed files with 586 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
MARS_Packages/REL02_POST/MARS-956/00_MARS_956_pre_check_prerequisites.sql
View File

@@ -44,22 +44,22 @@ BEGIN
BEGIN BEGIN
DBMS_OUTPUT.PUT_LINE('DATA_EXPORTER Version: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_VERSION); DBMS_OUTPUT.PUT_LINE('DATA_EXPORTER Version: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_VERSION);
DBMS_OUTPUT.PUT_LINE('Build Date: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_BUILD_DATE); DBMS_OUTPUT.PUT_LINE('Build Date: ' || CT_MRDS.DATA_EXPORTER.PACKAGE_BUILD_DATE);
DBMS_OUTPUT.PUT_LINE('✓ DATA_EXPORTER package is available'); DBMS_OUTPUT.PUT_LINE('SUCCESS: DATA_EXPORTER package is available');
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('✗ ERROR: DATA_EXPORTER package not available: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('ERROR: ERROR: DATA_EXPORTER package not available: ' || SQLERRM);
RAISE; RAISE;
END; END;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Check 2: Verify Source Tables in OU_C2D Schema'); DBMS_OUTPUT.PUT_LINE('Check 2: Verify Source Tables in OU_LEGACY_C2D Schema');
DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check source table row counts -- Check source table row counts
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
DBMS_OUTPUT.PUT_LINE('Source table row counts:'); DBMS_OUTPUT.PUT_LINE('Source table row counts:');
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' rows'); DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' rows');
@@ -67,9 +67,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' rows'); DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' rows');
IF vAdminRows > 0 AND vContentRows > 0 AND vCriterionRows > 0 THEN IF vAdminRows > 0 AND vContentRows > 0 AND vCriterionRows > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All source tables have data'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables have data');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ One or more source tables are empty'); DBMS_OUTPUT.PUT_LINE('ERROR: One or more source tables are empty');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
@@ -102,9 +102,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns');
IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All template tables have defined structure'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ One or more template tables missing columns'); DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
@@ -121,22 +121,22 @@ BEGIN
SELECT COUNT(DISTINCT etl_key) SELECT COUNT(DISTINCT etl_key)
INTO vTotalSourceKeys INTO vTotalSourceKeys
FROM ( FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
); );
-- Count how many exist in A_LOAD_HISTORY -- Count how many exist in A_LOAD_HISTORY
SELECT COUNT(DISTINCT etl_key) SELECT COUNT(DISTINCT etl_key)
INTO vValidKeys INTO vValidKeys
FROM ( FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
) src ) src
WHERE EXISTS ( WHERE EXISTS (
SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h
@@ -148,9 +148,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys);
IF vValidKeys = vTotalSourceKeys THEN IF vValidKeys = vTotalSourceKeys THEN
DBMS_OUTPUT.PUT_LINE('✓ All source ETL keys are valid'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys));
END IF; END IF;
END; END;
@@ -219,14 +219,14 @@ BEGIN
IF vAdminExtCount >= 0 AND vContentExtCount >= 0 AND vCriterionExtCount >= 0 THEN IF vAdminExtCount >= 0 AND vContentExtCount >= 0 AND vCriterionExtCount >= 0 THEN
IF vAdminExtCount = 0 AND vContentExtCount = 0 AND vCriterionExtCount = 0 THEN IF vAdminExtCount = 0 AND vContentExtCount = 0 AND vCriterionExtCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All target external tables are clean (ready for fresh export)'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All target external tables are clean (ready for fresh export)');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Target external tables contain data (' || DBMS_OUTPUT.PUT_LINE('WARNING: Target external tables contain data (' ||
(vAdminExtCount + vContentExtCount + vCriterionExtCount) || ' total records)'); (vAdminExtCount + vContentExtCount + vCriterionExtCount) || ' total records)');
DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run'); DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run');
END IF; END IF;
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ Some external tables are inaccessible - check table definitions'); DBMS_OUTPUT.PUT_LINE('ERROR: Some external tables are inaccessible - check table definitions');
END IF; END IF;
END; END;
@@ -253,30 +253,30 @@ BEGIN
END LOOP; END LOOP;
IF vFileCount = 0 THEN IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ No existing C2D MPEC files found - ready for clean export'); DBMS_OUTPUT.PUT_LINE('SUCCESS: No existing C2D MPEC files found - ready for clean export');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vFileCount || ' existing C2D MPEC files found'); DBMS_OUTPUT.PUT_LINE('WARNING: Warning: ' || vFileCount || ' existing C2D MPEC files found');
DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run'); DBMS_OUTPUT.PUT_LINE(' Consider rollback if this is a re-run');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('✗ ERROR checking existing files: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('ERROR: ERROR checking existing files: ' || SQLERRM);
END; END;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('MARS-956 Pre-Check Summary'); DBMS_OUTPUT.PUT_LINE('MARS-956 Pre-Check Summary');
DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('✓ DATA_EXPORTER package available'); DBMS_OUTPUT.PUT_LINE('SUCCESS: DATA_EXPORTER package available');
DBMS_OUTPUT.PUT_LINE('✓ Source tables: ' || (vAdminRows + vContentRows + vCriterionRows) || ' total rows'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables: ' || (vAdminRows + vContentRows + vCriterionRows) || ' total rows');
DBMS_OUTPUT.PUT_LINE('✓ Template tables: All structures defined'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Template tables: All structures defined');
DBMS_OUTPUT.PUT_LINE('✓ ETL keys: All validated in A_LOAD_HISTORY'); DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL keys: All validated in A_LOAD_HISTORY');
DBMS_OUTPUT.PUT_LINE('✓ External tables: Accessible and ready'); DBMS_OUTPUT.PUT_LINE('SUCCESS: External tables: Accessible and ready');
IF vFileCount > 0 THEN IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('⚠ Existing files: ' || vFileCount || ' (consider rollback)'); DBMS_OUTPUT.PUT_LINE('WARNING: Existing files: ' || vFileCount || ' (consider rollback)');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✓ Target bucket: Clean for export'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Target bucket: Clean for export');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');

260
MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql
View File

@@ -14,7 +14,7 @@ PROMPT =========================================================================
PROMPT MARS-956: C2D MPEC Historical Data Export PROMPT MARS-956: C2D MPEC Historical Data Export
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT Export Strategy: PROMPT Export Strategy:
PROMPT - Source: OU_C2D schema tables (operational database) PROMPT - Source: OU_LEGACY_C2D schema tables (operational database)
PROMPT - Target: ODS bucket as CSV files PROMPT - Target: ODS bucket as CSV files
PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA
PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED
@@ -26,6 +26,198 @@ INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_
VALUES ('MARS-956', 'EXPORT_C2D_MPEC_DATA', 'INFO', 'Starting historical C2D MPEC data export', VALUES ('MARS-956', 'EXPORT_C2D_MPEC_DATA', 'INFO', 'Starting historical C2D MPEC data export',
'Tables: MPEC_ADMIN, MPEC_CONTENT, MPEC_CONTENT_CRITERION'); 'Tables: MPEC_ADMIN, MPEC_CONTENT, MPEC_CONTENT_CRITERION');
PROMPT
PROMPT =====================================================================================
PROMPT PRE-EXPORT CHECK: Verify Existing Files in ODS Bucket
PROMPT =====================================================================================
-- Check 1: MPEC_ADMIN files
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
-- Get bucket URI for DATA bucket
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_ADMIN/';
-- Count existing files
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'; -- Exclude directories
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_ADMIN files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
-- List existing files
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
-- Count records in external table
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_ADMIN_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_ADMIN files found - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Check 2: MPEC_CONTENT files
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_CONTENT/';
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_CONTENT files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_CONTENT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_CONTENT files found - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Check 3: MPEC_CONTENT_CRITERION files
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/';
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: MPEC_CONTENT_CRITERION files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.C2D_MPEC_CONTENT_CRITERION_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing MPEC_CONTENT_CRITERION files found - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
PROMPT PROMPT
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT PRE-EXPORT: Verify Source and Target Table Readiness PROMPT PRE-EXPORT: Verify Source and Target Table Readiness
@@ -43,9 +235,9 @@ DECLARE
vTotalTarget NUMBER := 0; vTotalTarget NUMBER := 0;
BEGIN BEGIN
-- Source table counts -- Source table counts
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
vTotalSource := vAdminRows + vContentRows + vCriterionRows; vTotalSource := vAdminRows + vContentRows + vCriterionRows;
DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):'); DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):');
@@ -106,17 +298,17 @@ BEGIN
CASE WHEN vTotalTarget = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(vTotalTarget) END); CASE WHEN vTotalTarget = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(vTotalTarget) END);
IF vTotalSource > 0 THEN IF vTotalSource > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ Source tables contain data - ready for export'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ WARNING: No source data found'); DBMS_OUTPUT.PUT_LINE('ERROR: WARNING: No source data found');
END IF; END IF;
IF vTotalTarget = 0 THEN IF vTotalTarget = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ Target external tables are clean - ready for fresh export'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export');
ELSIF vTotalTarget > 0 THEN ELSIF vTotalTarget > 0 THEN
DBMS_OUTPUT.PUT_LINE('⚠ WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run'); DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ ERROR: Cannot access target external tables'); DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access target external tables');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
@@ -126,14 +318,12 @@ END;
PROMPT PROMPT
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT TABLE 1/3: OU_C2D.MPEC_ADMIN -> ODS/C2D/C2D_MPEC_ADMIN PROMPT TABLE 1/3: OU_LEGACY_C2D.MPEC_ADMIN -> ODS/C2D/C2D_MPEC_ADMIN
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT Expected: 5 records with ETL keys 2001-2005
PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/*.csv
BEGIN BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_C2D', pSchemaName => 'OU_LEGACY_C2D',
pTableName => 'MPEC_ADMIN', pTableName => 'MPEC_ADMIN',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS', pBucketArea => 'ODS',
@@ -143,13 +333,13 @@ BEGIN
pRegisterExport => TRUE -- Register files in A_SOURCE_FILE_RECEIVED pRegisterExport => TRUE -- Register files in A_SOURCE_FILE_RECEIVED
); );
DBMS_OUTPUT.PUT_LINE('✓ MPEC_ADMIN export completed successfully'); DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_ADMIN export completed successfully');
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DECLARE DECLARE
vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000);
BEGIN BEGIN
DBMS_OUTPUT.PUT_LINE('✗ MPEC_ADMIN export failed: ' || vErrorMsg); DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_ADMIN export failed: ' || vErrorMsg);
-- Log error using proper ENV_MANAGER pattern -- Log error using proper ENV_MANAGER pattern
INSERT INTO CT_MRDS.A_PROCESS_LOG INSERT INTO CT_MRDS.A_PROCESS_LOG
(guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
@@ -164,14 +354,12 @@ END;
PROMPT PROMPT
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT TABLE 2/3: OU_C2D.MPEC_CONTENT -> ODS/C2D/C2D_MPEC_CONTENT PROMPT TABLE 2/3: OU_LEGACY_C2D.MPEC_CONTENT -> ODS/C2D/C2D_MPEC_CONTENT
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT Expected: 5 records with ETL keys 2006-2008
PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv
BEGIN BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_C2D', pSchemaName => 'OU_LEGACY_C2D',
pTableName => 'MPEC_CONTENT', pTableName => 'MPEC_CONTENT',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ODS', pBucketArea => 'ODS',
@@ -181,13 +369,13 @@ BEGIN
pRegisterExport => TRUE pRegisterExport => TRUE
); );
DBMS_OUTPUT.PUT_LINE('✓ MPEC_CONTENT export completed successfully'); DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_CONTENT export completed successfully');
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DECLARE DECLARE
vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000);
BEGIN BEGIN
DBMS_OUTPUT.PUT_LINE('✗ MPEC_CONTENT export failed: ' || vErrorMsg); DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_CONTENT export failed: ' || vErrorMsg);
-- Log error using proper ENV_MANAGER pattern -- Log error using proper ENV_MANAGER pattern
INSERT INTO CT_MRDS.A_PROCESS_LOG INSERT INTO CT_MRDS.A_PROCESS_LOG
(guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
@@ -202,14 +390,12 @@ END;
PROMPT PROMPT
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT TABLE 3/3: OU_C2D.MPEC_CONTENT_CRITERION -> ODS/C2D/C2D_MPEC_CONTENT_CRITERION PROMPT TABLE 3/3: OU_LEGACY_C2D.MPEC_CONTENT_CRITERION -> ODS/C2D/C2D_MPEC_CONTENT_CRITERION
PROMPT ===================================================================================== PROMPT =====================================================================================
PROMPT Expected: 9 records with ETL keys 2009-2010
PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv
BEGIN BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_C2D', pSchemaName => 'OU_LEGACY_C2D',
pTableName => 'MPEC_CONTENT_CRITERION', pTableName => 'MPEC_CONTENT_CRITERION',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ODS', pBucketArea => 'ODS',
@@ -219,13 +405,13 @@ BEGIN
pRegisterExport => TRUE pRegisterExport => TRUE
); );
DBMS_OUTPUT.PUT_LINE('✓ MPEC_CONTENT_CRITERION export completed successfully'); DBMS_OUTPUT.PUT_LINE('SUCCESS: MPEC_CONTENT_CRITERION export completed successfully');
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DECLARE DECLARE
vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000);
BEGIN BEGIN
DBMS_OUTPUT.PUT_LINE('✗ MPEC_CONTENT_CRITERION export failed: ' || vErrorMsg); DBMS_OUTPUT.PUT_LINE('ERROR: MPEC_CONTENT_CRITERION export failed: ' || vErrorMsg);
-- Log error using proper ENV_MANAGER pattern -- Log error using proper ENV_MANAGER pattern
INSERT INTO CT_MRDS.A_PROCESS_LOG INSERT INTO CT_MRDS.A_PROCESS_LOG
(guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message) (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
@@ -281,9 +467,9 @@ DECLARE
vMismatchCount NUMBER := 0; vMismatchCount NUMBER := 0;
BEGIN BEGIN
-- Source table counts -- Source table counts
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminSource; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminSource;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentSource; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentSource;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionSource; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionSource;
vTotalSource := vAdminSource + vContentSource + vCriterionSource; vTotalSource := vAdminSource + vContentSource + vCriterionSource;
-- Target external table counts -- Target external table counts
@@ -302,40 +488,40 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('MPEC_ADMIN | ' || DBMS_OUTPUT.PUT_LINE('MPEC_ADMIN | ' ||
RPAD(vAdminSource, 8) || ' | ' || RPAD(vAdminSource, 8) || ' | ' ||
RPAD(vAdminTarget, 8) || ' | ' || RPAD(vAdminTarget, 8) || ' | ' ||
CASE WHEN vAdminSource = vAdminTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); CASE WHEN vAdminSource = vAdminTarget THEN 'OK' ELSE 'MISMATCH' END);
IF vAdminSource != vAdminTarget THEN vMismatchCount := vMismatchCount + 1; END IF; IF vAdminSource != vAdminTarget THEN vMismatchCount := vMismatchCount + 1; END IF;
-- MPEC_CONTENT comparison -- MPEC_CONTENT comparison
DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT | ' || DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT | ' ||
RPAD(vContentSource, 8) || ' | ' || RPAD(vContentSource, 8) || ' | ' ||
RPAD(vContentTarget, 8) || ' | ' || RPAD(vContentTarget, 8) || ' | ' ||
CASE WHEN vContentSource = vContentTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); CASE WHEN vContentSource = vContentTarget THEN 'OK' ELSE 'MISMATCH' END);
IF vContentSource != vContentTarget THEN vMismatchCount := vMismatchCount + 1; END IF; IF vContentSource != vContentTarget THEN vMismatchCount := vMismatchCount + 1; END IF;
-- MPEC_CONTENT_CRITERION comparison -- MPEC_CONTENT_CRITERION comparison
DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT_CRITERION | ' || DBMS_OUTPUT.PUT_LINE('MPEC_CONTENT_CRITERION | ' ||
RPAD(vCriterionSource, 8) || ' | ' || RPAD(vCriterionSource, 8) || ' | ' ||
RPAD(vCriterionTarget, 8) || ' | ' || RPAD(vCriterionTarget, 8) || ' | ' ||
CASE WHEN vCriterionSource = vCriterionTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); CASE WHEN vCriterionSource = vCriterionTarget THEN 'OK' ELSE 'MISMATCH' END);
IF vCriterionSource != vCriterionTarget THEN vMismatchCount := vMismatchCount + 1; END IF; IF vCriterionSource != vCriterionTarget THEN vMismatchCount := vMismatchCount + 1; END IF;
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('TOTAL | ' || DBMS_OUTPUT.PUT_LINE('TOTAL | ' ||
RPAD(vTotalSource, 8) || ' | ' || RPAD(vTotalSource, 8) || ' | ' ||
RPAD(vTotalTarget, 8) || ' | ' || RPAD(vTotalTarget, 8) || ' | ' ||
CASE WHEN vTotalSource = vTotalTarget THEN '✓ OK' ELSE '✗ MISMATCH' END); CASE WHEN vTotalSource = vTotalTarget THEN 'OK' ELSE 'MISMATCH' END);
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 THEN IF vMismatchCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ SUCCESS: All record counts match - export verified'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All record counts match - export verified');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ WARNING: ' || vMismatchCount || ' table(s) have record count mismatches'); DBMS_OUTPUT.PUT_LINE('WARNING: ' || vMismatchCount || ' table(s) have record count mismatches');
DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions'); DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('✗ ERROR: Cannot verify target external tables post-export'); DBMS_OUTPUT.PUT_LINE('ERROR: Cannot verify target external tables post-export');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
DBMS_OUTPUT.PUT_LINE('Please check external table configuration and ODS bucket access'); DBMS_OUTPUT.PUT_LINE('Please check external table configuration and ODS bucket access');
END; END;

28
MARS_Packages/REL02_POST/MARS-956/02_MARS_956_verify_exports.sql
View File

@@ -33,15 +33,15 @@ BEGIN
OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys
OR SOURCE_FILE_NAME LIKE '2010_%'); OR SOURCE_FILE_NAME LIKE '2010_%');
DBMS_OUTPUT.PUT_LINE('✓ Registered export files: ' || vFileCount); DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('✓ Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB');
IF vFileCount = 0 THEN IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: No export files found in registration'); DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration');
ELSIF vFileCount < 9 THEN ELSIF vFileCount < 9 THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount); DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount);
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✓ All expected export files found'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found');
END IF; END IF;
END; END;
/ /
@@ -71,13 +71,13 @@ BEGIN
WHERE PROCESS_NAME = 'MARS-956' WHERE PROCESS_NAME = 'MARS-956'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR; AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
DBMS_OUTPUT.PUT_LINE('✓ Process log entries: ' || vLogCount); DBMS_OUTPUT.PUT_LINE('SUCCESS: Process log entries: ' || vLogCount);
DBMS_OUTPUT.PUT_LINE('✓ Error entries: ' || vErrorCount); DBMS_OUTPUT.PUT_LINE('SUCCESS: Error entries: ' || vErrorCount);
IF vErrorCount > 0 THEN IF vErrorCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vErrorCount || ' errors found in process log'); DBMS_OUTPUT.PUT_LINE('WARNING: ' || vErrorCount || ' errors found in process log');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✓ No errors found in process log'); DBMS_OUTPUT.PUT_LINE('SUCCESS: No errors found in process log');
END IF; END IF;
END; END;
/ /
@@ -124,15 +124,15 @@ BEGIN
END IF; END IF;
END LOOP; END LOOP;
DBMS_OUTPUT.PUT_LINE('✓ Cloud bucket files found: ' || vCloudFileCount); DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount);
IF vCloudFileCount = 0 THEN IF vCloudFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: No files found in cloud bucket'); DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: Cannot access cloud bucket: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM);
END; END;
END; END;
/ /
@@ -178,9 +178,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount); DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount);
IF vOverallStatus = 'SUCCESS' THEN IF vOverallStatus = 'SUCCESS' THEN
DBMS_OUTPUT.PUT_LINE('✓ All validations passed - export successful'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Some issues detected - review logs'); DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs');
END IF; END IF;
END; END;
/ /

194
MARS_Packages/REL02_POST/MARS-956/03_MARS_956_verify_data_integrity.sql
View File

@@ -21,9 +21,9 @@ DECLARE
vTotalRows NUMBER := 0; vTotalRows NUMBER := 0;
vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average
BEGIN BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN' INTO vAdminRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT' INTO vContentRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows; EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
vTotalRows := vAdminRows + vContentRows + vCriterionRows; vTotalRows := vAdminRows + vContentRows + vCriterionRows;
@@ -34,9 +34,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records'); DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records');
IF vTotalRows > 0 THEN IF vTotalRows > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All source tables contain data'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ No data found in source tables'); DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables');
END IF; END IF;
END; END;
/ /
@@ -49,18 +49,18 @@ DECLARE
vCriterionKeys NUMBER := 0; vCriterionKeys NUMBER := 0;
vTotalKeys NUMBER := 0; vTotalKeys NUMBER := 0;
BEGIN BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_ADMIN' INTO vAdminKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT' INTO vContentKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
SELECT COUNT(DISTINCT etl_key) SELECT COUNT(DISTINCT etl_key)
INTO vTotalKeys INTO vTotalKeys
FROM ( FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
); );
DBMS_OUTPUT.PUT_LINE('ETL key distribution:'); DBMS_OUTPUT.PUT_LINE('ETL key distribution:');
@@ -70,9 +70,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys); DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys);
IF vTotalKeys > 0 THEN IF vTotalKeys > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ ETL key distribution looks normal'); DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ No ETL keys found in source data'); DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data');
END IF; END IF;
END; END;
/ /
@@ -111,9 +111,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns'); DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns');
IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All template tables have defined structure'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ One or more template tables missing columns'); DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
END IF; END IF;
END; END;
/ /
@@ -128,22 +128,22 @@ BEGIN
SELECT COUNT(DISTINCT etl_key) SELECT COUNT(DISTINCT etl_key)
INTO vTotalSourceKeys INTO vTotalSourceKeys
FROM ( FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
); );
-- Count how many exist in A_LOAD_HISTORY -- Count how many exist in A_LOAD_HISTORY
SELECT COUNT(DISTINCT etl_key) SELECT COUNT(DISTINCT etl_key)
INTO vValidKeys INTO vValidKeys
FROM ( FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_C2D.MPEC_ADMIN SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_C2D.MPEC_CONTENT_CRITERION SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
) src ) src
WHERE EXISTS ( WHERE EXISTS (
SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h
@@ -155,16 +155,18 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys); DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys);
IF vValidKeys = vTotalSourceKeys THEN IF vValidKeys = vTotalSourceKeys THEN
DBMS_OUTPUT.PUT_LINE('✓ All source ETL keys are valid'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✗ Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys)); DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys));
END IF; END IF;
END; END;
/ /
PROMPT ========================================================================= PROMPT =====================================================================================
PROMPT Data Integrity Verification Summary: Source vs Target Record Counts PROMPT MARS-956 Record Count Verification
PROMPT ========================================================================= PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT =====================================================================================
DECLARE DECLARE
TYPE t_table_info IS RECORD ( TYPE t_table_info IS RECORD (
@@ -190,13 +192,13 @@ BEGIN
-- Initialize table list with C2D MPEC configuration -- Initialize table list with C2D MPEC configuration
vTables := t_table_list( vTables := t_table_list(
t_table_info('OU_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'), t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'),
t_table_info('OU_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'), t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'),
t_table_info('OU_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)') t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)')
); );
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status Difference'); DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP FOR i IN 1..vTables.COUNT LOOP
@@ -209,7 +211,7 @@ BEGIN
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
vSourceCount := -1; vSourceCount := -1;
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || 'ERROR: Cannot access source table'); DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table');
CONTINUE; CONTINUE;
END; END;
@@ -224,39 +226,65 @@ BEGIN
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout -- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error -- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket) -- KUP-13023: nothing matched wildcard query (no files in bucket)
IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' OR SQLERRM LIKE '%does not exist%' THEN -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vTargetCount := 0; -- Treat as empty (no files exported yet) vTargetCount := 0; -- Treat as empty (no files exported yet)
ELSE ELSE
vTargetCount := -1; -- Real error vTargetCount := -1; -- Real error
END IF; END IF;
END; END;
-- Display comparison results -- Display comparison results with thousands separators
IF vSourceCount = -1 OR vTargetCount = -1 THEN DECLARE
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || vStatus VARCHAR2(20);
RPAD(CASE WHEN vSourceCount = -1 THEN 'ERROR' ELSE TO_CHAR(vSourceCount) END, 14) || vSourceDisplay VARCHAR2(17);
RPAD(CASE WHEN vTargetCount = -1 THEN 'ERROR' ELSE TO_CHAR(vTargetCount) END, 14) || vTargetDisplay VARCHAR2(17);
'ERROR ' || 'N/A'); BEGIN
ELSIF vSourceCount = vTargetCount THEN -- Format source count display
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || IF vSourceCount = -1 THEN
RPAD(TO_CHAR(vSourceCount), 14) || vSourceDisplay := 'ERROR';
RPAD(TO_CHAR(vTargetCount), 14) || ELSE
'MATCH ' || '0'); vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999');
ELSE END IF;
vMismatchCount := vMismatchCount + 1;
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 28) || -- Format target count display
RPAD(TO_CHAR(vSourceCount), 14) || IF vTargetCount = -1 THEN
RPAD(TO_CHAR(vTargetCount), 14) || vTargetDisplay := 'ERROR';
'MISMATCH ' || TO_CHAR(vTargetCount - vSourceCount)); ELSE
END IF; vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999');
END IF;
-- Determine status
IF vSourceCount = vTargetCount THEN
vStatus := 'PASS';
ELSIF vTargetCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSIF vSourceCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(vSourceDisplay, 15) ||
LPAD(vTargetDisplay, 15) || ' ' ||
vStatus
);
END;
END LOOP; END LOOP;
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE(RPAD('TOTALS:', 28) || DBMS_OUTPUT.PUT_LINE(
RPAD(TO_CHAR(vTotalSourceCount), 14) || RPAD('TOTALS', 24) ||
RPAD(TO_CHAR(vTotalTargetCount), 14) || LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
CASE WHEN vTotalSourceCount = vTotalTargetCount THEN 'MATCH' ELSE 'MISMATCH' END || ' ' || LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15)
TO_CHAR(vTotalTargetCount - vTotalSourceCount)); );
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
-- Count exported files for additional verification -- Count exported files for additional verification
SELECT COUNT(*) SELECT COUNT(*)
@@ -265,33 +293,37 @@ BEGIN
WHERE RECEPTION_DATE >= SYSDATE - 1/24 WHERE RECEPTION_DATE >= SYSDATE - 1/24
AND (SOURCE_FILE_NAME LIKE '200_%'); AND (SOURCE_FILE_NAME LIKE '200_%');
-- Determine overall validation result DBMS_OUTPUT.PUT_LINE('=====================================================================================');
IF vTotalSourceCount = vTotalTargetCount AND vMismatchCount = 0 AND vFileCount > 0 THEN DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
vValidationResult := 'SUCCESS'; DBMS_OUTPUT.PUT_LINE('=====================================================================================');
ELSIF vTotalTargetCount = 0 AND vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
vValidationResult := 'NO EXPORT DETECTED'; DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)');
ELSIF vMismatchCount > 0 THEN DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount);
vValidationResult := 'DATA MISMATCH';
ELSE
vValidationResult := 'PARTIAL SUCCESS';
END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('MARS-956 Record Count Verification: ' || vValidationResult);
DBMS_OUTPUT.PUT_LINE('- Source records: ' || vTotalSourceCount);
DBMS_OUTPUT.PUT_LINE('- Target records: ' || vTotalTargetCount);
DBMS_OUTPUT.PUT_LINE('- Export files: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('- Mismatched tables: ' || vMismatchCount);
IF vValidationResult = 'SUCCESS' THEN IF vMismatchCount = 0 AND vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All record counts match - export successful'); DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED');
ELSIF vValidationResult = 'NO EXPORT DETECTED' THEN DBMS_OUTPUT.PUT_LINE(' All record counts match between source and exported data');
DBMS_OUTPUT.PUT_LINE('⚠ No export detected - check export execution'); DBMS_OUTPUT.PUT_LINE(' Export completed successfully');
ELSIF vValidationResult = 'DATA MISMATCH' THEN ELSIF vMismatchCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('✗ Record count mismatches found - investigate data integrity'); DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES');
DBMS_OUTPUT.PUT_LINE(' Found ' || vMismatchCount || ' table(s) with count mismatches');
DBMS_OUTPUT.PUT_LINE(' NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)');
DBMS_OUTPUT.PUT_LINE(' Review export logs and pre-check results before re-running exports');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Partial success - review individual table results'); DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED');
DBMS_OUTPUT.PUT_LINE(' No files found in export registration');
DBMS_OUTPUT.PUT_LINE(' Verify export execution completed successfully');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Legend:');
DBMS_OUTPUT.PUT_LINE(' PASS - Record counts match (export successful)');
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Additional ETL key analysis for C2D MPEC data -- Additional ETL key analysis for C2D MPEC data
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');
@@ -302,9 +334,9 @@ BEGIN
vContentKeys NUMBER; vContentKeys NUMBER;
vCriterionKeys NUMBER; vCriterionKeys NUMBER;
BEGIN BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_ADMIN' INTO vAdminKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT' INTO vContentKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys; EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)'); DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)');
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)'); DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)');

169
MARS_Packages/REL02_POST/MARS-956/90_MARS_956_rollback_delete_csv_files.sql Normal file
View File

@@ -0,0 +1,169 @@
--=============================================================================================================================
-- MARS-956 ROLLBACK: Delete Exported CSV Files from DATA Bucket
--=============================================================================================================================
-- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables
-- WARNING: This will permanently delete exported data files!
-- Author: System Generated
-- Date: 2026-02-12
-- Related: MARS-956 - C2D MPEC Data Export Rollback
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...');
-- Delete CSV files from DATA bucket (only files matching export pattern)
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/'
))
WHERE object_name LIKE '%.csv'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...');
-- Delete CSV files from DATA bucket (only files matching export pattern)
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/'
))
WHERE object_name LIKE '%.csv'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...');
-- Delete CSV files from DATA bucket (only files matching export pattern)
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/'
))
WHERE object_name LIKE '%.csv'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT SUCCESS: All CSV file deletion operations completed

11
MARS_Packages/REL02_POST/MARS-956/91_MARS_956_rollback_file_registrations.sql
View File

@@ -15,6 +15,7 @@ PROMPT =========================================================================
DECLARE DECLARE
vFileCount NUMBER := 0; vFileCount NUMBER := 0;
vDeletedCount NUMBER := 0; vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN BEGIN
-- Count files to be deleted -- Count files to be deleted
SELECT COUNT(*) SELECT COUNT(*)
@@ -48,7 +49,7 @@ BEGIN
vDeletedCount := SQL%ROWCOUNT; vDeletedCount := SQL%ROWCOUNT;
COMMIT; COMMIT;
DBMS_OUTPUT.PUT_LINE('✓ Successfully deleted ' || vDeletedCount || ' file registrations'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' file registrations');
-- Log the rollback action -- Log the rollback action
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
@@ -57,17 +58,17 @@ BEGIN
COMMIT; COMMIT;
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✓ No file registrations found to delete'); DBMS_OUTPUT.PUT_LINE('SUCCESS: No file registrations found to delete');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
ROLLBACK; ROLLBACK;
DBMS_OUTPUT.PUT_LINE('✗ Error during file registration deletion: ' || SQLERRM); vErrorMsg := 'Failed to delete file registrations: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during file registration deletion: ' || SQLERRM);
-- Log the error -- Log the error
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-956-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', VALUES ('MARS-956-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', vErrorMsg);
'Failed to delete file registrations: ' || SQLERRM);
COMMIT; COMMIT;
RAISE; RAISE;
END; END;

11
MARS_Packages/REL02_POST/MARS-956/92_MARS_956_rollback_process_logs.sql
View File

@@ -15,6 +15,7 @@ PROMPT =========================================================================
DECLARE DECLARE
vLogCount NUMBER := 0; vLogCount NUMBER := 0;
vDeletedCount NUMBER := 0; vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN BEGIN
-- Count logs to be deleted -- Count logs to be deleted
SELECT COUNT(*) SELECT COUNT(*)
@@ -52,20 +53,20 @@ BEGIN
vDeletedCount := SQL%ROWCOUNT; vDeletedCount := SQL%ROWCOUNT;
COMMIT; COMMIT;
DBMS_OUTPUT.PUT_LINE('✓ Successfully deleted ' || vDeletedCount || ' process log entries'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' process log entries');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('✓ No process log entries found to delete'); DBMS_OUTPUT.PUT_LINE('SUCCESS: No process log entries found to delete');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
ROLLBACK; ROLLBACK;
DBMS_OUTPUT.PUT_LINE('✗ Error during process log cleanup: ' || SQLERRM); vErrorMsg := 'Failed to clean process logs: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during process log cleanup: ' || SQLERRM);
-- Log the error (will remain after rollback for debugging) -- Log the error (will remain after rollback for debugging)
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE) INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-956-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', VALUES ('MARS-956-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', vErrorMsg);
'Failed to clean process logs: ' || SQLERRM);
COMMIT; COMMIT;
RAISE; RAISE;
END; END;

22
MARS_Packages/REL02_POST/MARS-956/99_MARS_956_verify_rollback.sql
View File

@@ -24,9 +24,9 @@ BEGIN
AND RECEPTION_DATE >= SYSDATE - 7; -- Last week AND RECEPTION_DATE >= SYSDATE - 7; -- Last week
IF vRemainingFiles = 0 THEN IF vRemainingFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All MARS-956 file registrations successfully removed'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-956 file registrations successfully removed');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vRemainingFiles || ' file registrations still exist'); DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingFiles || ' file registrations still exist');
-- Show remaining files -- Show remaining files
FOR rec IN ( FOR rec IN (
@@ -54,9 +54,9 @@ BEGIN
AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week
IF vRemainingLogs = 0 THEN IF vRemainingLogs = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ All MARS-956 process logs successfully removed'); DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-956 process logs successfully removed');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Warning: ' || vRemainingLogs || ' process log entries still exist'); DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingLogs || ' process log entries still exist');
-- Show remaining logs (first few) -- Show remaining logs (first few)
FOR rec IN ( FOR rec IN (
@@ -105,16 +105,16 @@ BEGIN
END LOOP; END LOOP;
IF vCloudFileCount = 0 THEN IF vCloudFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ No C2D MPEC files found in cloud bucket'); DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('ℹ Info: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket'); DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket');
DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback'); DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback');
DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed'); DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed');
END IF; END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('⚠ Warning: Cannot check cloud bucket: ' || SQLERRM); DBMS_OUTPUT.PUT_LINE('WARNING: Cannot check cloud bucket: ' || SQLERRM);
END; END;
END; END;
/ /
@@ -131,7 +131,7 @@ BEGIN
AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour
IF vRollbackLogs > 0 THEN IF vRollbackLogs > 0 THEN
DBMS_OUTPUT.PUT_LINE('✓ Rollback operation logs found: ' || vRollbackLogs); DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback operation logs found: ' || vRollbackLogs);
-- Show recent rollback logs -- Show recent rollback logs
FOR rec IN ( FOR rec IN (
@@ -148,7 +148,7 @@ BEGIN
rec.PROCEDURE_NAME || ': ' || rec.MESSAGE); rec.PROCEDURE_NAME || ': ' || rec.MESSAGE);
END LOOP; END LOOP;
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Warning: No rollback operation logs found'); DBMS_OUTPUT.PUT_LINE('WARNING: Warning: No rollback operation logs found');
END IF; END IF;
END; END;
/ /
@@ -191,9 +191,9 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs); DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs);
IF vRollbackStatus = 'COMPLETE' THEN IF vRollbackStatus = 'COMPLETE' THEN
DBMS_OUTPUT.PUT_LINE('✓ Rollback completed successfully - system clean'); DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback completed successfully - system clean');
ELSE ELSE
DBMS_OUTPUT.PUT_LINE('⚠ Rollback incomplete - manual cleanup may be required'); DBMS_OUTPUT.PUT_LINE('WARNING: Rollback incomplete - manual cleanup may be required');
END IF; END IF;
DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('');

296
MARS_Packages/REL02_POST/MARS-956/README.md
View File

@@ -1,296 +0,0 @@
# MARS-956: One-Time C2D MPEC Data Export from Operational Database to External Tables
## Overview
This package performs a one-time bulk export of C2D MPEC data from operational database tables (OU_C2D schema) to new external tables in OCI buckets. The export uses DATA_EXPORTER v2.7.5 with pRegisterExport parameter to move historical data to ODS bucket in CSV format with automatic file registration.
**Migration Strategy:**
- **Source**: OU_C2D schema tables (MPEC_ADMIN, MPEC_CONTENT, MPEC_CONTENT_CRITERION)
- **Target**: ODS bucket (CSV format)
- **Registration**: Files automatically registered in A_SOURCE_FILE_RECEIVED
- **Template Compatibility**: Uses CT_ET_TEMPLATES.C2D_MPEC_* for column ordering
**Key Features:**
- Automatic file registration with pRegisterExport=TRUE
- Template table column order compliance
- Comprehensive verification and rollback capabilities
- ODS/C2D bucket path structure
## Contents
- `install_mars956.sql` - Master installation script with SPOOL logging
- `rollback_mars956.sql` - Master rollback script
- `00_MARS_956_*.sql` - Prerequisites check
- `01_MARS_956_*.sql` - Data export script
- `02_MARS_956_*.sql` - Export verification
- `03_MARS_956_*.sql` - Data integrity validation
- `91_MARS_956_*.sql` - Rollback scripts
- `99_MARS_956_*.sql` - Rollback verification
## Prerequisites
- Oracle Database 23ai with DBMS_CLOUD
- ADMIN user access (required for all MARS installations)
- DATA_EXPORTER package v2.7.5 or higher
- OU_C2D schema with populated MPEC tables
- CT_ET_TEMPLATES schema with C2D_MPEC_* template tables
- OCI bucket access credentials configured
## Data Structure
### Source Tables (OU_C2D schema)
- **MPEC_ADMIN**: 5 records with ETL keys 2001-2005
- **MPEC_CONTENT**: 5 records with ETL keys 2006-2008
- **MPEC_CONTENT_CRITERION**: 9 records with ETL keys 2009-2010
### Template Tables (CT_ET_TEMPLATES schema)
- **C2D_MPEC_ADMIN**: 10 columns
- **C2D_MPEC_CONTENT**: 87 columns
- **C2D_MPEC_CONTENT_CRITERION**: 4 columns
### Target Structure (OCI ODS bucket)
```
mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/*.csv
mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv
mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv
```
## Enhanced Validation Framework
### Pre-Export Validation (Check 1-5)
1. **Source Table Analysis**: Comprehensive source table record counting with business rule validation
2. **Template Table Verification**: Ensures external table templates exist and are accessible
3. **ETL Key Validation**: Verifies integrity of load history tracking keys (expected: 2001-2010)
4. **Schema Permissions**: Validates all required schema access permissions
5. **External Table Readiness**: Checks accessibility of target external tables (ODS.C2D_MPEC_*_ODS) with proper handling of empty states
### In-Export Validation
- **Pre-Export Record Count**: Source and target table counts before export begins
- **Live Progress Monitoring**: Real-time feedback during export operations
- **Post-Export Verification**: Immediate source vs target COUNT(*) comparison table
- **Error Handling**: Comprehensive error capture with detailed diagnostic information
### Post-Export Validation
- **Record Count Verification**: Detailed source vs target COUNT(*) comparison for all exported tables
- **File Registration Validation**: Confirms all exported files are properly registered in A_SOURCE_FILE_RECEIVED
- **Data Integrity Analysis**: Validates data structure and completeness using external table access
- **Delta Query Readiness**: Confirms external tables ready for operational delta queries
mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/*.csv
mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/*.csv
```
## Installation
## Installation
### Prerequisites
**IMPORTANT**: External tables must already exist on the database (created via FILE_MANAGER procedures)
Required external tables in ODS schema:
- `ODS.C2D_MPEC_ADMIN_ODS`
- `ODS.C2D_MPEC_CONTENT_ODS`
- `ODS.C2D_MPEC_CONTENT_CRITERION_ODS`
If external tables don't exist, use `FILE_MANAGER.CREATE_EXTERNAL_TABLE` or `ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE` procedures to create them.
### Installation Command
```powershell
# Run as ADMIN user
cd c:\_git\MARS\wkcopy02\MARS_Packages\REL02_POST\MARS-956-NEW
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars956.sql"
# Log file created: log/INSTALL_MARS_956_<PDB>_<timestamp>.log
```
### Alternative: Individual Scripts
```powershell
# Execute individual scripts for step-by-step validation
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@00_MARS_956_pre_check_prerequisites.sql"
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@01_MARS_956_export_c2d_mpec_data.sql"
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@02_MARS_956_verify_exports.sql"
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@03_MARS_956_verify_data_integrity.sql"
```
## Expected Results
### File Registration
- **9 CSV files** registered in A_SOURCE_FILE_RECEIVED
- **File naming pattern**: {ETL_KEY}_1_{TIMESTAMP}Z.csv
- **Processing status**: INGESTED
- **Total size**: ~0.33 KB
### Export Distribution
- **MPEC_ADMIN**: 3 files (ETL keys 2001-2005, 3 distinct keys)
- **MPEC_CONTENT**: 3 files (ETL keys 2006-2008, 3 distinct keys)
- **MPEC_CONTENT_CRITERION**: 3 files (ETL keys 2009-2010, 3 distinct keys)
### Duration
- **Expected**: 2-5 minutes (small datasets)
- **Actual**: ~16 seconds (based on testing)
## Verification
### Manual Verification
```sql
-- Check export results
SELECT COUNT(*) AS TOTAL_FILES FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE RECEPTION_DATE >= SYSDATE - 1/24 AND SOURCE_FILE_NAME LIKE '200%';
-- Check process logs
SELECT PROCESS_NAME, LOG_LEVEL, LOG_MESSAGE FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-956' ORDER BY LOG_TIMESTAMP DESC;
```
### Cloud Bucket Verification
```sql
-- List exported files in OCI bucket
SELECT object_name FROM TABLE(MRDS_LOADER.cloud_wrapper.list_objects(
credential_name => 'DEF_CRED_ARN',
location_uri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/data/'
)) WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%';
```
## Rollback
### Full Rollback (if needed)
```powershell
cd c:\_git\MARS\wkcopy02\MARS_Packages\REL02_POST\MARS-956-NEW
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars956.sql"
```
### Manual Rollback
```sql
-- Remove file registrations
DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE SOURCE_FILE_NAME LIKE '200%' AND RECEPTION_DATE >= SYSDATE - 7;
-- Remove process logs
DELETE FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-956', 'MARS-956-ROLLBACK') AND LOG_TIMESTAMP >= SYSDATE - 7;
```
## Troubleshooting
### Common Issues
1. **ORA-17008: Closed connection**
- Use MCP SQLcl server instead of traditional SQLcl
- Connection: `CT_MRDS@ggmichalski_high`
2. **No files registered**
- Check DATA_EXPORTER package version (requires v2.7.5+)
- Verify pRegisterExport=TRUE parameter
3. **Template table errors**
- Ensure CT_ET_TEMPLATES.C2D_MPEC_* tables exist
- Check column compatibility with source tables
4. **ETL key validation errors**
- Verify A_ETL_LOAD_SET_FK values exist in CT_ODS.A_LOAD_HISTORY
- Check ETL key range: 2001-2010
### Error Resolution
- Check `log/INSTALL_MARS_956_*.log` for detailed error information
- Review A_PROCESS_LOG table for runtime errors
- Use verification scripts to diagnose specific issues
### Common Validation Issues
#### External Table Access Errors
**Problem**: "✗ ERROR: Cannot access C2D_MPEC_*_ODS tables"
- **Cause**: Missing external table definitions or incorrect bucket paths
- **Solution**: Verify external tables exist and point to correct OCI bucket paths
- **Check**: `SELECT * FROM ODS.C2D_MPEC_ADMIN_ODS WHERE ROWNUM <= 1;`
- **Create**: Use `ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE` if tables don't exist
#### Record Count Mismatches
**Problem**: Source vs target COUNT(*) differences in post-export validation
- **Cause**: Export failure, external table configuration, or bucket synchronization lag
- **Solution**: Wait 30-60 seconds and re-run verification script
- **Check**: Run `@03_MARS_956_verify_data_integrity.sql` again
#### Empty Source Tables
**Problem**: "⚠ WARNING: No source data found"
- **Cause**: Source tables not populated or incorrect schema permissions
- **Solution**: Verify OU_C2D schema contains expected MPEC data
- **Check**: `SELECT COUNT(*) FROM OU_C2D.MPEC_ADMIN;` (should return 5)
#### ETL Key Validation Failures
**Problem**: ETL keys outside expected range 2001-2010
- **Cause**: Data inconsistency or load history corruption
- **Solution**: Review source data quality and CT_ODS.A_LOAD_HISTORY entries
- **Check**: Verify ETL key distribution in source tables
#### External Table Empty State Handling
**Problem**: KUP-13023 or ORA-29913 errors during validation
- **Expected**: These errors are normal for empty external tables
- **Handling**: Pre-checks properly handle empty states as normal condition
- **Action**: No action required if source tables contain data
## Dependencies
### External Tables Setup (If Not Already Created)
If external tables don't exist, create them using FILE_MANAGER procedures:
```sql
-- Option 1: Using ODS.FILE_MANAGER_ODS (recommended - AUTHID DEFINER)
BEGIN
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
pTableName => 'C2D_MPEC_ADMIN_ODS',
pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_ADMIN',
pPrefix => 'ODS/C2D/C2D_MPEC_ADMIN',
pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri
);
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
pTableName => 'C2D_MPEC_CONTENT_ODS',
pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT',
pPrefix => 'ODS/C2D/C2D_MPEC_CONTENT',
pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri
);
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
pTableName => 'C2D_MPEC_CONTENT_CRITERION_ODS',
pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT_CRITERION',
pPrefix => 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION',
pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri
);
END;
/
-- Option 2: Using CT_MRDS.FILE_MANAGER (AUTHID CURRENT_USER - requires ODS context)
-- Connect as ODS user first
-- Then call CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE with same parameters
```
### Package Dependencies
- **CT_MRDS.DATA_EXPORTER** v2.7.5+ (with pRegisterExport support)
- **CT_MRDS.ENV_MANAGER** v3.1.0+
- **MRDS_LOADER.cloud_wrapper** (for OCI bucket access)
### Schema Dependencies
- **OU_C2D**: Source data schema
- **CT_ET_TEMPLATES**: Template table definitions
- **CT_MRDS**: Process logging and configuration
- **CT_ODS**: ETL load history validation
### Infrastructure Dependencies
- **OCI Object Storage**: ODS bucket access
- **Database credentials**: DEF_CRED_ARN configured
- **Network connectivity**: Frankfurt region access
## Related MARS Issues
- **MARS-956**: C2D MPEC Historical Data Export (this package)
- **MARS-835**: CSDB Data Export (reference implementation)
- **Related**: DATA_EXPORTER package enhancements
## Support
For technical support or issues:
1. Check installation log files in `log/` directory
2. Review verification script outputs
3. Contact Database Team with specific error messages
4. Include A_PROCESS_LOG entries for debugging
---
**Package Version**: 1.0.0
**Created**: 2026-02-12
**Author**: System Generated (based on MARS-835 structure)
**Status**: Ready for deployment

10
MARS_Packages/REL02_POST/MARS-956/install_mars956.sql
View File

@@ -1,7 +1,7 @@
-- =================================================================== -- ===================================================================
-- MARS-956 INSTALL SCRIPT: C2D MPEC Data Export to External Tables -- MARS-956 INSTALL SCRIPT: C2D MPEC Data Export to External Tables
-- =================================================================== -- ===================================================================
-- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_C2D schema -- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema
-- to OCI buckets (ODS bucket CSV format) -- to OCI buckets (ODS bucket CSV format)
-- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration -- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration
-- Author: System Generated -- Author: System Generated
@@ -37,16 +37,14 @@ PROMPT
PROMPT This script will export 3 C2D MPEC tables to OCI buckets: PROMPT This script will export 3 C2D MPEC tables to OCI buckets:
PROMPT PROMPT
PROMPT TARGET: ODS Bucket (CSV format): PROMPT TARGET: ODS Bucket (CSV format):
PROMPT - MPEC_ADMIN: 5 records with ETL keys 2001-2005 PROMPT - MPEC_ADMIN
PROMPT - MPEC_CONTENT: 5 records with ETL keys 2006-2008 PROMPT - MPEC_CONTENT
PROMPT - MPEC_CONTENT_CRITERION: 9 records with ETL keys 2009-2010 PROMPT - MPEC_CONTENT_CRITERION
PROMPT PROMPT
PROMPT Key Features: PROMPT Key Features:
PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking
PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*) PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*)
PROMPT - ODS/C2D bucket path structure PROMPT - ODS/C2D bucket path structure
PROMPT
PROMPT Expected Duration: 2-5 minutes (small datasets)
PROMPT ========================================================================= PROMPT =========================================================================
-- Confirm installation with user -- Confirm installation with user

12
MARS_Packages/REL02_POST/MARS-956/rollback_mars956.sql
View File

@@ -47,19 +47,25 @@ WHENEVER SQLERROR CONTINUE
PROMPT PROMPT
PROMPT ========================================================================= PROMPT =========================================================================
PROMPT Step 1: Delete File Registrations PROMPT Step 1: Delete Exported CSV Files from DATA Bucket
PROMPT =========================================================================
@@90_MARS_956_rollback_delete_csv_files.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 2: Delete File Registrations
PROMPT ========================================================================= PROMPT =========================================================================
@@91_MARS_956_rollback_file_registrations.sql @@91_MARS_956_rollback_file_registrations.sql
PROMPT PROMPT
PROMPT ========================================================================= PROMPT =========================================================================
PROMPT Step 2: Clean Process Logs PROMPT Step 3: Clean Process Logs
PROMPT ========================================================================= PROMPT =========================================================================
@@92_MARS_956_rollback_process_logs.sql @@92_MARS_956_rollback_process_logs.sql
PROMPT PROMPT
PROMPT ========================================================================= PROMPT =========================================================================
PROMPT Step 3: Verify Rollback Completion PROMPT Step 4: Verify Rollback Completion
PROMPT ========================================================================= PROMPT =========================================================================
@@99_MARS_956_verify_rollback.sql @@99_MARS_956_verify_rollback.sql