admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MARS-1005

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-06 12:06:18 +01:00
parent 113ea0a618
commit 73e99b6e76
10 changed files with 1822 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
MARS_Packages/REL03/MARS-1005/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
# Exclude temporary folders from version control
confluence/
log/
test/
mock_data/

575
MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql Normal file
View File

@@ -0,0 +1,575 @@
-- =====================================================================================
-- Script: 01_MARS_1005_export_top_data.sql
-- Purpose: Export OU_TOP historical data to ODS bucket (DATA bucket, CSV format)
-- Author: Grzegorz Michalski
-- Created: 2026-03-06
-- MARS Issue: MARS-1005
-- Target: mrds_data_dev/ODS/TOP/
-- Tables:
-- 1. OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
-- 2. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
-- 3. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
-- 4. OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
-- 5. OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
-- 6. OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
SET TIMING ON;
PROMPT =====================================================================================
PROMPT MARS-1005: OU_TOP Historical Data Export
PROMPT =====================================================================================
PROMPT Export Strategy:
PROMPT - Source: OU_TOP schema tables (operational database)
PROMPT - Target: DATA/ODS bucket as CSV files
PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA
PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED
PROMPT - Path Structure: ODS/TOP/TOP_*/
PROMPT Tables (6):
PROMPT ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM,
PROMPT ANNOUNCEMENT, FBL_ITEM (->TOP_FULLBIDLIST_ITEM), FULLBID_ARRAY_COMPILED
PROMPT =====================================================================================
-- Log export start
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Starting historical OU_TOP data export',
'Tables: ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM, ANNOUNCEMENT, FBL_ITEM, FULLBID_ARRAY_COMPILED');
PROMPT
PROMPT =====================================================================================
PROMPT PRE-EXPORT CHECK: Verify Existing Files in ODS Bucket
PROMPT =====================================================================================
-- Helper procedure (inline) to check one folder
-- Check 1: ALLOTMENT
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT files found - bucket is clean');
END IF;
END;
/
-- Check 2: ALLOTMENT_MODIFICATION_HEADER
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_HEADER files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_HEADER files found - bucket is clean');
END IF;
END;
/
-- Check 3: ALLOTMENT_MODIFICATION_ITEM
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_ITEM files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_ITEM files found - bucket is clean');
END IF;
END;
/
-- Check 4: ANNOUNCEMENT
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ANNOUNCEMENT/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ANNOUNCEMENT files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ANNOUNCEMENT files found - bucket is clean');
END IF;
END;
/
-- Check 5: FBL_ITEM (folder: TOP_FULLBIDLIST_ITEM)
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBIDLIST_ITEM/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBIDLIST_ITEM files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBIDLIST_ITEM files found - bucket is clean');
END IF;
END;
/
-- Check 6: FULLBID_ARRAY_COMPILED
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBID_ARRAY_COMPILED files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
END LOOP;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBID_ARRAY_COMPILED files found - bucket is clean');
END IF;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT PRE-EXPORT: Verify Source and Target Table Readiness
PROMPT =====================================================================================
DECLARE
v1Source NUMBER := 0; v2Source NUMBER := 0; v3Source NUMBER := 0;
v4Source NUMBER := 0; v5Source NUMBER := 0; v6Source NUMBER := 0;
v1Target NUMBER := 0; v2Target NUMBER := 0; v3Target NUMBER := 0;
v4Target NUMBER := 0; v5Target NUMBER := 0; v6Target NUMBER := 0;
vTotalSource NUMBER := 0;
vTotalTarget NUMBER := 0;
PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS
BEGIN
EXECUTE IMMEDIATE pSql INTO pResult;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
pResult := 0;
ELSE
pResult := -1;
END IF;
END;
BEGIN
-- Source counts
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Source;
vTotalSource := v1Source + v2Source + v3Source + v4Source + v5Source + v6Source;
DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):');
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT: ' || v1Source);
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_HEADER: ' || v2Source);
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_ITEM: ' || v3Source);
DBMS_OUTPUT.PUT_LINE('- ANNOUNCEMENT: ' || v4Source);
DBMS_OUTPUT.PUT_LINE('- FBL_ITEM: ' || v5Source);
DBMS_OUTPUT.PUT_LINE('- FULLBID_ARRAY_COMPILED: ' || v6Source);
DBMS_OUTPUT.PUT_LINE('- TOTAL SOURCE: ' || vTotalSource);
-- Target external table counts
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6Target);
vTotalTarget := GREATEST(v1Target,0) + GREATEST(v2Target,0) + GREATEST(v3Target,0)
+ GREATEST(v4Target,0) + GREATEST(v5Target,0) + GREATEST(v6Target,0);
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Target external table record counts (pre-export):');
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_ODS: ' || CASE WHEN v1Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v1Target) END);
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_HEADER_ODS: ' || CASE WHEN v2Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v2Target) END);
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_ITEM_ODS: ' || CASE WHEN v3Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v3Target) END);
DBMS_OUTPUT.PUT_LINE('- TOP_ANNOUNCEMENT_ODS: ' || CASE WHEN v4Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v4Target) END);
DBMS_OUTPUT.PUT_LINE('- TOP_FULLBIDLIST_ITEM_ODS: ' || CASE WHEN v5Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v5Target) END);
DBMS_OUTPUT.PUT_LINE('- TOP_FULLBID_ARRAY_COMPILED_ODS: ' || CASE WHEN v6Target = -1 THEN 'ERROR/INACCESSIBLE' ELSE TO_CHAR(v6Target) END);
IF vTotalSource > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: No source data found');
END IF;
IF vTotalTarget = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export');
ELSIF vTotalTarget > 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access target external tables');
END IF;
DBMS_OUTPUT.PUT_LINE('Proceeding with export...');
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 1/6: OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 2/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_HEADER',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_HEADER export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_HEADER export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_HEADER', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 3/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_ITEM',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_ITEM export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_ITEM export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 4/6: OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ANNOUNCEMENT',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ANNOUNCEMENT',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ANNOUNCEMENT',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ANNOUNCEMENT export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ANNOUNCEMENT export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ANNOUNCEMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 5/6: OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_FBL_ITEM',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_FULLBIDLIST_ITEM',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: FBL_ITEM export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: FBL_ITEM export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_FBL_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 6/6: OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_FULLBID_ARRAY_COMPILED',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: FULLBID_ARRAY_COMPILED export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: FULLBID_ARRAY_COMPILED export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_FULLBID_ARRAY_COMPILED', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT Export Summary - Checking Results
PROMPT =====================================================================================
-- Log completion
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'All OU_TOP historical exports completed successfully');
PROMPT
PROMPT =====================================================================================
PROMPT MARS-1005 OU_TOP Export Completed!
PROMPT =====================================================================================
PROMPT POST-EXPORT: Source vs Target Record Count Comparison
PROMPT =====================================================================================
DECLARE
v1S NUMBER := 0; v2S NUMBER := 0; v3S NUMBER := 0;
v4S NUMBER := 0; v5S NUMBER := 0; v6S NUMBER := 0;
v1T NUMBER := 0; v2T NUMBER := 0; v3T NUMBER := 0;
v4T NUMBER := 0; v5T NUMBER := 0; v6T NUMBER := 0;
vTotalS NUMBER := 0;
vTotalT NUMBER := 0;
vMismatch NUMBER := 0;
PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS
BEGIN
EXECUTE IMMEDIATE pSql INTO pResult;
EXCEPTION WHEN OTHERS THEN pResult := -1;
END;
PROCEDURE print_row(pTable VARCHAR2, pSrc NUMBER, pTgt NUMBER) IS
BEGIN
DBMS_OUTPUT.PUT_LINE(
RPAD(pTable, 40) || ' | ' ||
RPAD(CASE WHEN pSrc = -1 THEN 'N/A' ELSE TO_CHAR(pSrc) END, 8) || ' | ' ||
RPAD(CASE WHEN pTgt = -1 THEN 'ERROR' ELSE TO_CHAR(pTgt) END, 8) || ' | ' ||
CASE WHEN pSrc = pTgt THEN 'OK' ELSE 'MISMATCH' END);
END;
BEGIN
-- Source
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6S;
vTotalS := v1S + v2S + v3S + v4S + v5S + v6S;
-- Target
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6T);
vTotalT := GREATEST(v1T,0) + GREATEST(v2T,0) + GREATEST(v3T,0)
+ GREATEST(v4T,0) + GREATEST(v5T,0) + GREATEST(v6T,0);
DBMS_OUTPUT.PUT_LINE('POST-EXPORT VERIFICATION SUMMARY');
DBMS_OUTPUT.PUT_LINE(RPAD('Table', 40) || ' | Source | Target | Match');
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
print_row('ALLOTMENT', v1S, v1T); IF v1S != v1T THEN vMismatch := vMismatch + 1; END IF;
print_row('ALLOTMENT_MODIFICATION_HEADER', v2S, v2T); IF v2S != v2T THEN vMismatch := vMismatch + 1; END IF;
print_row('ALLOTMENT_MODIFICATION_ITEM', v3S, v3T); IF v3S != v3T THEN vMismatch := vMismatch + 1; END IF;
print_row('ANNOUNCEMENT', v4S, v4T); IF v4S != v4T THEN vMismatch := vMismatch + 1; END IF;
print_row('FBL_ITEM (->FULLBIDLIST_ITEM)', v5S, v5T); IF v5S != v5T THEN vMismatch := vMismatch + 1; END IF;
print_row('FULLBID_ARRAY_COMPILED', v6S, v6T); IF v6S != v6T THEN vMismatch := vMismatch + 1; END IF;
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
print_row('TOTAL', vTotalS, vTotalT);
DBMS_OUTPUT.PUT_LINE('');
IF vMismatch = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All record counts match - export verified');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vMismatch || ' table(s) have record count mismatches');
DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions');
END IF;
END;
/
-- Log export completion
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Historical OU_TOP data export completed',
'Check verification scripts for detailed results');
COMMIT;
PROMPT
PROMPT =====================================================================================
PROMPT MARS-1005 OU_TOP Historical Data Export - COMPLETED
PROMPT
PROMPT Next steps:
PROMPT 1. Run: @02_MARS_1005_verify_exports.sql (verify file registration)
PROMPT 2. Run: @03_MARS_1005_verify_data_integrity.sql (full data verification)
PROMPT =====================================================================================

190
MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql Normal file
View File

@@ -0,0 +1,190 @@
-- ===================================================================
-- MARS-1005 Verify Exports: Check Export Results and File Creation
-- ===================================================================
-- Purpose: Verify that C2D MPEC export completed successfully
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Export Verification
PROMPT =========================================================================
-- Check 1: Verify files were registered in A_SOURCE_FILE_RECEIVED
PROMPT Checking export file registration...
DECLARE
vFileCount NUMBER := 0;
vTotalBytes NUMBER := 0;
BEGIN
SELECT COUNT(*), NVL(SUM(BYTES), 0)
INTO vFileCount, vTotalBytes
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour
AND (SOURCE_FILE_NAME LIKE '2001_%' -- MPEC_ADMIN ETL keys
OR SOURCE_FILE_NAME LIKE '2002_%'
OR SOURCE_FILE_NAME LIKE '2003_%'
OR SOURCE_FILE_NAME LIKE '2004_%'
OR SOURCE_FILE_NAME LIKE '2005_%'
OR SOURCE_FILE_NAME LIKE '2006_%' -- MPEC_CONTENT ETL keys
OR SOURCE_FILE_NAME LIKE '2007_%'
OR SOURCE_FILE_NAME LIKE '2008_%'
OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys
OR SOURCE_FILE_NAME LIKE '2010_%');
DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB');
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration');
ELSIF vFileCount < 9 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount);
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found');
END IF;
END;
/
-- Check 2: Show recent export registrations
PROMPT Recent export file registrations:
SELECT
SUBSTR(SOURCE_FILE_NAME, 1, 40) AS FILE_NAME,
A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY,
PROCESSING_STATUS,
ROUND(BYTES/1024, 2) AS SIZE_KB,
TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour
AND (SOURCE_FILE_NAME LIKE '200%') -- ETL keys starting with 200
ORDER BY RECEPTION_DATE DESC;
-- Check 3: Verify export process logs
PROMPT Checking export process logs...
DECLARE
vLogCount NUMBER := 0;
vErrorCount NUMBER := 0;
BEGIN
SELECT COUNT(*), SUM(CASE WHEN LOG_LEVEL = 'ERROR' THEN 1 ELSE 0 END)
INTO vLogCount, vErrorCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Process log entries: ' || vLogCount);
DBMS_OUTPUT.PUT_LINE('SUCCESS: Error entries: ' || vErrorCount);
IF vErrorCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vErrorCount || ' errors found in process log');
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No errors found in process log');
END IF;
END;
/
-- Check 4: Display recent process logs
PROMPT Recent MARS-1005 process logs:
SELECT
TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 60) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 10 ROWS ONLY;
-- Check 5: Cloud bucket file verification (if cloud_wrapper available)
PROMPT Checking cloud bucket files...
DECLARE
vCloudFileCount NUMBER := 0;
vCredentialName VARCHAR2(100);
vDataBucketUri VARCHAR2(500);
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri);
-- Count files in cloud bucket
BEGIN
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri
))
WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%'
) LOOP
vCloudFileCount := vCloudFileCount + 1;
IF vCloudFileCount <= 5 THEN -- Show first 5 files
DBMS_OUTPUT.PUT_LINE('- ' || rec.object_name);
END IF;
END LOOP;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount);
IF vCloudFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM);
END;
END;
/
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Export Verification Summary
PROMPT =========================================================================
-- Final verification summary
DECLARE
vFileRegCount NUMBER := 0;
vCloudFileCount NUMBER := 0;
vLogErrorCount NUMBER := 0;
vOverallStatus VARCHAR2(20);
BEGIN
-- Count registered files
SELECT COUNT(*)
INTO vFileRegCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE RECEPTION_DATE >= SYSDATE - 1/24
AND SOURCE_FILE_NAME LIKE '200%';
-- Count process errors
SELECT COUNT(*)
INTO vLogErrorCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_LEVEL = 'ERROR'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
-- Determine overall status
IF vFileRegCount >= 9 AND vLogErrorCount = 0 THEN
vOverallStatus := 'SUCCESS';
ELSIF vFileRegCount > 0 AND vLogErrorCount = 0 THEN
vOverallStatus := 'PARTIAL SUCCESS';
ELSE
vOverallStatus := 'ISSUES DETECTED';
END IF;
DBMS_OUTPUT.PUT_LINE('MARS-1005 Export Verification: ' || vOverallStatus);
DBMS_OUTPUT.PUT_LINE('- Registered files: ' || vFileRegCount || ' (expected: 9)');
DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount);
IF vOverallStatus = 'SUCCESS' THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs');
END IF;
END;
/
PROMPT =========================================================================
PROMPT Export Verification Completed
PROMPT =========================================================================

354
MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql Normal file
View File

@@ -0,0 +1,354 @@
-- ===================================================================
-- MARS-1005 Verify Data Integrity: Source vs Exported Data Validation
-- ===================================================================
-- Purpose: Verify data integrity between source tables and exported files
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Data Integrity Verification
PROMPT =========================================================================
-- Check 1: Source table record counts vs expected ETL keys
PROMPT Checking source table record counts...
DECLARE
vAdminRows NUMBER := 0;
vContentRows NUMBER := 0;
vCriterionRows NUMBER := 0;
vTotalRows NUMBER := 0;
vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
vTotalRows := vAdminRows + vContentRows + vCriterionRows;
DBMS_OUTPUT.PUT_LINE('Source table record counts:');
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' records');
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT: ' || vContentRows || ' records');
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' records');
DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records');
IF vTotalRows > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables');
END IF;
END;
/
-- Check 2: ETL key distribution analysis
PROMPT Checking ETL key distribution...
DECLARE
vAdminKeys NUMBER := 0;
vContentKeys NUMBER := 0;
vCriterionKeys NUMBER := 0;
vTotalKeys NUMBER := 0;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
SELECT COUNT(DISTINCT etl_key)
INTO vTotalKeys
FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
);
DBMS_OUTPUT.PUT_LINE('ETL key distribution:');
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct keys: ' || vAdminKeys);
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct keys: ' || vContentKeys);
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct keys: ' || vCriterionKeys);
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys);
IF vTotalKeys > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data');
END IF;
END;
/
-- Check 3: Template table compatibility verification
PROMPT Checking template table compatibility...
DECLARE
vAdminCols NUMBER := 0;
vContentCols NUMBER := 0;
vCriterionCols NUMBER := 0;
BEGIN
-- Check MPEC_ADMIN template compatibility
SELECT COUNT(*)
INTO vAdminCols
FROM all_tab_columns
WHERE owner = 'CT_ET_TEMPLATES'
AND table_name = 'C2D_MPEC_ADMIN';
-- Check MPEC_CONTENT template compatibility
SELECT COUNT(*)
INTO vContentCols
FROM all_tab_columns
WHERE owner = 'CT_ET_TEMPLATES'
AND table_name = 'C2D_MPEC_CONTENT';
-- Check MPEC_CONTENT_CRITERION template compatibility
SELECT COUNT(*)
INTO vCriterionCols
FROM all_tab_columns
WHERE owner = 'CT_ET_TEMPLATES'
AND table_name = 'C2D_MPEC_CONTENT_CRITERION';
DBMS_OUTPUT.PUT_LINE('Template table column counts:');
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_ADMIN: ' || vAdminCols || ' columns');
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT: ' || vContentCols || ' columns');
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns');
IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
END IF;
END;
/
-- Check 4: Verify A_ETL_LOAD_SET_FK values exist in A_LOAD_HISTORY
PROMPT Checking ETL key references in A_LOAD_HISTORY...
DECLARE
vValidKeys NUMBER := 0;
vTotalSourceKeys NUMBER := 0;
BEGIN
-- Count total distinct ETL keys in source tables
SELECT COUNT(DISTINCT etl_key)
INTO vTotalSourceKeys
FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
);
-- Count how many exist in A_LOAD_HISTORY
SELECT COUNT(DISTINCT etl_key)
INTO vValidKeys
FROM (
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
UNION
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
) src
WHERE EXISTS (
SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h
WHERE h.A_ETL_LOAD_SET_KEY = src.etl_key
);
DBMS_OUTPUT.PUT_LINE('ETL key validation:');
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys in source: ' || vTotalSourceKeys);
DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys);
IF vValidKeys = vTotalSourceKeys THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys));
END IF;
END;
/
PROMPT =====================================================================================
PROMPT MARS-1005 Record Count Verification
PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT =====================================================================================
DECLARE
TYPE t_table_info IS RECORD (
source_schema VARCHAR2(50),
source_table VARCHAR2(100),
external_table VARCHAR2(100),
description VARCHAR2(200)
);
TYPE t_table_list IS TABLE OF t_table_info;
vTables t_table_list;
vSourceCount NUMBER;
vTargetCount NUMBER;
vTotalSourceCount NUMBER := 0;
vTotalTargetCount NUMBER := 0;
vMismatchCount NUMBER := 0;
vSql VARCHAR2(4000);
vFileCount NUMBER := 0;
vValidationResult VARCHAR2(100);
BEGIN
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
DBMS_OUTPUT.PUT_LINE('');
-- Initialize table list with C2D MPEC configuration
vTables := t_table_list(
t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'),
t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'),
t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)')
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP
-- Get source table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).source_schema || '.' || vTables(i).source_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vSourceCount;
vTotalSourceCount := vTotalSourceCount + vSourceCount;
EXCEPTION
WHEN OTHERS THEN
vSourceCount := -1;
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table');
CONTINUE;
END;
-- Get target external table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).external_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vTargetCount;
vTotalTargetCount := vTotalTargetCount + vTargetCount;
EXCEPTION
WHEN OTHERS THEN
-- Handle expected errors for empty external tables
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vTargetCount := 0; -- Treat as empty (no files exported yet)
ELSE
vTargetCount := -1; -- Real error
END IF;
END;
-- Display comparison results with thousands separators
DECLARE
vStatus VARCHAR2(20);
vSourceDisplay VARCHAR2(17);
vTargetDisplay VARCHAR2(17);
BEGIN
-- Format source count display
IF vSourceCount = -1 THEN
vSourceDisplay := 'ERROR';
ELSE
vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999');
END IF;
-- Format target count display
IF vTargetCount = -1 THEN
vTargetDisplay := 'ERROR';
ELSE
vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999');
END IF;
-- Determine status
IF vSourceCount = vTargetCount THEN
vStatus := 'PASS';
ELSIF vTargetCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSIF vSourceCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(vSourceDisplay, 15) ||
LPAD(vTargetDisplay, 15) || ' ' ||
vStatus
);
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE(
RPAD('TOTALS', 24) ||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15)
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
-- Count exported files for additional verification
SELECT COUNT(*)
INTO vFileCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE RECEPTION_DATE >= SYSDATE - 1/24
AND (SOURCE_FILE_NAME LIKE '200_%');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)');
DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 AND vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED');
DBMS_OUTPUT.PUT_LINE(' All record counts match between source and exported data');
DBMS_OUTPUT.PUT_LINE(' Export completed successfully');
ELSIF vMismatchCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES');
DBMS_OUTPUT.PUT_LINE(' Found ' || vMismatchCount || ' table(s) with count mismatches');
DBMS_OUTPUT.PUT_LINE(' NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)');
DBMS_OUTPUT.PUT_LINE(' Review export logs and pre-check results before re-running exports');
ELSE
DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED');
DBMS_OUTPUT.PUT_LINE(' No files found in export registration');
DBMS_OUTPUT.PUT_LINE(' Verify export execution completed successfully');
END IF;
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Legend:');
DBMS_OUTPUT.PUT_LINE(' PASS - Record counts match (export successful)');
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Additional ETL key analysis for C2D MPEC data
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('ETL Key Analysis:');
DECLARE
vAdminKeys NUMBER;
vContentKeys NUMBER;
vCriterionKeys NUMBER;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)');
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)');
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct ETL keys: ' || vCriterionKeys || ' (expected: 2 for keys 2009-2010)');
-- Expected file count = sum of distinct ETL keys per table
DBMS_OUTPUT.PUT_LINE('- Expected export files: ' || (vAdminKeys + vContentKeys + vCriterionKeys));
DBMS_OUTPUT.PUT_LINE('- Actual export files: ' || vFileCount);
END;
END;
/
PROMPT =========================================================================
PROMPT Data Integrity Verification Completed
PROMPT =========================================================================

167
MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql Normal file
View File

@@ -0,0 +1,167 @@
--=============================================================================================================================
-- MARS-1005 ROLLBACK: Delete Exported CSV Files from DATA Bucket
--=============================================================================================================================
-- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables
-- WARNING: This will permanently delete exported data files!
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
-- Related: MARS-1005 - C2D MPEC Data Export Rollback
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
-- Delete CSV files registered by MARS-1005 process
FOR rec IN (
SELECT SOURCE_FILE_NAME AS object_name
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND SOURCE_FILE_NAME LIKE '%MPEC_ADMIN%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
-- Delete CSV files registered by MARS-1005 process
FOR rec IN (
SELECT SOURCE_FILE_NAME AS object_name
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT%'
AND SOURCE_FILE_NAME NOT LIKE '%CRITERION%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
-- Delete CSV files registered by MARS-1005 process
FOR rec IN (
SELECT SOURCE_FILE_NAME AS object_name
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT_CRITERION%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT SUCCESS: All CSV file deletion operations completed

78
MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql Normal file
View File

@@ -0,0 +1,78 @@
-- ===================================================================
-- MARS-1005 Rollback Step 1: Delete File Registrations
-- ===================================================================
-- Purpose: Remove MARS-1005 export file registrations from A_SOURCE_FILE_RECEIVED
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Step 1: Delete File Registrations
PROMPT =========================================================================
DECLARE
vFileCount NUMBER := 0;
vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN
-- Count files to be deleted (using PROCESS_NAME)
SELECT COUNT(*)
INTO vFileCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005';
DBMS_OUTPUT.PUT_LINE('Files to be deleted: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('Using PROCESS_NAME = ''MARS-1005'' filter');
IF vFileCount > 0 THEN
-- Show files before deletion
DBMS_OUTPUT.PUT_LINE('Files being removed:');
FOR rec IN (
SELECT A_SOURCE_FILE_RECEIVED_KEY,
SUBSTR(SOURCE_FILE_NAME, 1, 60) AS FILE_NAME,
TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME,
PROCESS_NAME
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
ORDER BY RECEPTION_DATE DESC
) LOOP
DBMS_OUTPUT.PUT_LINE('- ' || rec.FILE_NAME || ' (ID: ' || rec.A_SOURCE_FILE_RECEIVED_KEY || ', Process: ' || rec.PROCESS_NAME || ')');
END LOOP;
-- Delete the file registrations using PROCESS_NAME
DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005';
vDeletedCount := SQL%ROWCOUNT;
COMMIT;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' file registrations');
-- Log the rollback action
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'INFO',
'Deleted ' || vDeletedCount || ' file registrations');
COMMIT;
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No file registrations found to delete');
END IF;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
vErrorMsg := 'Failed to delete file registrations: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during file registration deletion: ' || SQLERRM);
-- Log the error
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', vErrorMsg);
COMMIT;
RAISE;
END;
/
PROMPT =========================================================================
PROMPT File Registration Rollback Completed
PROMPT =========================================================================

77
MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql Normal file
View File

@@ -0,0 +1,77 @@
-- ===================================================================
-- MARS-1005 Rollback Step 2: Clean Process Logs
-- ===================================================================
-- Purpose: Remove MARS-1005 process logs from A_PROCESS_LOG
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Step 2: Clean Process Logs
PROMPT =========================================================================
DECLARE
vLogCount NUMBER := 0;
vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN
-- Count logs to be deleted
SELECT COUNT(*)
INTO vLogCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week (safety)
DBMS_OUTPUT.PUT_LINE('Process log entries to be deleted: ' || vLogCount);
IF vLogCount > 0 THEN
-- Show recent logs before deletion
DBMS_OUTPUT.PUT_LINE('Recent MARS-1005 log entries being removed:');
FOR rec IN (
SELECT A_PROCESS_LOG_KEY,
TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 10 ROWS ONLY
) LOOP
DBMS_OUTPUT.PUT_LINE('- ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' ||
rec.PROCEDURE_NAME || ': ' || rec.MESSAGE);
END LOOP;
-- Delete the process logs
DELETE FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7;
vDeletedCount := SQL%ROWCOUNT;
COMMIT;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' process log entries');
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No process log entries found to delete');
END IF;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
vErrorMsg := 'Failed to clean process logs: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during process log cleanup: ' || SQLERRM);
-- Log the error (will remain after rollback for debugging)
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', vErrorMsg);
COMMIT;
RAISE;
END;
/
PROMPT =========================================================================
PROMPT Process Log Cleanup Completed
PROMPT =========================================================================

207
MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql Normal file
View File

@@ -0,0 +1,207 @@
-- ===================================================================
-- MARS-1005 Rollback Verification: Confirm Rollback Completion
-- ===================================================================
-- Purpose: Verify that MARS-1005 rollback completed successfully
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Verification
PROMPT =========================================================================
-- Check 1: Verify file registrations were removed
PROMPT Checking file registration cleanup...
DECLARE
vRemainingFiles NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRemainingFiles
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE SOURCE_FILE_NAME LIKE '200%' -- ETL keys 2001-2010
AND RECEPTION_DATE >= SYSDATE - 7; -- Last week
IF vRemainingFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 file registrations successfully removed');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingFiles || ' file registrations still exist');
-- Show remaining files
FOR rec IN (
SELECT SUBSTR(SOURCE_FILE_NAME, 1, 50) AS FILE_NAME,
TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE SOURCE_FILE_NAME LIKE '200%'
AND RECEPTION_DATE >= SYSDATE - 7
) LOOP
DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.FILE_NAME);
END LOOP;
END IF;
END;
/
-- Check 2: Verify process logs were cleaned
PROMPT Checking process log cleanup...
DECLARE
vRemainingLogs NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRemainingLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week
IF vRemainingLogs = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 process logs successfully removed');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingLogs || ' process log entries still exist');
-- Show remaining logs (first few)
FOR rec IN (
SELECT TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 3 ROWS ONLY
) LOOP
DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.LOG_TIME || ' ' || rec.PROCEDURE_NAME);
END LOOP;
END IF;
END;
/
-- Check 3: Verify cloud bucket cleanup (informational only)
PROMPT Checking cloud bucket status...
DECLARE
vCloudFileCount NUMBER := 0;
vCredentialName VARCHAR2(100);
vDataBucketUri VARCHAR2(500);
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri);
-- Count remaining files in cloud bucket
BEGIN
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri
))
WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%'
) LOOP
vCloudFileCount := vCloudFileCount + 1;
IF vCloudFileCount <= 3 THEN -- Show first 3 files
DBMS_OUTPUT.PUT_LINE(' Cloud file: ' || rec.object_name);
END IF;
END LOOP;
IF vCloudFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket');
ELSE
DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket');
DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback');
DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot check cloud bucket: ' || SQLERRM);
END;
END;
/
-- Check 4: Verify rollback logs were created
PROMPT Checking rollback operation logs...
DECLARE
vRollbackLogs NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRollbackLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK'
AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour
IF vRollbackLogs > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback operation logs found: ' || vRollbackLogs);
-- Show recent rollback logs
FOR rec IN (
SELECT TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 50) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK'
AND LOG_TIMESTAMP >= SYSDATE - 1/24
ORDER BY LOG_TIMESTAMP DESC
) LOOP
DBMS_OUTPUT.PUT_LINE(' ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' ||
rec.PROCEDURE_NAME || ': ' || rec.MESSAGE);
END LOOP;
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Warning: No rollback operation logs found');
END IF;
END;
/
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Verification Summary
PROMPT =========================================================================
DECLARE
vRemainingFiles NUMBER := 0;
vRemainingLogs NUMBER := 0;
vRollbackStatus VARCHAR2(20);
BEGIN
-- Count remaining registrations
SELECT COUNT(*)
INTO vRemainingFiles
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE SOURCE_FILE_NAME LIKE '200%'
AND RECEPTION_DATE >= SYSDATE - 7;
-- Count remaining process logs
SELECT COUNT(*)
INTO vRemainingLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7;
-- Determine rollback status
IF vRemainingFiles = 0 AND vRemainingLogs = 0 THEN
vRollbackStatus := 'COMPLETE';
ELSIF vRemainingFiles = 0 OR vRemainingLogs = 0 THEN
vRollbackStatus := 'PARTIAL';
ELSE
vRollbackStatus := 'INCOMPLETE';
END IF;
DBMS_OUTPUT.PUT_LINE('MARS-1005 Rollback Status: ' || vRollbackStatus);
DBMS_OUTPUT.PUT_LINE('- Remaining file registrations: ' || vRemainingFiles);
DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs);
IF vRollbackStatus = 'COMPLETE' THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback completed successfully - system clean');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Rollback incomplete - manual cleanup may be required');
END IF;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Note: Cloud bucket files (OCI) are not automatically removed');
DBMS_OUTPUT.PUT_LINE(' Use OCI console or DBMS_CLOUD commands for file deletion if needed');
END;
/
PROMPT =========================================================================
PROMPT Rollback Verification Completed
PROMPT =========================================================================

88
MARS_Packages/REL03/MARS-1005/install_mars1005.sql Normal file
View File

@@ -0,0 +1,88 @@
-- ===================================================================
-- MARS-1005 INSTALL SCRIPT: C2D MPEC Data Export to External Tables
-- ===================================================================
-- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema
-- to OCI buckets (ODS bucket CSV format)
-- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
-- Log files are automatically created in log/ subdirectory
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
host mkdir log 2>nul
var filename VARCHAR2(100)
BEGIN
:filename := 'log/INSTALL_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
SET ECHO OFF
SET TIMING ON
SET SERVEROUTPUT ON SIZE UNLIMITED
SET PAUSE OFF
-- Set current schema context (optional - use when modifying packages in specific schema)
-- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS;
PROMPT =========================================================================
PROMPT MARS-1005: C2D MPEC Data Export to External Tables (One-Time Migration)
PROMPT =========================================================================
PROMPT
PROMPT This script will export 3 C2D MPEC tables to OCI buckets:
PROMPT
PROMPT TARGET: ODS Bucket (CSV format):
PROMPT - MPEC_ADMIN
PROMPT - MPEC_CONTENT
PROMPT - MPEC_CONTENT_CRITERION
PROMPT
PROMPT Key Features:
PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking
PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*)
PROMPT - ODS/C2D bucket path structure
PROMPT =========================================================================
-- Confirm installation with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20001, 'Installation aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT =========================================================================
PROMPT Step 1: Export OU_TOP Data to ODS Bucket
PROMPT =========================================================================
@@01_MARS_1005_export_top_data.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 2: Verify Exports (File Registration Check)
PROMPT =========================================================================
@@02_MARS_1005_verify_exports.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 3: Verify Data Integrity (Source vs Exported)
PROMPT =========================================================================
@@03_MARS_1005_verify_data_integrity.sql
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Installation - COMPLETED
PROMPT =========================================================================
PROMPT Check the log file for complete installation details.
PROMPT For rollback, use: rollback_mars1005.sql
PROMPT =========================================================================
spool off
quit;

81
MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql Normal file
View File

@@ -0,0 +1,81 @@
-- ===================================================================
-- MARS-1005 ROLLBACK SCRIPT: C2D MPEC Data Export Rollback
-- ===================================================================
-- Purpose: Rollback MARS-1005 - Delete exported CSV files and file registrations
-- WARNING: This will DELETE all exported data files and registrations!
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
host mkdir log 2>nul
var filename VARCHAR2(100)
BEGIN
:filename := 'log/ROLLBACK_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
SET ECHO OFF
SET TIMING ON
SET SERVEROUTPUT ON SIZE UNLIMITED
SET PAUSE OFF
PROMPT =========================================================================
PROMPT MARS-1005: Rollback C2D MPEC Data Export
PROMPT =========================================================================
PROMPT WARNING: This will DELETE exported CSV files and file registrations!
PROMPT - ODS bucket: mrds_data_dev/ODS/C2D/
PROMPT - File registrations: A_SOURCE_FILE_RECEIVED entries
PROMPT
PROMPT Only proceed if export failed and needs to be restarted!
PROMPT =========================================================================
-- Confirm rollback with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT =========================================================================
PROMPT Step 1: Delete Exported CSV Files from DATA Bucket
PROMPT =========================================================================
@@90_MARS_1005_rollback_delete_csv_files.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 2: Delete File Registrations
PROMPT =========================================================================
@@91_MARS_1005_rollback_file_registrations.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 3: Clean Process Logs
PROMPT =========================================================================
@@92_MARS_1005_rollback_process_logs.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 4: Verify Rollback Completion
PROMPT =========================================================================
@@99_MARS_1005_verify_rollback.sql
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Rollback - COMPLETED
PROMPT =========================================================================
PROMPT Check the log file for complete rollback details.
PROMPT =========================================================================
spool off
quit;