admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: admin:113ea0a6188551ffeaa90f720c0a2e318b1eac6f
Branches Tags
admin:main admin:develop
...
compare: admin:0ecc119ee934408085b23f54314c5f229171d4cb
Branches Tags
admin:main admin:develop

4 Commits
113ea0a618 ... 0ecc119ee9

Author SHA1 Message Date
Grzegorz Michalski
0ecc119ee9 Refactor data integrity verification script to use A_ETL_LOAD_SET_FK instead of A_WORKFLOW_HISTORY_KEY 2026-03-09 09:31:30 +01:00
Grzegorz Michalski
182e6240d3 Update export script comments for clarity and consistency 2026-03-09 09:25:20 +01:00
Grzegorz Michalski
b81e524351 Refactor MARS-1005 scripts for OU_TOP legacy data export and rollback 
- Updated SQL scripts to verify data integrity for 6 OU_TOP.LEGACY_* tables instead of 3 C2D MPEC tables.
- Modified rollback script to delete exported CSV files from ODS/TOP/ bucket paths.
- Enhanced verification script to check for remaining files and cloud bucket contents specific to MARS-1005.
- Adjusted install script to reflect changes in target tables and their corresponding paths in the ODS bucket.
- Updated README to include instructions for the new MARS-1005 installation and rollback processes.
 2026-03-06 14:34:12 +01:00
Grzegorz Michalski
73e99b6e76 MARS-1005 2026-03-06 12:06:18 +01:00
11 changed files with 1945 additions and 0 deletions
Expand all files Collapse all files

5
MARS_Packages/REL03/MARS-1005/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
# Exclude temporary folders from version control
confluence/
log/
test/
mock_data/

618
MARS_Packages/REL03/MARS-1005/01_MARS_1005_export_top_data.sql Normal file
View File

@@ -0,0 +1,618 @@
-- =====================================================================================
-- Script: 01_MARS_1005_export_top_data.sql
-- Purpose: Export OU_TOP historical data to ODS bucket (DATA bucket, CSV format)
-- Author: Grzegorz Michalski
-- Created: 2026-03-06
-- MARS Issue: MARS-1005
-- Target: mrds_data_dev/ODS/TOP/
-- Tables:
-- 1. OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
-- 2. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
-- 3. OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
-- 4. OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
-- 5. OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
-- 6. OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
SET TIMING ON;
PROMPT =====================================================================================
PROMPT MARS-1005: OU_TOP Historical Data Export
PROMPT =====================================================================================
PROMPT Export Strategy:
PROMPT - Source: OU_TOP schema tables (operational database)
PROMPT - Target: DATA/ODS bucket as CSV files
PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA
PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED
PROMPT - Path Structure: ODS/TOP/TOP_*/
PROMPT Tables (6):
PROMPT ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM,
PROMPT ANNOUNCEMENT, FBL_ITEM (->TOP_FULLBIDLIST_ITEM), FULLBID_ARRAY_COMPILED
PROMPT =====================================================================================
-- Log export start
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Starting historical OU_TOP data export',
'Tables: ALLOTMENT, ALLOTMENT_MODIFICATION_HEADER, ALLOTMENT_MODIFICATION_ITEM, ANNOUNCEMENT, FBL_ITEM, FULLBID_ARRAY_COMPILED');
PROMPT
PROMPT =====================================================================================
PROMPT PRE-EXPORT CHECK: Verify Existing Files in ODS Bucket
PROMPT =====================================================================================
-- Helper procedure (inline) to check one folder
-- Check 1: ALLOTMENT
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT files found - bucket is clean');
END IF;
END;
/
-- Check 2: ALLOTMENT_MODIFICATION_HEADER
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_HEADER files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_HEADER files found - bucket is clean');
END IF;
END;
/
-- Check 3: ALLOTMENT_MODIFICATION_ITEM
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ALLOTMENT_MODIFICATION_ITEM files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ALLOTMENT_MODIFICATION_ITEM files found - bucket is clean');
END IF;
END;
/
-- Check 4: ANNOUNCEMENT
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_ANNOUNCEMENT/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_ANNOUNCEMENT files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_ANNOUNCEMENT files found - bucket is clean');
END IF;
END;
/
-- Check 5: FBL_ITEM (folder: TOP_FULLBIDLIST_ITEM)
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBIDLIST_ITEM/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBIDLIST_ITEM files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBIDLIST_ITEM files found - bucket is clean');
END IF;
END;
/
-- Check 6: FULLBID_ARRAY_COMPILED
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
vPrintCount NUMBER := 0;
BEGIN
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/';
SELECT COUNT(*) INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri))
WHERE object_name NOT LIKE '%/';
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: TOP_FULLBID_ARRAY_COMPILED files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri || ' Files found: ' || vFileCount);
FOR rec IN (SELECT object_name, bytes FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(credential_name => 'OCI$RESOURCE_PRINCIPAL', location_uri => vLocationUri)) WHERE object_name NOT LIKE '%/' ORDER BY object_name) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes)');
vPrintCount := vPrintCount + 1;
EXIT WHEN vPrintCount >= 10;
END LOOP;
IF vFileCount > 10 THEN
DBMS_OUTPUT.PUT_LINE(' ... and ' || (vFileCount - 10) || ' more file(s)');
END IF;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('>>> Records via external table: ' || vRecordCount);
EXCEPTION WHEN OTHERS THEN DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count via external table: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing TOP_FULLBID_ARRAY_COMPILED files found - bucket is clean');
END IF;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT PRE-EXPORT: Verify Source and Target Table Readiness
PROMPT =====================================================================================
DECLARE
v1Source NUMBER := 0; v2Source NUMBER := 0; v3Source NUMBER := 0;
v4Source NUMBER := 0; v5Source NUMBER := 0; v6Source NUMBER := 0;
v1Target NUMBER := 0; v2Target NUMBER := 0; v3Target NUMBER := 0;
v4Target NUMBER := 0; v5Target NUMBER := 0; v6Target NUMBER := 0;
vTotalSource NUMBER := 0;
vTotalTarget NUMBER := 0;
-- safe_count: ONLY for ODS external tables
-- Returns 0 when no data file (ORA-29913, ORA-29400, KUP-13023); re-raises all other errors
PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS
BEGIN
EXECUTE IMMEDIATE pSql INTO pResult;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
pResult := 0;
ELSE
RAISE;
END IF;
END;
BEGIN
-- Source counts (direct - if table does not exist, error propagates)
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Source;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Source;
vTotalSource := v1Source + v2Source + v3Source + v4Source + v5Source + v6Source;
DBMS_OUTPUT.PUT_LINE('Source table record counts (pre-export):');
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT: ' || v1Source);
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_HEADER: ' || v2Source);
DBMS_OUTPUT.PUT_LINE('- ALLOTMENT_MODIFICATION_ITEM: ' || v3Source);
DBMS_OUTPUT.PUT_LINE('- ANNOUNCEMENT: ' || v4Source);
DBMS_OUTPUT.PUT_LINE('- FBL_ITEM: ' || v5Source);
DBMS_OUTPUT.PUT_LINE('- FULLBID_ARRAY_COMPILED: ' || v6Source);
DBMS_OUTPUT.PUT_LINE('- TOTAL SOURCE: ' || vTotalSource);
-- Target external table counts
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5Target);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6Target);
vTotalTarget := GREATEST(v1Target,0) + GREATEST(v2Target,0) + GREATEST(v3Target,0)
+ GREATEST(v4Target,0) + GREATEST(v5Target,0) + GREATEST(v6Target,0);
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Target external table record counts (pre-export):');
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_ODS: ' || v1Target);
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_HEADER_ODS: ' || v2Target);
DBMS_OUTPUT.PUT_LINE('- TOP_ALLOTMENT_MODIFICATION_ITEM_ODS: ' || v3Target);
DBMS_OUTPUT.PUT_LINE('- TOP_ANNOUNCEMENT_ODS: ' || v4Target);
DBMS_OUTPUT.PUT_LINE('- TOP_FULLBIDLIST_ITEM_ODS: ' || v5Target);
DBMS_OUTPUT.PUT_LINE('- TOP_FULLBID_ARRAY_COMPILED_ODS: ' || v6Target);
IF vTotalSource > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Source tables contain data - ready for export');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Source tables exist but contain no data - export will produce empty files');
END IF;
IF vTotalTarget = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Target external tables are clean - ready for fresh export');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Target tables contain ' || vTotalTarget || ' records - may be re-run');
END IF;
DBMS_OUTPUT.PUT_LINE('Proceeding with export...');
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 1/6: OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 2/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_HEADER export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_HEADER export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_HEADER', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 3/6: OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ALLOTMENT_MODIFICATION_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ALLOTMENT_MODIFICATION_ITEM export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ALLOTMENT_MODIFICATION_ITEM export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ALLOTMENT_MOD_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 4/6: OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_ANNOUNCEMENT',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_ANNOUNCEMENT',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_ANNOUNCEMENT',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: ANNOUNCEMENT export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: ANNOUNCEMENT export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_ANNOUNCEMENT', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 5/6: OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_FBL_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_FULLBIDLIST_ITEM',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: FBL_ITEM export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: FBL_ITEM export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_FBL_ITEM', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT TABLE 6/6: OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
PROMPT =====================================================================================
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA(
pSchemaName => 'OU_TOP',
pTableName => 'LEGACY_FULLBID_ARRAY_COMPILED',
pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup
pBucketArea => 'ODS',
pFolderName => 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED',
pTemplateTableName => 'CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED',
pMaxFileSize => 104857600,
pRegisterExport => TRUE,
pProcessName => 'MARS-1005'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: FULLBID_ARRAY_COMPILED export completed successfully');
EXCEPTION
WHEN OTHERS THEN
DECLARE vErrorMsg VARCHAR2(4000) := SUBSTR(SQLERRM, 1, 4000); BEGIN
DBMS_OUTPUT.PUT_LINE('ERROR: FULLBID_ARRAY_COMPILED export failed: ' || vErrorMsg);
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, Username, Osuser, Machine, Module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES ('MARS-1005', USER, SYS_CONTEXT('USERENV','OS_USER'), SYS_CONTEXT('USERENV','HOST'),
'MARS-1005', 'MARS-1005', 'EXPORT_FULLBID_ARRAY_COMPILED', NULL, 'ERROR', 'Export failed: ' || vErrorMsg);
COMMIT;
END;
END;
/
PROMPT
PROMPT =====================================================================================
PROMPT Export Summary - Checking Results
PROMPT =====================================================================================
-- Log completion
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'All OU_TOP historical exports completed successfully');
PROMPT
PROMPT =====================================================================================
PROMPT MARS-1005 OU_TOP Export Completed!
PROMPT =====================================================================================
PROMPT POST-EXPORT: Source vs Target Record Count Comparison
PROMPT =====================================================================================
DECLARE
v1S NUMBER := 0; v2S NUMBER := 0; v3S NUMBER := 0;
v4S NUMBER := 0; v5S NUMBER := 0; v6S NUMBER := 0;
v1T NUMBER := 0; v2T NUMBER := 0; v3T NUMBER := 0;
v4T NUMBER := 0; v5T NUMBER := 0; v6T NUMBER := 0;
vTotalS NUMBER := 0;
vTotalT NUMBER := 0;
vMismatch NUMBER := 0;
-- safe_count: ONLY for ODS external tables
-- Returns 0 when no data file (ORA-29913, ORA-29400, KUP-13023); re-raises all other errors
PROCEDURE safe_count(pSql VARCHAR2, pResult OUT NUMBER) IS
BEGIN
EXECUTE IMMEDIATE pSql INTO pResult;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
pResult := 0;
ELSE
RAISE;
END IF;
END;
PROCEDURE print_row(pTable VARCHAR2, pSrc NUMBER, pTgt NUMBER) IS
BEGIN
DBMS_OUTPUT.PUT_LINE(
RPAD(pTable, 40) || ' | ' ||
RPAD(TO_CHAR(pSrc), 8) || ' | ' ||
RPAD(TO_CHAR(pTgt), 8) || ' | ' ||
CASE WHEN pSrc = pTgt THEN 'OK' ELSE 'MISMATCH' END);
END;
BEGIN
-- Source (direct - if table does not exist, error propagates)
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5S;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6S;
vTotalS := v1S + v2S + v3S + v4S + v5S + v6S;
-- Target
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_ODS', v1T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', v2T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', v3T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_ANNOUNCEMENT_ODS', v4T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBIDLIST_ITEM_ODS', v5T);
safe_count('SELECT COUNT(*) FROM ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', v6T);
vTotalT := v1T + v2T + v3T + v4T + v5T + v6T;
DBMS_OUTPUT.PUT_LINE('POST-EXPORT VERIFICATION SUMMARY');
DBMS_OUTPUT.PUT_LINE(RPAD('Table', 40) || ' | Source | Target | Match');
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
print_row('ALLOTMENT', v1S, v1T); IF v1S != v1T THEN vMismatch := vMismatch + 1; END IF;
print_row('ALLOTMENT_MODIFICATION_HEADER', v2S, v2T); IF v2S != v2T THEN vMismatch := vMismatch + 1; END IF;
print_row('ALLOTMENT_MODIFICATION_ITEM', v3S, v3T); IF v3S != v3T THEN vMismatch := vMismatch + 1; END IF;
print_row('ANNOUNCEMENT', v4S, v4T); IF v4S != v4T THEN vMismatch := vMismatch + 1; END IF;
print_row('FBL_ITEM (->FULLBIDLIST_ITEM)', v5S, v5T); IF v5S != v5T THEN vMismatch := vMismatch + 1; END IF;
print_row('FULLBID_ARRAY_COMPILED', v6S, v6T); IF v6S != v6T THEN vMismatch := vMismatch + 1; END IF;
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
print_row('TOTAL', vTotalS, vTotalT);
DBMS_OUTPUT.PUT_LINE('');
IF vMismatch = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All record counts match - export verified');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vMismatch || ' table(s) have record count mismatches');
DBMS_OUTPUT.PUT_LINE(' Please review export logs and external table access permissions');
END IF;
END;
/
-- Log export completion
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE, PROCEDURE_PARAMETERS)
VALUES ('MARS-1005', 'EXPORT_TOP_DATA', 'INFO', 'Historical OU_TOP data export completed',
'Check verification scripts for detailed results');
COMMIT;
PROMPT
PROMPT =====================================================================================
PROMPT MARS-1005 OU_TOP Historical Data Export - COMPLETED
PROMPT
PROMPT Next steps:
PROMPT 1. Run: @02_MARS_1005_verify_exports.sql (verify file registration)
PROMPT 2. Run: @03_MARS_1005_verify_data_integrity.sql (full data verification)
PROMPT =====================================================================================

215
MARS_Packages/REL03/MARS-1005/02_MARS_1005_verify_exports.sql Normal file
View File

@@ -0,0 +1,215 @@
-- ===================================================================
-- MARS-1005 Verify Exports: Check Export Results and File Creation
-- ===================================================================
-- Purpose: Verify that OU_TOP historical data export completed successfully
-- Author: Grzegorz Michalski
-- Date: 2026-03-06
-- MARS Issue: MARS-1005
-- Tables: 6 OU_TOP.LEGACY_* tables exported to ODS/TOP/ bucket paths
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Export Verification
PROMPT =========================================================================
-- Check 1: Verify files were registered in A_SOURCE_FILE_RECEIVED
PROMPT Checking export file registration (PROCESS_NAME = MARS-1005)...
DECLARE
vFileCount NUMBER := 0;
vTotalBytes NUMBER := 0;
BEGIN
SELECT COUNT(*), NVL(SUM(BYTES), 0)
INTO vFileCount, vTotalBytes
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 1/24; -- Last hour
DBMS_OUTPUT.PUT_LINE('Registered export files (last hour): ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('Total file size: ' || ROUND(vTotalBytes / 1024, 2) || ' KB');
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration');
ELSIF vFileCount < 6 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Expected at least 6 files (1 per table), found: ' || vFileCount);
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files registered (>= 6)');
END IF;
END;
/
-- Check 2: Show recent export registrations by table
PROMPT Recent export file registrations per table:
SELECT
A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY,
SUBSTR(SOURCE_FILE_NAME, 1, 55) AS FILE_NAME,
PROCESSING_STATUS,
ROUND(NVL(BYTES, 0) / 1024, 2) AS SIZE_KB,
TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 1/24
ORDER BY A_SOURCE_FILE_CONFIG_KEY, RECEPTION_DATE DESC;
-- Check 2b: File count per A_SOURCE_FILE_CONFIG_KEY
PROMPT Export file count per source config key:
SELECT
r.A_SOURCE_FILE_CONFIG_KEY,
c.TABLE_ID,
COUNT(*) AS FILE_COUNT,
ROUND(NVL(SUM(r.BYTES), 0) / 1024, 2) AS TOTAL_KB
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE r.PROCESS_NAME = 'MARS-1005'
AND r.RECEPTION_DATE >= SYSDATE - 1/24
GROUP BY r.A_SOURCE_FILE_CONFIG_KEY, c.TABLE_ID
ORDER BY r.A_SOURCE_FILE_CONFIG_KEY;
-- Check 3: Verify export process logs
PROMPT Checking export process logs...
DECLARE
vLogCount NUMBER := 0;
vErrorCount NUMBER := 0;
BEGIN
SELECT COUNT(*), SUM(CASE WHEN LOG_LEVEL = 'ERROR' THEN 1 ELSE 0 END)
INTO vLogCount, vErrorCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Process log entries: ' || vLogCount);
DBMS_OUTPUT.PUT_LINE('SUCCESS: Error entries: ' || vErrorCount);
IF vErrorCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vErrorCount || ' errors found in process log');
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No errors found in process log');
END IF;
END;
/
-- Check 4: Display recent process logs
PROMPT Recent MARS-1005 process logs:
SELECT
TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 60) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 10 ROWS ONLY;
-- Check 5: Cloud bucket file verification across all 6 TOP folders
PROMPT Checking cloud bucket files in ODS/TOP/ paths...
DECLARE
vCredentialName VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL';
vDataBucketUri VARCHAR2(500);
vTotalFiles NUMBER := 0;
TYPE t_folder IS TABLE OF VARCHAR2(200);
vFolders t_folder := t_folder(
'ODS/TOP/TOP_ALLOTMENT/',
'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/',
'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/',
'ODS/TOP/TOP_ANNOUNCEMENT/',
'ODS/TOP/TOP_FULLBIDLIST_ITEM/',
'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/'
);
vFolderFiles NUMBER;
vFolderSize NUMBER;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Bucket URI: ' || vDataBucketUri);
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE(RPAD('Folder', 55) || RPAD('Files', 8) || 'Total KB');
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
FOR i IN 1..vFolders.COUNT LOOP
vFolderFiles := 0;
vFolderSize := 0;
BEGIN
SELECT COUNT(*), NVL(SUM(bytes), 0)
INTO vFolderFiles, vFolderSize
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || vFolders(i)
))
WHERE object_name NOT LIKE '%/';
EXCEPTION WHEN OTHERS THEN NULL;
END;
DBMS_OUTPUT.PUT_LINE(
RPAD(vFolders(i), 55) ||
RPAD(TO_CHAR(vFolderFiles), 8) ||
ROUND(vFolderSize / 1024, 2) || ' KB'
);
vTotalFiles := vTotalFiles + vFolderFiles;
END LOOP;
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
DBMS_OUTPUT.PUT_LINE('Total files across all TOP folders: ' || vTotalFiles);
IF vTotalFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('WARNING: No files found in any TOP folder');
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: Files present in ODS/TOP/ bucket paths');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM);
END;
/
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Export Verification Summary
PROMPT =========================================================================
DECLARE
vFileRegCount NUMBER := 0;
vLogErrorCount NUMBER := 0;
vOverallStatus VARCHAR2(20);
BEGIN
SELECT COUNT(*)
INTO vFileRegCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 1/24;
SELECT COUNT(*)
INTO vLogErrorCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_LEVEL = 'ERROR'
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
IF vFileRegCount >= 6 AND vLogErrorCount = 0 THEN
vOverallStatus := 'SUCCESS';
ELSIF vFileRegCount > 0 AND vLogErrorCount = 0 THEN
vOverallStatus := 'PARTIAL SUCCESS';
ELSE
vOverallStatus := 'ISSUES DETECTED';
END IF;
DBMS_OUTPUT.PUT_LINE('MARS-1005 Export Verification: ' || vOverallStatus);
DBMS_OUTPUT.PUT_LINE('- Registered files (last hour): ' || vFileRegCount || ' (expected: >= 6, one per table)');
DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount);
IF vOverallStatus = 'SUCCESS' THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful');
ELSIF vOverallStatus = 'PARTIAL SUCCESS' THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Some tables may have incomplete exports - review registrations above');
ELSE
DBMS_OUTPUT.PUT_LINE('ISSUES DETECTED: Review process logs and bucket contents above');
END IF;
END;
/
PROMPT =========================================================================
PROMPT Export Verification Completed
PROMPT =========================================================================

359
MARS_Packages/REL03/MARS-1005/03_MARS_1005_verify_data_integrity.sql Normal file
View File

@@ -0,0 +1,359 @@
-- ===================================================================
-- MARS-1005 Verify Data Integrity: Source vs Exported Data Validation
-- ===================================================================
-- Purpose: Verify data integrity between 6 OU_TOP.LEGACY_* source tables
-- and corresponding ODS external tables after export
-- Author: Grzegorz Michalski
-- Date: 2026-03-06
-- MARS Issue: MARS-1005
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Data Integrity Verification
PROMPT =========================================================================
-- Check 1: Source table record counts
PROMPT Checking source table record counts (OU_TOP.LEGACY_* tables)...
DECLARE
v1Rows NUMBER := 0; v2Rows NUMBER := 0; v3Rows NUMBER := 0;
v4Rows NUMBER := 0; v5Rows NUMBER := 0; v6Rows NUMBER := 0;
vTotalRows NUMBER := 0;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Rows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Rows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Rows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Rows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Rows;
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Rows;
vTotalRows := v1Rows + v2Rows + v3Rows + v4Rows + v5Rows + v6Rows;
DBMS_OUTPUT.PUT_LINE('Source table record counts:');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Rows || ' records');
DBMS_OUTPUT.PUT_LINE('- TOTAL : ' || vTotalRows || ' records');
IF vTotalRows > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access source tables: ' || SQLERRM);
DBMS_OUTPUT.PUT_LINE('NOTE: Ensure SELECT privilege on OU_TOP.LEGACY_* is granted to CT_MRDS');
END;
/
-- Check 2: A_ETL_LOAD_SET_FK distribution across source tables
PROMPT Checking A_ETL_LOAD_SET_FK distribution...
DECLARE
v1Keys NUMBER := 0; v2Keys NUMBER := 0; v3Keys NUMBER := 0;
v4Keys NUMBER := 0; v5Keys NUMBER := 0; v6Keys NUMBER := 0;
vDistinctAllKeys NUMBER := 0;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys;
SELECT COUNT(DISTINCT wk)
INTO vDistinctAllKeys
FROM (
SELECT A_ETL_LOAD_SET_FK AS wk FROM OU_TOP.LEGACY_ALLOTMENT UNION ALL
SELECT A_ETL_LOAD_SET_FK FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER UNION ALL
SELECT A_ETL_LOAD_SET_FK FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM UNION ALL
SELECT A_ETL_LOAD_SET_FK FROM OU_TOP.LEGACY_ANNOUNCEMENT UNION ALL
SELECT A_ETL_LOAD_SET_FK FROM OU_TOP.LEGACY_FBL_ITEM UNION ALL
SELECT A_ETL_LOAD_SET_FK FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED
);
DBMS_OUTPUT.PUT_LINE('Distinct A_ETL_LOAD_SET_FK values per table (source):');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys);
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys);
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys);
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys);
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys);
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys);
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL load keys (all tables): ' || vDistinctAllKeys);
IF vDistinctAllKeys > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL load key distribution looks normal');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: No ETL load keys found in source data');
END IF;
END;
/
-- Check 3: Template table compatibility verification
PROMPT Checking template table compatibility (CT_ET_TEMPLATES.TOP_*)...
DECLARE
vCols1 NUMBER := 0; vCols2 NUMBER := 0; vCols3 NUMBER := 0;
vCols4 NUMBER := 0; vCols5 NUMBER := 0; vCols6 NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vCols1
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT';
SELECT COUNT(*) INTO vCols2
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_HEADER';
SELECT COUNT(*) INTO vCols3
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_ITEM';
SELECT COUNT(*) INTO vCols4
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ANNOUNCEMENT';
SELECT COUNT(*) INTO vCols5
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBIDLIST_ITEM';
SELECT COUNT(*) INTO vCols6
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBID_ARRAY_COMPILED';
DBMS_OUTPUT.PUT_LINE('Template table column counts:');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT : ' || vCols1 || ' columns');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER: ' || vCols2 || ' columns');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM : ' || vCols3 || ' columns');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ANNOUNCEMENT : ' || vCols4 || ' columns');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM : ' || vCols5 || ' columns');
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED : ' || vCols6 || ' columns');
IF vCols1 > 0 AND vCols2 > 0 AND vCols3 > 0 AND vCols4 > 0 AND vCols5 > 0 AND vCols6 > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 template tables have defined structure');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
END IF;
END;
/
-- Check 4: Verify A_SOURCE_FILE_CONFIG entries for 6 TOP tables
PROMPT Checking A_SOURCE_FILE_CONFIG registration for TOP tables...
DECLARE
vConfigCount NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vConfigCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697); -- MARS-1005 config keys
DBMS_OUTPUT.PUT_LINE('A_SOURCE_FILE_CONFIG entries for MARS-1005 tables: ' || vConfigCount || ' (expected: 6)');
DBMS_OUTPUT.PUT_LINE('Config keys: 705(ALLOTMENT), 683(MOD_HDR), 684(MOD_ITEM), 689(ANNOUNCEMENT), 696(FBL_ITEM), 697(FBA_COMPILED)');
IF vConfigCount = 6 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 source file config entries confirmed');
ELSE
DBMS_OUTPUT.PUT_LINE('ERROR: Missing config entries (' || (6 - vConfigCount) || ' missing)');
END IF;
END;
/
PROMPT A_SOURCE_FILE_CONFIG details for TOP tables:
SELECT
A_SOURCE_FILE_CONFIG_KEY,
TABLE_ID,
TEMPLATE_TABLE_NAME,
SUBSTR(SOURCE_FILE_NAME_PATTERN, 1, 40) AS FILE_PATTERN
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697)
ORDER BY A_SOURCE_FILE_CONFIG_KEY;
PROMPT =====================================================================================
PROMPT MARS-1005 Record Count Verification
PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT =====================================================================================
DECLARE
TYPE t_table_info IS RECORD (
source_schema VARCHAR2(50),
source_table VARCHAR2(100),
external_table VARCHAR2(100),
description VARCHAR2(200)
);
TYPE t_table_list IS TABLE OF t_table_info;
vTables t_table_list;
vSourceCount NUMBER;
vTargetCount NUMBER;
vTotalSourceCount NUMBER := 0;
vTotalTargetCount NUMBER := 0;
vMismatchCount NUMBER := 0;
vSql VARCHAR2(4000);
vFileCount NUMBER := 0;
vValidationResult VARCHAR2(100);
BEGIN
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
DBMS_OUTPUT.PUT_LINE('');
-- Initialize table list with 6 OU_TOP LEGACY table configuration
vTables := t_table_list(
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT', 'ODS.TOP_ALLOTMENT_ODS', 'ALLOTMENT data (A_SOURCE_FILE_CONFIG_KEY=705)'),
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_HEADER', 'ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', 'MOD HEADER data (A_SOURCE_FILE_CONFIG_KEY=683)'),
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_ITEM', 'ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', 'MOD ITEM data (A_SOURCE_FILE_CONFIG_KEY=684)'),
t_table_info('OU_TOP', 'LEGACY_ANNOUNCEMENT', 'ODS.TOP_ANNOUNCEMENT_ODS', 'ANNOUNCEMENT data (A_SOURCE_FILE_CONFIG_KEY=689)'),
t_table_info('OU_TOP', 'LEGACY_FBL_ITEM', 'ODS.TOP_FULLBIDLIST_ITEM_ODS', 'FBL ITEM data (A_SOURCE_FILE_CONFIG_KEY=696)'),
t_table_info('OU_TOP', 'LEGACY_FULLBID_ARRAY_COMPILED', 'ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', 'FBA COMPILED data (A_SOURCE_FILE_CONFIG_KEY=697)')
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count Target Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP
-- Get source table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).source_schema || '.' || vTables(i).source_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vSourceCount;
vTotalSourceCount := vTotalSourceCount + vSourceCount;
EXCEPTION
WHEN OTHERS THEN
vSourceCount := -1;
DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table');
CONTINUE;
END;
-- Get target external table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).external_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vTargetCount;
vTotalTargetCount := vTotalTargetCount + vTargetCount;
EXCEPTION
WHEN OTHERS THEN
-- Handle expected errors for empty external tables
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vTargetCount := 0; -- Treat as empty (no files exported yet)
ELSE
vTargetCount := -1; -- Real error
END IF;
END;
-- Display comparison results with thousands separators
DECLARE
vStatus VARCHAR2(20);
vSourceDisplay VARCHAR2(17);
vTargetDisplay VARCHAR2(17);
BEGIN
-- Format source count display
IF vSourceCount = -1 THEN
vSourceDisplay := 'ERROR';
ELSE
vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999');
END IF;
-- Format target count display
IF vTargetCount = -1 THEN
vTargetDisplay := 'ERROR';
ELSE
vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999');
END IF;
-- Determine status
IF vSourceCount = vTargetCount THEN
vStatus := 'PASS';
ELSIF vTargetCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSIF vSourceCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(vSourceDisplay, 15) ||
LPAD(vTargetDisplay, 15) || ' ' ||
vStatus
);
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE(
RPAD('TOTALS', 24) ||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15)
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
-- Count MARS-1005 registered export files
SELECT COUNT(*)
INTO vFileCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 1/24;
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)');
DBMS_OUTPUT.PUT_LINE('Export files registered (PROCESS_NAME=MARS-1005): ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 AND vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED');
DBMS_OUTPUT.PUT_LINE(' All record counts match between source and exported data');
DBMS_OUTPUT.PUT_LINE(' Export completed successfully');
ELSIF vMismatchCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES');
DBMS_OUTPUT.PUT_LINE(' Found ' || vMismatchCount || ' table(s) with count mismatches');
DBMS_OUTPUT.PUT_LINE(' NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)');
DBMS_OUTPUT.PUT_LINE(' Review export logs and pre-check results before re-running exports');
ELSE
DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED');
DBMS_OUTPUT.PUT_LINE(' No files found in export registration');
DBMS_OUTPUT.PUT_LINE(' Verify export execution completed successfully');
END IF;
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Legend:');
DBMS_OUTPUT.PUT_LINE(' PASS - Record counts match (export successful)');
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Workflow Key Analysis
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('ETL Load Key Analysis (distinct A_ETL_LOAD_SET_FK per source table):');
DECLARE
v1Keys NUMBER; v2Keys NUMBER; v3Keys NUMBER;
v4Keys NUMBER; v5Keys NUMBER; v6Keys NUMBER;
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys;
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys;
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys || ' distinct keys');
DBMS_OUTPUT.PUT_LINE('- Actual export files registered: ' || vFileCount);
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot query ETL load keys: ' || SQLERRM);
END;
END;
/
PROMPT =========================================================================
PROMPT Data Integrity Verification Completed
PROMPT =========================================================================

209
MARS_Packages/REL03/MARS-1005/90_MARS_1005_rollback_delete_csv_files.sql Normal file
View File

@@ -0,0 +1,209 @@
--=============================================================================================================================
-- MARS-1005 ROLLBACK: Delete Exported CSV Files from DATA Bucket
--=============================================================================================================================
-- Purpose: Delete exported CSV files from ODS/TOP/ bucket folders for 6 OU_TOP LEGACY tables
-- WARNING: This will permanently delete exported data files!
-- Author: Grzegorz Michalski
-- Date: 2026-03-06
-- Related: MARS-1005 - OU_TOP Historical Data Export Rollback
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting OU_TOP LEGACY exported files from DATA Bucket
PROMPT ========================================================================
PROMPT WARNING: This will delete files registered with PROCESS_NAME = 'MARS-1005'
PROMPT from ODS/TOP/* paths in the DATA bucket.
PROMPT ========================================================================
-- Helper: generic delete procedure for one TOP table folder
-- Deletes objects whose SOURCE_FILE_NAME matches the folder prefix pattern
-- ROLLBACK TABLE 1/6: LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT/
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT files from: ' || vDataBucketUri || vFolderPath);
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
))
WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
object_uri => vDataBucketUri || vFolderPath || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE;
END IF;
END;
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT files deleted (' || vFileCount || ' file(s))');
END;
/
-- ROLLBACK TABLE 2/6: LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files...');
FOR rec IN (
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
)) WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION WHEN OTHERS THEN
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE; END IF;
END;
END LOOP;
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_HEADER files deleted (' || vFileCount || ' file(s))');
END;
/
-- ROLLBACK TABLE 3/6: LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files...');
FOR rec IN (
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
)) WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION WHEN OTHERS THEN
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE; END IF;
END;
END LOOP;
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_ITEM files deleted (' || vFileCount || ' file(s))');
END;
/
-- ROLLBACK TABLE 4/6: LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT/
PROMPT ROLLBACK: Deleting TOP_ANNOUNCEMENT files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ANNOUNCEMENT/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ANNOUNCEMENT files...');
FOR rec IN (
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
)) WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION WHEN OTHERS THEN
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE; END IF;
END;
END LOOP;
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ANNOUNCEMENT files deleted (' || vFileCount || ' file(s))');
END;
/
-- ROLLBACK TABLE 5/6: LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM/
PROMPT ROLLBACK: Deleting TOP_FULLBIDLIST_ITEM files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBIDLIST_ITEM/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBIDLIST_ITEM files...');
FOR rec IN (
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
)) WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION WHEN OTHERS THEN
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE; END IF;
END;
END LOOP;
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBIDLIST_ITEM files deleted (' || vFileCount || ' file(s))');
END;
/
-- ROLLBACK TABLE 6/6: LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/
PROMPT ROLLBACK: Deleting TOP_FULLBID_ARRAY_COMPILED files...
DECLARE
vDataBucketUri VARCHAR2(500);
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/';
vFileCount NUMBER := 0;
BEGIN
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBID_ARRAY_COMPILED files...');
FOR rec IN (
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vDataBucketUri || vFolderPath
)) WHERE object_name NOT LIKE '%/'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION WHEN OTHERS THEN
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE RAISE; END IF;
END;
END LOOP;
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBID_ARRAY_COMPILED files deleted (' || vFileCount || ' file(s))');
END;
/
PROMPT SUCCESS: All CSV file deletion operations completed

78
MARS_Packages/REL03/MARS-1005/91_MARS_1005_rollback_file_registrations.sql Normal file
View File

@@ -0,0 +1,78 @@
-- ===================================================================
-- MARS-1005 Rollback Step 1: Delete File Registrations
-- ===================================================================
-- Purpose: Remove MARS-1005 export file registrations from A_SOURCE_FILE_RECEIVED
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Step 1: Delete File Registrations
PROMPT =========================================================================
DECLARE
vFileCount NUMBER := 0;
vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN
-- Count files to be deleted (using PROCESS_NAME)
SELECT COUNT(*)
INTO vFileCount
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005';
DBMS_OUTPUT.PUT_LINE('Files to be deleted: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('Using PROCESS_NAME = ''MARS-1005'' filter');
IF vFileCount > 0 THEN
-- Show files before deletion
DBMS_OUTPUT.PUT_LINE('Files being removed:');
FOR rec IN (
SELECT A_SOURCE_FILE_RECEIVED_KEY,
SUBSTR(SOURCE_FILE_NAME, 1, 60) AS FILE_NAME,
TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME,
PROCESS_NAME
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
ORDER BY RECEPTION_DATE DESC
) LOOP
DBMS_OUTPUT.PUT_LINE('- ' || rec.FILE_NAME || ' (ID: ' || rec.A_SOURCE_FILE_RECEIVED_KEY || ', Process: ' || rec.PROCESS_NAME || ')');
END LOOP;
-- Delete the file registrations using PROCESS_NAME
DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005';
vDeletedCount := SQL%ROWCOUNT;
COMMIT;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' file registrations');
-- Log the rollback action
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'INFO',
'Deleted ' || vDeletedCount || ' file registrations');
COMMIT;
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No file registrations found to delete');
END IF;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
vErrorMsg := 'Failed to delete file registrations: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during file registration deletion: ' || SQLERRM);
-- Log the error
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'DELETE_FILE_REGISTRATIONS', 'ERROR', vErrorMsg);
COMMIT;
RAISE;
END;
/
PROMPT =========================================================================
PROMPT File Registration Rollback Completed
PROMPT =========================================================================

77
MARS_Packages/REL03/MARS-1005/92_MARS_1005_rollback_process_logs.sql Normal file
View File

@@ -0,0 +1,77 @@
-- ===================================================================
-- MARS-1005 Rollback Step 2: Clean Process Logs
-- ===================================================================
-- Purpose: Remove MARS-1005 process logs from A_PROCESS_LOG
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Step 2: Clean Process Logs
PROMPT =========================================================================
DECLARE
vLogCount NUMBER := 0;
vDeletedCount NUMBER := 0;
vErrorMsg VARCHAR2(4000);
BEGIN
-- Count logs to be deleted
SELECT COUNT(*)
INTO vLogCount
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week (safety)
DBMS_OUTPUT.PUT_LINE('Process log entries to be deleted: ' || vLogCount);
IF vLogCount > 0 THEN
-- Show recent logs before deletion
DBMS_OUTPUT.PUT_LINE('Recent MARS-1005 log entries being removed:');
FOR rec IN (
SELECT A_PROCESS_LOG_KEY,
TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 10 ROWS ONLY
) LOOP
DBMS_OUTPUT.PUT_LINE('- ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' ||
rec.PROCEDURE_NAME || ': ' || rec.MESSAGE);
END LOOP;
-- Delete the process logs
DELETE FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME IN ('MARS-1005', 'MARS-1005-ROLLBACK')
AND LOG_TIMESTAMP >= SYSDATE - 7;
vDeletedCount := SQL%ROWCOUNT;
COMMIT;
DBMS_OUTPUT.PUT_LINE('SUCCESS: Successfully deleted ' || vDeletedCount || ' process log entries');
ELSE
DBMS_OUTPUT.PUT_LINE('SUCCESS: No process log entries found to delete');
END IF;
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
vErrorMsg := 'Failed to clean process logs: ' || SQLERRM;
DBMS_OUTPUT.PUT_LINE('ERROR: Error during process log cleanup: ' || SQLERRM);
-- Log the error (will remain after rollback for debugging)
INSERT INTO CT_MRDS.A_PROCESS_LOG (PROCESS_NAME, PROCEDURE_NAME, LOG_LEVEL, LOG_MESSAGE)
VALUES ('MARS-1005-ROLLBACK', 'CLEANUP_PROCESS_LOGS', 'ERROR', vErrorMsg);
COMMIT;
RAISE;
END;
/
PROMPT =========================================================================
PROMPT Process Log Cleanup Completed
PROMPT =========================================================================

207
MARS_Packages/REL03/MARS-1005/99_MARS_1005_verify_rollback.sql Normal file
View File

@@ -0,0 +1,207 @@
-- ===================================================================
-- MARS-1005 Rollback Verification: Confirm Rollback Completion
-- ===================================================================
-- Purpose: Verify that MARS-1005 rollback completed successfully
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Verification
PROMPT =========================================================================
-- Check 1: Verify file registrations were removed
PROMPT Checking file registration cleanup...
DECLARE
vRemainingFiles NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRemainingFiles
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 7; -- Last week
IF vRemainingFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 file registrations successfully removed');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingFiles || ' file registrations still exist');
-- Show remaining files
FOR rec IN (
SELECT SUBSTR(SOURCE_FILE_NAME, 1, 50) AS FILE_NAME,
TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 7
) LOOP
DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.FILE_NAME);
END LOOP;
END IF;
END;
/
-- Check 2: Verify process logs were cleaned
PROMPT Checking process log cleanup...
DECLARE
vRemainingLogs NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRemainingLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7; -- Last week
IF vRemainingLogs = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: All MARS-1005 process logs successfully removed');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: ' || vRemainingLogs || ' process log entries still exist');
-- Show remaining logs (first few)
FOR rec IN (
SELECT TO_CHAR(LOG_TIMESTAMP, 'YYYY-MM-DD HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
SUBSTR(LOG_MESSAGE, 1, 40) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7
ORDER BY LOG_TIMESTAMP DESC
FETCH FIRST 3 ROWS ONLY
) LOOP
DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.LOG_TIME || ' ' || rec.PROCEDURE_NAME);
END LOOP;
END IF;
END;
/
-- Check 3: Verify cloud bucket cleanup (informational only)
PROMPT Checking cloud bucket status...
DECLARE
vCloudFileCount NUMBER := 0;
vCredentialName VARCHAR2(100);
vDataBucketUri VARCHAR2(500);
BEGIN
-- Get bucket URI and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri);
-- Count remaining files in cloud bucket
BEGIN
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri
))
WHERE object_name LIKE 'ODS/TOP/%'
) LOOP
vCloudFileCount := vCloudFileCount + 1;
IF vCloudFileCount <= 3 THEN -- Show first 3 files
DBMS_OUTPUT.PUT_LINE(' Cloud file: ' || rec.object_name);
END IF;
END LOOP;
IF vCloudFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: No TOP files found in cloud bucket - clean');
ELSE
DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' TOP file(s) still in cloud bucket');
DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback');
DBMS_OUTPUT.PUT_LINE(' Run 90_MARS_1005_rollback_delete_csv_files.sql to remove them');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot check cloud bucket: ' || SQLERRM);
END;
END;
/
-- Check 4: Verify rollback logs were created
PROMPT Checking rollback operation logs...
DECLARE
vRollbackLogs NUMBER := 0;
BEGIN
SELECT COUNT(*)
INTO vRollbackLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK'
AND LOG_TIMESTAMP >= SYSDATE - 1/24; -- Last hour
IF vRollbackLogs > 0 THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback operation logs found: ' || vRollbackLogs);
-- Show recent rollback logs
FOR rec IN (
SELECT TO_CHAR(LOG_TIMESTAMP, 'HH24:MI:SS') AS LOG_TIME,
PROCEDURE_NAME,
LOG_LEVEL,
SUBSTR(LOG_MESSAGE, 1, 50) AS MESSAGE
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005-ROLLBACK'
AND LOG_TIMESTAMP >= SYSDATE - 1/24
ORDER BY LOG_TIMESTAMP DESC
) LOOP
DBMS_OUTPUT.PUT_LINE(' ' || rec.LOG_TIME || ' [' || rec.LOG_LEVEL || '] ' ||
rec.PROCEDURE_NAME || ': ' || rec.MESSAGE);
END LOOP;
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Warning: No rollback operation logs found');
END IF;
END;
/
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Rollback Verification Summary
PROMPT =========================================================================
DECLARE
vRemainingFiles NUMBER := 0;
vRemainingLogs NUMBER := 0;
vRollbackStatus VARCHAR2(20);
BEGIN
-- Count remaining registrations
SELECT COUNT(*)
INTO vRemainingFiles
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE PROCESS_NAME = 'MARS-1005'
AND RECEPTION_DATE >= SYSDATE - 7;
-- Count remaining process logs
SELECT COUNT(*)
INTO vRemainingLogs
FROM CT_MRDS.A_PROCESS_LOG
WHERE PROCESS_NAME = 'MARS-1005'
AND LOG_TIMESTAMP >= SYSDATE - 7;
-- Determine rollback status
IF vRemainingFiles = 0 AND vRemainingLogs = 0 THEN
vRollbackStatus := 'COMPLETE';
ELSIF vRemainingFiles = 0 OR vRemainingLogs = 0 THEN
vRollbackStatus := 'PARTIAL';
ELSE
vRollbackStatus := 'INCOMPLETE';
END IF;
DBMS_OUTPUT.PUT_LINE('MARS-1005 Rollback Status: ' || vRollbackStatus);
DBMS_OUTPUT.PUT_LINE('- Remaining file registrations: ' || vRemainingFiles);
DBMS_OUTPUT.PUT_LINE('- Remaining process logs: ' || vRemainingLogs);
IF vRollbackStatus = 'COMPLETE' THEN
DBMS_OUTPUT.PUT_LINE('SUCCESS: Rollback completed successfully - system clean');
ELSE
DBMS_OUTPUT.PUT_LINE('WARNING: Rollback incomplete - manual cleanup may be required');
END IF;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Note: Cloud bucket files (OCI) are not automatically removed');
DBMS_OUTPUT.PUT_LINE(' Use OCI console or DBMS_CLOUD commands for file deletion if needed');
END;
/
PROMPT =========================================================================
PROMPT Rollback Verification Completed
PROMPT =========================================================================

91
MARS_Packages/REL03/MARS-1005/install_mars1005.sql Normal file
View File

@@ -0,0 +1,91 @@
-- ===================================================================
-- MARS-1005 INSTALL SCRIPT: OU_TOP Historical Data Export to ODS Bucket
-- ===================================================================
-- Purpose: One-time bulk export of 6 OU_TOP LEGACY tables to OCI DATA bucket
-- (ODS bucket area, CSV format)
-- Uses DATA_EXPORTER EXPORT_TABLE_DATA with pRegisterExport for file tracking
-- Author: Grzegorz Michalski
-- Date: 2026-03-06
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
-- Log files are automatically created in log/ subdirectory
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
host mkdir log 2>nul
var filename VARCHAR2(100)
BEGIN
:filename := 'log/INSTALL_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
SET ECHO OFF
SET TIMING ON
SET SERVEROUTPUT ON SIZE UNLIMITED
SET PAUSE OFF
-- Set current schema context (optional - use when modifying packages in specific schema)
-- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS;
PROMPT =========================================================================
PROMPT MARS-1005: OU_TOP Historical Data Export to ODS Bucket (One-Time Migration)
PROMPT =========================================================================
PROMPT
PROMPT This script will export 6 OU_TOP LEGACY tables to OCI DATA bucket:
PROMPT
PROMPT TARGET: DATA Bucket / ODS area (CSV format):
PROMPT - OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
PROMPT - OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
PROMPT - OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
PROMPT - OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
PROMPT
PROMPT Key Features:
PROMPT - Files registered in A_SOURCE_FILE_RECEIVED with PROCESS_NAME = 'MARS-1005'
PROMPT - Template table column order matching (CT_ET_TEMPLATES.TOP_*)
PROMPT - ODS/TOP bucket path structure
PROMPT =========================================================================
-- Confirm installation with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20001, 'Installation aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT =========================================================================
PROMPT Step 1: Export OU_TOP Data to ODS Bucket
PROMPT =========================================================================
@@01_MARS_1005_export_top_data.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 2: Verify Exports (File Registration Check)
PROMPT =========================================================================
@@02_MARS_1005_verify_exports.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 3: Verify Data Integrity (Source vs Exported)
PROMPT =========================================================================
@@03_MARS_1005_verify_data_integrity.sql
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Installation - COMPLETED
PROMPT =========================================================================
PROMPT Check the log file for complete installation details.
PROMPT For rollback, use: rollback_mars1005.sql
PROMPT =========================================================================
spool off
quit;

81
MARS_Packages/REL03/MARS-1005/rollback_mars1005.sql Normal file
View File

@@ -0,0 +1,81 @@
-- ===================================================================
-- MARS-1005 ROLLBACK SCRIPT: C2D MPEC Data Export Rollback
-- ===================================================================
-- Purpose: Rollback MARS-1005 - Delete exported CSV files and file registrations
-- WARNING: This will DELETE all exported data files and registrations!
-- Author: Grzegorz Michalski
-- Date: 2026-02-12
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
host mkdir log 2>nul
var filename VARCHAR2(100)
BEGIN
:filename := 'log/ROLLBACK_MARS_1005_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
END;
/
column filename new_value _filename
select :filename filename from dual;
spool &_filename
SET ECHO OFF
SET TIMING ON
SET SERVEROUTPUT ON SIZE UNLIMITED
SET PAUSE OFF
PROMPT =========================================================================
PROMPT MARS-1005: Rollback C2D MPEC Data Export
PROMPT =========================================================================
PROMPT WARNING: This will DELETE exported CSV files and file registrations!
PROMPT - ODS bucket: mrds_data_dev/ODS/C2D/
PROMPT - File registrations: A_SOURCE_FILE_RECEIVED entries
PROMPT
PROMPT Only proceed if export failed and needs to be restarted!
PROMPT =========================================================================
-- Confirm rollback with user
ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: '
WHENEVER SQLERROR EXIT SQL.SQLCODE
BEGIN
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user');
END IF;
END;
/
WHENEVER SQLERROR CONTINUE
PROMPT
PROMPT =========================================================================
PROMPT Step 1: Delete Exported CSV Files from DATA Bucket
PROMPT =========================================================================
@@90_MARS_1005_rollback_delete_csv_files.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 2: Delete File Registrations
PROMPT =========================================================================
@@91_MARS_1005_rollback_file_registrations.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 3: Clean Process Logs
PROMPT =========================================================================
@@92_MARS_1005_rollback_process_logs.sql
PROMPT
PROMPT =========================================================================
PROMPT Step 4: Verify Rollback Completion
PROMPT =========================================================================
@@99_MARS_1005_verify_rollback.sql
PROMPT
PROMPT =========================================================================
PROMPT MARS-1005 Rollback - COMPLETED
PROMPT =========================================================================
PROMPT Check the log file for complete rollback details.
PROMPT =========================================================================
spool off
quit;

5
README.md
View File

@@ -92,3 +92,8 @@ cd .\MARS_Packages\REL02_POST\MARS-1409
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1409.sql"
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1409.sql"
7z a -pMojeSuperHaslo#123 -mhe=on M1409_arch.7z MARS-1409
cd .\MARS_Packages\REL03\MARS-1005
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1005.sql"
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1005.sql"
7z a -pMojeSuperHaslo#123 -mhe=on M1005_arch.7z MARS-1005