Refactor MARS-1005 scripts for OU_TOP legacy data export and rollback
- Updated SQL scripts to verify data integrity for 6 OU_TOP.LEGACY_* tables instead of 3 C2D MPEC tables. - Modified rollback script to delete exported CSV files from ODS/TOP/ bucket paths. - Enhanced verification script to check for remaining files and cloud bucket contents specific to MARS-1005. - Adjusted install script to reflect changes in target tables and their corresponding paths in the ODS bucket. - Updated README to include instructions for the new MARS-1005 installation and rollback processes.
This commit is contained in:
Grzegorz Michalski
@@ -1,9 +1,11 @@
|
||||
-- ===================================================================
|
||||
-- MARS-1005 Verify Exports: Check Export Results and File Creation
|
||||
-- ===================================================================
|
||||
-- Purpose: Verify that C2D MPEC export completed successfully
|
||||
-- Purpose: Verify that OU_TOP historical data export completed successfully
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-12
|
||||
-- Date: 2026-03-06
|
||||
-- MARS Issue: MARS-1005
|
||||
-- Tables: 6 OU_TOP.LEGACY_* tables exported to ODS/TOP/ bucket paths
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET TIMING ON
|
||||
@@ -13,7 +15,7 @@ PROMPT MARS-1005 Export Verification
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Check 1: Verify files were registered in A_SOURCE_FILE_RECEIVED
|
||||
PROMPT Checking export file registration...
|
||||
PROMPT Checking export file registration (PROCESS_NAME = MARS-1005)...
|
||||
DECLARE
|
||||
vFileCount NUMBER := 0;
|
||||
vTotalBytes NUMBER := 0;
|
||||
@@ -21,43 +23,49 @@ BEGIN
|
||||
SELECT COUNT(*), NVL(SUM(BYTES), 0)
|
||||
INTO vFileCount, vTotalBytes
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour
|
||||
AND (SOURCE_FILE_NAME LIKE '2001_%' -- MPEC_ADMIN ETL keys
|
||||
OR SOURCE_FILE_NAME LIKE '2002_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2003_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2004_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2005_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2006_%' -- MPEC_CONTENT ETL keys
|
||||
OR SOURCE_FILE_NAME LIKE '2007_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2008_%'
|
||||
OR SOURCE_FILE_NAME LIKE '2009_%' -- MPEC_CONTENT_CRITERION ETL keys
|
||||
OR SOURCE_FILE_NAME LIKE '2010_%');
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 1/24; -- Last hour
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Registered export files: ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Total file size: ' || ROUND(vTotalBytes/1024, 2) || ' KB');
|
||||
DBMS_OUTPUT.PUT_LINE('Registered export files (last hour): ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('Total file size: ' || ROUND(vTotalBytes / 1024, 2) || ' KB');
|
||||
|
||||
IF vFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: No export files found in registration');
|
||||
ELSIF vFileCount < 9 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Expected 9 files (3 tables x 3 ETL keys), found: ' || vFileCount);
|
||||
ELSIF vFileCount < 6 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Expected at least 6 files (1 per table), found: ' || vFileCount);
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files found');
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All expected export files registered (>= 6)');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Check 2: Show recent export registrations
|
||||
PROMPT Recent export file registrations:
|
||||
-- Check 2: Show recent export registrations by table
|
||||
PROMPT Recent export file registrations per table:
|
||||
SELECT
|
||||
SUBSTR(SOURCE_FILE_NAME, 1, 40) AS FILE_NAME,
|
||||
A_SOURCE_FILE_CONFIG_KEY AS CONFIG_KEY,
|
||||
SUBSTR(SOURCE_FILE_NAME, 1, 55) AS FILE_NAME,
|
||||
PROCESSING_STATUS,
|
||||
ROUND(BYTES/1024, 2) AS SIZE_KB,
|
||||
ROUND(NVL(BYTES, 0) / 1024, 2) AS SIZE_KB,
|
||||
TO_CHAR(RECEPTION_DATE, 'HH24:MI:SS') AS TIME_EXPORTED
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE RECEPTION_DATE >= SYSDATE - 1/24 -- Last hour
|
||||
AND (SOURCE_FILE_NAME LIKE '200%') -- ETL keys starting with 200
|
||||
ORDER BY RECEPTION_DATE DESC;
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 1/24
|
||||
ORDER BY A_SOURCE_FILE_CONFIG_KEY, RECEPTION_DATE DESC;
|
||||
|
||||
-- Check 2b: File count per A_SOURCE_FILE_CONFIG_KEY
|
||||
PROMPT Export file count per source config key:
|
||||
SELECT
|
||||
r.A_SOURCE_FILE_CONFIG_KEY,
|
||||
c.TABLE_ID,
|
||||
COUNT(*) AS FILE_COUNT,
|
||||
ROUND(NVL(SUM(r.BYTES), 0) / 1024, 2) AS TOTAL_KB
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.PROCESS_NAME = 'MARS-1005'
|
||||
AND r.RECEPTION_DATE >= SYSDATE - 1/24
|
||||
GROUP BY r.A_SOURCE_FILE_CONFIG_KEY, c.TABLE_ID
|
||||
ORDER BY r.A_SOURCE_FILE_CONFIG_KEY;
|
||||
|
||||
-- Check 3: Verify export process logs
|
||||
PROMPT Checking export process logs...
|
||||
@@ -95,46 +103,66 @@ WHERE PROCESS_NAME = 'MARS-1005'
|
||||
ORDER BY LOG_TIMESTAMP DESC
|
||||
FETCH FIRST 10 ROWS ONLY;
|
||||
|
||||
-- Check 5: Cloud bucket file verification (if cloud_wrapper available)
|
||||
PROMPT Checking cloud bucket files...
|
||||
-- Check 5: Cloud bucket file verification across all 6 TOP folders
|
||||
PROMPT Checking cloud bucket files in ODS/TOP/ paths...
|
||||
DECLARE
|
||||
vCloudFileCount NUMBER := 0;
|
||||
vCredentialName VARCHAR2(100);
|
||||
vCredentialName VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL';
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
BEGIN
|
||||
-- Get bucket URI and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
vTotalFiles NUMBER := 0;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Checking ODS bucket: ' || vDataBucketUri);
|
||||
TYPE t_folder IS TABLE OF VARCHAR2(200);
|
||||
vFolders t_folder := t_folder(
|
||||
'ODS/TOP/TOP_ALLOTMENT/',
|
||||
'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/',
|
||||
'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/',
|
||||
'ODS/TOP/TOP_ANNOUNCEMENT/',
|
||||
'ODS/TOP/TOP_FULLBIDLIST_ITEM/',
|
||||
'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/'
|
||||
);
|
||||
|
||||
-- Count files in cloud bucket
|
||||
vFolderFiles NUMBER;
|
||||
vFolderSize NUMBER;
|
||||
BEGIN
|
||||
FOR rec IN (
|
||||
SELECT object_name
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
DBMS_OUTPUT.PUT_LINE('Bucket URI: ' || vDataBucketUri);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE(RPAD('Folder', 55) || RPAD('Files', 8) || 'Total KB');
|
||||
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
|
||||
|
||||
FOR i IN 1..vFolders.COUNT LOOP
|
||||
vFolderFiles := 0;
|
||||
vFolderSize := 0;
|
||||
BEGIN
|
||||
SELECT COUNT(*), NVL(SUM(bytes), 0)
|
||||
INTO vFolderFiles, vFolderSize
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri
|
||||
location_uri => vDataBucketUri || vFolders(i)
|
||||
))
|
||||
WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%'
|
||||
) LOOP
|
||||
vCloudFileCount := vCloudFileCount + 1;
|
||||
IF vCloudFileCount <= 5 THEN -- Show first 5 files
|
||||
DBMS_OUTPUT.PUT_LINE('- ' || rec.object_name);
|
||||
END IF;
|
||||
WHERE object_name NOT LIKE '%/';
|
||||
EXCEPTION WHEN OTHERS THEN NULL;
|
||||
END;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
RPAD(vFolders(i), 55) ||
|
||||
RPAD(TO_CHAR(vFolderFiles), 8) ||
|
||||
ROUND(vFolderSize / 1024, 2) || ' KB'
|
||||
);
|
||||
vTotalFiles := vTotalFiles + vFolderFiles;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Cloud bucket files found: ' || vCloudFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE(RPAD('-', 75, '-'));
|
||||
DBMS_OUTPUT.PUT_LINE('Total files across all TOP folders: ' || vTotalFiles);
|
||||
|
||||
IF vCloudFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: No files found in cloud bucket');
|
||||
IF vTotalFiles = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: No files found in any TOP folder');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Files present in ODS/TOP/ bucket paths');
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot access cloud bucket: ' || SQLERRM);
|
||||
END;
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
@@ -142,21 +170,17 @@ PROMPT =========================================================================
|
||||
PROMPT MARS-1005 Export Verification Summary
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Final verification summary
|
||||
DECLARE
|
||||
vFileRegCount NUMBER := 0;
|
||||
vCloudFileCount NUMBER := 0;
|
||||
vLogErrorCount NUMBER := 0;
|
||||
vOverallStatus VARCHAR2(20);
|
||||
BEGIN
|
||||
-- Count registered files
|
||||
SELECT COUNT(*)
|
||||
INTO vFileRegCount
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE RECEPTION_DATE >= SYSDATE - 1/24
|
||||
AND SOURCE_FILE_NAME LIKE '200%';
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 1/24;
|
||||
|
||||
-- Count process errors
|
||||
SELECT COUNT(*)
|
||||
INTO vLogErrorCount
|
||||
FROM CT_MRDS.A_PROCESS_LOG
|
||||
@@ -164,8 +188,7 @@ BEGIN
|
||||
AND LOG_LEVEL = 'ERROR'
|
||||
AND LOG_TIMESTAMP >= SYSTIMESTAMP - INTERVAL '1' HOUR;
|
||||
|
||||
-- Determine overall status
|
||||
IF vFileRegCount >= 9 AND vLogErrorCount = 0 THEN
|
||||
IF vFileRegCount >= 6 AND vLogErrorCount = 0 THEN
|
||||
vOverallStatus := 'SUCCESS';
|
||||
ELSIF vFileRegCount > 0 AND vLogErrorCount = 0 THEN
|
||||
vOverallStatus := 'PARTIAL SUCCESS';
|
||||
@@ -174,13 +197,15 @@ BEGIN
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('MARS-1005 Export Verification: ' || vOverallStatus);
|
||||
DBMS_OUTPUT.PUT_LINE('- Registered files: ' || vFileRegCount || ' (expected: 9)');
|
||||
DBMS_OUTPUT.PUT_LINE('- Registered files (last hour): ' || vFileRegCount || ' (expected: >= 6, one per table)');
|
||||
DBMS_OUTPUT.PUT_LINE('- Process errors: ' || vLogErrorCount);
|
||||
|
||||
IF vOverallStatus = 'SUCCESS' THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All validations passed - export successful');
|
||||
ELSIF vOverallStatus = 'PARTIAL SUCCESS' THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Some tables may have incomplete exports - review registrations above');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Some issues detected - review logs');
|
||||
DBMS_OUTPUT.PUT_LINE('ISSUES DETECTED: Review process logs and bucket contents above');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
-- ===================================================================
|
||||
-- MARS-1005 Verify Data Integrity: Source vs Exported Data Validation
|
||||
-- ===================================================================
|
||||
-- Purpose: Verify data integrity between source tables and exported files
|
||||
-- Purpose: Verify data integrity between 6 OU_TOP.LEGACY_* source tables
|
||||
-- and corresponding ODS external tables after export
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-12
|
||||
-- Date: 2026-03-06
|
||||
-- MARS Issue: MARS-1005
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET TIMING ON
|
||||
@@ -12,25 +14,29 @@ PROMPT =========================================================================
|
||||
PROMPT MARS-1005 Data Integrity Verification
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Check 1: Source table record counts vs expected ETL keys
|
||||
PROMPT Checking source table record counts...
|
||||
-- Check 1: Source table record counts
|
||||
PROMPT Checking source table record counts (OU_TOP.LEGACY_* tables)...
|
||||
DECLARE
|
||||
vAdminRows NUMBER := 0;
|
||||
vContentRows NUMBER := 0;
|
||||
vCriterionRows NUMBER := 0;
|
||||
v1Rows NUMBER := 0; v2Rows NUMBER := 0; v3Rows NUMBER := 0;
|
||||
v4Rows NUMBER := 0; v5Rows NUMBER := 0; v6Rows NUMBER := 0;
|
||||
vTotalRows NUMBER := 0;
|
||||
vExpectedFiles NUMBER := 9; -- 3 tables x 3 ETL keys average
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Rows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Rows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Rows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Rows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Rows;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Rows;
|
||||
|
||||
vTotalRows := vAdminRows + vContentRows + vCriterionRows;
|
||||
vTotalRows := v1Rows + v2Rows + v3Rows + v4Rows + v5Rows + v6Rows;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Source table record counts:');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT: ' || vContentRows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Rows || ' records');
|
||||
DBMS_OUTPUT.PUT_LINE('- TOTAL : ' || vTotalRows || ' records');
|
||||
|
||||
IF vTotalRows > 0 THEN
|
||||
@@ -38,130 +44,121 @@ BEGIN
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables');
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: Cannot access source tables: ' || SQLERRM);
|
||||
DBMS_OUTPUT.PUT_LINE('NOTE: Ensure SELECT privilege on OU_TOP.LEGACY_* is granted to CT_MRDS');
|
||||
END;
|
||||
/
|
||||
|
||||
-- Check 2: ETL key distribution analysis
|
||||
PROMPT Checking ETL key distribution...
|
||||
-- Check 2: A_WORKFLOW_HISTORY_KEY distribution across source tables
|
||||
PROMPT Checking A_WORKFLOW_HISTORY_KEY distribution...
|
||||
DECLARE
|
||||
vAdminKeys NUMBER := 0;
|
||||
vContentKeys NUMBER := 0;
|
||||
vCriterionKeys NUMBER := 0;
|
||||
vTotalKeys NUMBER := 0;
|
||||
v1Keys NUMBER := 0; v2Keys NUMBER := 0; v3Keys NUMBER := 0;
|
||||
v4Keys NUMBER := 0; v5Keys NUMBER := 0; v6Keys NUMBER := 0;
|
||||
vDistinctAllKeys NUMBER := 0;
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys;
|
||||
|
||||
SELECT COUNT(DISTINCT etl_key)
|
||||
INTO vTotalKeys
|
||||
SELECT COUNT(DISTINCT wk)
|
||||
INTO vDistinctAllKeys
|
||||
FROM (
|
||||
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
|
||||
SELECT A_WORKFLOW_HISTORY_KEY AS wk FROM OU_TOP.LEGACY_ALLOTMENT UNION ALL
|
||||
SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER UNION ALL
|
||||
SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM UNION ALL
|
||||
SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_ANNOUNCEMENT UNION ALL
|
||||
SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_FBL_ITEM UNION ALL
|
||||
SELECT A_WORKFLOW_HISTORY_KEY FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('ETL key distribution:');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct keys: ' || vAdminKeys);
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct keys: ' || vContentKeys);
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct keys: ' || vCriterionKeys);
|
||||
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys);
|
||||
DBMS_OUTPUT.PUT_LINE('Distinct A_WORKFLOW_HISTORY_KEY values per table:');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys);
|
||||
DBMS_OUTPUT.PUT_LINE('- Total distinct workflow keys (all tables): ' || vDistinctAllKeys);
|
||||
|
||||
IF vTotalKeys > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal');
|
||||
IF vDistinctAllKeys > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Workflow key distribution looks normal');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data');
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: No workflow keys found in source data');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Check 3: Template table compatibility verification
|
||||
PROMPT Checking template table compatibility...
|
||||
PROMPT Checking template table compatibility (CT_ET_TEMPLATES.TOP_*)...
|
||||
DECLARE
|
||||
vAdminCols NUMBER := 0;
|
||||
vContentCols NUMBER := 0;
|
||||
vCriterionCols NUMBER := 0;
|
||||
vCols1 NUMBER := 0; vCols2 NUMBER := 0; vCols3 NUMBER := 0;
|
||||
vCols4 NUMBER := 0; vCols5 NUMBER := 0; vCols6 NUMBER := 0;
|
||||
BEGIN
|
||||
-- Check MPEC_ADMIN template compatibility
|
||||
SELECT COUNT(*)
|
||||
INTO vAdminCols
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_ET_TEMPLATES'
|
||||
AND table_name = 'C2D_MPEC_ADMIN';
|
||||
|
||||
-- Check MPEC_CONTENT template compatibility
|
||||
SELECT COUNT(*)
|
||||
INTO vContentCols
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_ET_TEMPLATES'
|
||||
AND table_name = 'C2D_MPEC_CONTENT';
|
||||
|
||||
-- Check MPEC_CONTENT_CRITERION template compatibility
|
||||
SELECT COUNT(*)
|
||||
INTO vCriterionCols
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_ET_TEMPLATES'
|
||||
AND table_name = 'C2D_MPEC_CONTENT_CRITERION';
|
||||
SELECT COUNT(*) INTO vCols1
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT';
|
||||
SELECT COUNT(*) INTO vCols2
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_HEADER';
|
||||
SELECT COUNT(*) INTO vCols3
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ALLOTMENT_MODIFICATION_ITEM';
|
||||
SELECT COUNT(*) INTO vCols4
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_ANNOUNCEMENT';
|
||||
SELECT COUNT(*) INTO vCols5
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBIDLIST_ITEM';
|
||||
SELECT COUNT(*) INTO vCols6
|
||||
FROM all_tab_columns WHERE owner = 'CT_ET_TEMPLATES' AND table_name = 'TOP_FULLBID_ARRAY_COMPILED';
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Template table column counts:');
|
||||
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_ADMIN: ' || vAdminCols || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT: ' || vContentCols || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT : ' || vCols1 || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_HEADER: ' || vCols2 || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ALLOTMENT_MODIFICATION_ITEM : ' || vCols3 || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_ANNOUNCEMENT : ' || vCols4 || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBIDLIST_ITEM : ' || vCols5 || ' columns');
|
||||
DBMS_OUTPUT.PUT_LINE('- CT_ET_TEMPLATES.TOP_FULLBID_ARRAY_COMPILED : ' || vCols6 || ' columns');
|
||||
|
||||
IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure');
|
||||
IF vCols1 > 0 AND vCols2 > 0 AND vCols3 > 0 AND vCols4 > 0 AND vCols5 > 0 AND vCols6 > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 template tables have defined structure');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Check 4: Verify A_ETL_LOAD_SET_FK values exist in A_LOAD_HISTORY
|
||||
PROMPT Checking ETL key references in A_LOAD_HISTORY...
|
||||
-- Check 4: Verify A_SOURCE_FILE_CONFIG entries for 6 TOP tables
|
||||
PROMPT Checking A_SOURCE_FILE_CONFIG registration for TOP tables...
|
||||
DECLARE
|
||||
vValidKeys NUMBER := 0;
|
||||
vTotalSourceKeys NUMBER := 0;
|
||||
vConfigCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Count total distinct ETL keys in source tables
|
||||
SELECT COUNT(DISTINCT etl_key)
|
||||
INTO vTotalSourceKeys
|
||||
FROM (
|
||||
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
|
||||
);
|
||||
SELECT COUNT(*)
|
||||
INTO vConfigCount
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697); -- MARS-1005 config keys
|
||||
|
||||
-- Count how many exist in A_LOAD_HISTORY
|
||||
SELECT COUNT(DISTINCT etl_key)
|
||||
INTO vValidKeys
|
||||
FROM (
|
||||
SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
|
||||
UNION
|
||||
SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
|
||||
) src
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h
|
||||
WHERE h.A_ETL_LOAD_SET_KEY = src.etl_key
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('A_SOURCE_FILE_CONFIG entries for MARS-1005 tables: ' || vConfigCount || ' (expected: 6)');
|
||||
DBMS_OUTPUT.PUT_LINE('Config keys: 705(ALLOTMENT), 683(MOD_HDR), 684(MOD_ITEM), 689(ANNOUNCEMENT), 696(FBL_ITEM), 697(FBA_COMPILED)');
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('ETL key validation:');
|
||||
DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys in source: ' || vTotalSourceKeys);
|
||||
DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys);
|
||||
|
||||
IF vValidKeys = vTotalSourceKeys THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid');
|
||||
IF vConfigCount = 6 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: All 6 source file config entries confirmed');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys));
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: Missing config entries (' || (6 - vConfigCount) || ' missing)');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT A_SOURCE_FILE_CONFIG details for TOP tables:
|
||||
SELECT
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
SUBSTR(SOURCE_FILE_NAME_PATTERN, 1, 40) AS FILE_PATTERN
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY IN (705, 683, 684, 689, 696, 697)
|
||||
ORDER BY A_SOURCE_FILE_CONFIG_KEY;
|
||||
|
||||
PROMPT =====================================================================================
|
||||
PROMPT MARS-1005 Record Count Verification
|
||||
PROMPT =====================================================================================
|
||||
@@ -190,11 +187,14 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- Initialize table list with C2D MPEC configuration
|
||||
-- Initialize table list with 6 OU_TOP LEGACY table configuration
|
||||
vTables := t_table_list(
|
||||
t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'),
|
||||
t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'),
|
||||
t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)')
|
||||
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT', 'ODS.TOP_ALLOTMENT_ODS', 'ALLOTMENT data (A_SOURCE_FILE_CONFIG_KEY=705)'),
|
||||
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_HEADER', 'ODS.TOP_ALLOTMENT_MODIFICATION_HEADER_ODS', 'MOD HEADER data (A_SOURCE_FILE_CONFIG_KEY=683)'),
|
||||
t_table_info('OU_TOP', 'LEGACY_ALLOTMENT_MODIFICATION_ITEM', 'ODS.TOP_ALLOTMENT_MODIFICATION_ITEM_ODS', 'MOD ITEM data (A_SOURCE_FILE_CONFIG_KEY=684)'),
|
||||
t_table_info('OU_TOP', 'LEGACY_ANNOUNCEMENT', 'ODS.TOP_ANNOUNCEMENT_ODS', 'ANNOUNCEMENT data (A_SOURCE_FILE_CONFIG_KEY=689)'),
|
||||
t_table_info('OU_TOP', 'LEGACY_FBL_ITEM', 'ODS.TOP_FULLBIDLIST_ITEM_ODS', 'FBL ITEM data (A_SOURCE_FILE_CONFIG_KEY=696)'),
|
||||
t_table_info('OU_TOP', 'LEGACY_FULLBID_ARRAY_COMPILED', 'ODS.TOP_FULLBID_ARRAY_COMPILED_ODS', 'FBA COMPILED data (A_SOURCE_FILE_CONFIG_KEY=697)')
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
|
||||
@@ -286,19 +286,19 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- Count exported files for additional verification
|
||||
-- Count MARS-1005 registered export files
|
||||
SELECT COUNT(*)
|
||||
INTO vFileCount
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE RECEPTION_DATE >= SYSDATE - 1/24
|
||||
AND (SOURCE_FILE_NAME LIKE '200_%');
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 1/24;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
|
||||
DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)');
|
||||
DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('Export files registered (PROCESS_NAME=MARS-1005): ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
IF vMismatchCount = 0 AND vFileCount > 0 THEN
|
||||
@@ -325,26 +325,31 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (verify table exists and permissions)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
|
||||
-- Additional ETL key analysis for C2D MPEC data
|
||||
-- Workflow Key Analysis
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('ETL Key Analysis:');
|
||||
DBMS_OUTPUT.PUT_LINE('Workflow Key Analysis (distinct A_WORKFLOW_HISTORY_KEY per table):');
|
||||
|
||||
DECLARE
|
||||
vAdminKeys NUMBER;
|
||||
vContentKeys NUMBER;
|
||||
vCriterionKeys NUMBER;
|
||||
v1Keys NUMBER; v2Keys NUMBER; v3Keys NUMBER;
|
||||
v4Keys NUMBER; v5Keys NUMBER; v6Keys NUMBER;
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT' INTO v1Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER' INTO v2Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM' INTO v3Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_ANNOUNCEMENT' INTO v4Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FBL_ITEM' INTO v5Keys;
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED' INTO v6Keys;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)');
|
||||
DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct ETL keys: ' || vCriterionKeys || ' (expected: 2 for keys 2009-2010)');
|
||||
|
||||
-- Expected file count = sum of distinct ETL keys per table
|
||||
DBMS_OUTPUT.PUT_LINE('- Expected export files: ' || (vAdminKeys + vContentKeys + vCriterionKeys));
|
||||
DBMS_OUTPUT.PUT_LINE('- Actual export files: ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT : ' || v1Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_HEADER: ' || v2Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ALLOTMENT_MODIFICATION_ITEM : ' || v3Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_ANNOUNCEMENT : ' || v4Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FBL_ITEM : ' || v5Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- LEGACY_FULLBID_ARRAY_COMPILED : ' || v6Keys || ' distinct keys');
|
||||
DBMS_OUTPUT.PUT_LINE('- Actual export files registered: ' || vFileCount);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot query workflow keys: ' || SQLERRM);
|
||||
END;
|
||||
END;
|
||||
/
|
||||
|
||||
@@ -1,45 +1,47 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-1005 ROLLBACK: Delete Exported CSV Files from DATA Bucket
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Delete exported CSV files from ODS/C2D bucket folders for MPEC tables
|
||||
-- Purpose: Delete exported CSV files from ODS/TOP/ bucket folders for 6 OU_TOP LEGACY tables
|
||||
-- WARNING: This will permanently delete exported data files!
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-12
|
||||
-- Related: MARS-1005 - C2D MPEC Data Export Rollback
|
||||
-- Date: 2026-03-06
|
||||
-- Related: MARS-1005 - OU_TOP Historical Data Export Rollback
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT ROLLBACK: Deleting C2D_MPEC_ADMIN exported files
|
||||
PROMPT ROLLBACK: Deleting OU_TOP LEGACY exported files from DATA Bucket
|
||||
PROMPT ========================================================================
|
||||
PROMPT WARNING: This will delete files from:
|
||||
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN/
|
||||
PROMPT WARNING: This will delete files registered with PROCESS_NAME = 'MARS-1005'
|
||||
PROMPT from ODS/TOP/* paths in the DATA bucket.
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- Helper: generic delete procedure for one TOP table folder
|
||||
-- Deletes objects whose SOURCE_FILE_NAME matches the folder prefix pattern
|
||||
|
||||
-- ROLLBACK TABLE 1/6: LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT/
|
||||
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Get bucket URI and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT files from: ' || vDataBucketUri || vFolderPath);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_ADMIN files from DATA bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
|
||||
|
||||
-- Delete CSV files registered by MARS-1005 process
|
||||
FOR rec IN (
|
||||
SELECT SOURCE_FILE_NAME AS object_name
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND SOURCE_FILE_NAME LIKE '%MPEC_ADMIN%'
|
||||
SELECT object_name
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
))
|
||||
WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => vCredentialName,
|
||||
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_ADMIN/' || rec.object_name
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
object_uri => vDataBucketUri || vFolderPath || rec.object_name
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
@@ -47,8 +49,7 @@ BEGIN
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE
|
||||
RAISE;
|
||||
ELSE RAISE;
|
||||
END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
@@ -56,111 +57,152 @@ BEGIN
|
||||
IF vFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_ADMIN files deleted (' || vFileCount || ' file(s))');
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT exported files
|
||||
PROMPT ========================================================================
|
||||
PROMPT WARNING: This will delete files from:
|
||||
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT/
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- ROLLBACK TABLE 2/6: LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/
|
||||
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Get bucket URI and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT files from DATA bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
|
||||
|
||||
-- Delete CSV files registered by MARS-1005 process
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_HEADER files...');
|
||||
FOR rec IN (
|
||||
SELECT SOURCE_FILE_NAME AS object_name
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT%'
|
||||
AND SOURCE_FILE_NAME NOT LIKE '%CRITERION%'
|
||||
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
)) WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => vCredentialName,
|
||||
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT/' || rec.object_name
|
||||
);
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE RAISE; END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
IF vFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT files deleted (' || vFileCount || ' file(s))');
|
||||
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_HEADER files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT ROLLBACK: Deleting C2D_MPEC_CONTENT_CRITERION exported files
|
||||
PROMPT ========================================================================
|
||||
PROMPT WARNING: This will delete files from:
|
||||
PROMPT - DATA bucket: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION/
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- ROLLBACK TABLE 3/6: LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/
|
||||
PROMPT ROLLBACK: Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Get bucket URI and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting C2D_MPEC_CONTENT_CRITERION files from DATA bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using A_SOURCE_FILE_RECEIVED with PROCESS_NAME = ''MARS-1005''');
|
||||
|
||||
-- Delete CSV files registered by MARS-1005 process
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ALLOTMENT_MODIFICATION_ITEM files...');
|
||||
FOR rec IN (
|
||||
SELECT SOURCE_FILE_NAME AS object_name
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND SOURCE_FILE_NAME LIKE '%MPEC_CONTENT_CRITERION%'
|
||||
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
)) WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => vCredentialName,
|
||||
object_uri => vDataBucketUri || 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION/' || rec.object_name
|
||||
);
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE RAISE; END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ALLOTMENT_MODIFICATION_ITEM files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
IF vFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete');
|
||||
END IF;
|
||||
-- ROLLBACK TABLE 4/6: LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT/
|
||||
PROMPT ROLLBACK: Deleting TOP_ANNOUNCEMENT files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_ANNOUNCEMENT/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_ANNOUNCEMENT files...');
|
||||
FOR rec IN (
|
||||
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
)) WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE RAISE; END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_ANNOUNCEMENT files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: C2D_MPEC_CONTENT_CRITERION files deleted (' || vFileCount || ' file(s))');
|
||||
-- ROLLBACK TABLE 5/6: LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM/
|
||||
PROMPT ROLLBACK: Deleting TOP_FULLBIDLIST_ITEM files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBIDLIST_ITEM/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBIDLIST_ITEM files...');
|
||||
FOR rec IN (
|
||||
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
)) WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE RAISE; END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBIDLIST_ITEM files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
-- ROLLBACK TABLE 6/6: LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/
|
||||
PROMPT ROLLBACK: Deleting TOP_FULLBID_ARRAY_COMPILED files...
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vFolderPath VARCHAR2(200) := 'ODS/TOP/TOP_FULLBID_ARRAY_COMPILED/';
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting TOP_FULLBID_ARRAY_COMPILED files...');
|
||||
FOR rec IN (
|
||||
SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vDataBucketUri || vFolderPath
|
||||
)) WHERE object_name NOT LIKE '%/'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => 'OCI$RESOURCE_PRINCIPAL', object_uri => vDataBucketUri || vFolderPath || rec.object_name);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE RAISE; END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
IF vFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE(' INFO: No files found to delete'); END IF;
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: TOP_FULLBID_ARRAY_COMPILED files deleted (' || vFileCount || ' file(s))');
|
||||
END;
|
||||
/
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ BEGIN
|
||||
SELECT COUNT(*)
|
||||
INTO vRemainingFiles
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE SOURCE_FILE_NAME LIKE '200%' -- ETL keys 2001-2010
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 7; -- Last week
|
||||
|
||||
IF vRemainingFiles = 0 THEN
|
||||
@@ -33,7 +33,7 @@ BEGIN
|
||||
SELECT SUBSTR(SOURCE_FILE_NAME, 1, 50) AS FILE_NAME,
|
||||
TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS RECEIVED_TIME
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE SOURCE_FILE_NAME LIKE '200%'
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 7
|
||||
) LOOP
|
||||
DBMS_OUTPUT.PUT_LINE(' Remaining: ' || rec.FILE_NAME);
|
||||
@@ -96,7 +96,7 @@ BEGIN
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri
|
||||
))
|
||||
WHERE object_name LIKE 'ODS/C2D/C2D_MPEC_%'
|
||||
WHERE object_name LIKE 'ODS/TOP/%'
|
||||
) LOOP
|
||||
vCloudFileCount := vCloudFileCount + 1;
|
||||
IF vCloudFileCount <= 3 THEN -- Show first 3 files
|
||||
@@ -105,11 +105,11 @@ BEGIN
|
||||
END LOOP;
|
||||
|
||||
IF vCloudFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: No C2D MPEC files found in cloud bucket');
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: No TOP files found in cloud bucket - clean');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' C2D MPEC files still in cloud bucket');
|
||||
DBMS_OUTPUT.PUT_LINE('INFO: ' || vCloudFileCount || ' TOP file(s) still in cloud bucket');
|
||||
DBMS_OUTPUT.PUT_LINE(' Note: Cloud files are not automatically deleted by rollback');
|
||||
DBMS_OUTPUT.PUT_LINE(' Manual deletion required if needed');
|
||||
DBMS_OUTPUT.PUT_LINE(' Run 90_MARS_1005_rollback_delete_csv_files.sql to remove them');
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
@@ -167,7 +167,7 @@ BEGIN
|
||||
SELECT COUNT(*)
|
||||
INTO vRemainingFiles
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE SOURCE_FILE_NAME LIKE '200%'
|
||||
WHERE PROCESS_NAME = 'MARS-1005'
|
||||
AND RECEPTION_DATE >= SYSDATE - 7;
|
||||
|
||||
-- Count remaining process logs
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
-- ===================================================================
|
||||
-- MARS-1005 INSTALL SCRIPT: C2D MPEC Data Export to External Tables
|
||||
-- MARS-1005 INSTALL SCRIPT: OU_TOP Historical Data Export to ODS Bucket
|
||||
-- ===================================================================
|
||||
-- Purpose: One-time bulk export of 3 C2D MPEC tables from OU_LEGACY_C2D schema
|
||||
-- to OCI buckets (ODS bucket CSV format)
|
||||
-- Uses DATA_EXPORTER v2.7.5 with pRegisterExport for file registration
|
||||
-- Purpose: One-time bulk export of 6 OU_TOP LEGACY tables to OCI DATA bucket
|
||||
-- (ODS bucket area, CSV format)
|
||||
-- Uses DATA_EXPORTER EXPORT_TABLE_DATA with pRegisterExport for file tracking
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-12
|
||||
-- Date: 2026-03-06
|
||||
|
||||
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
|
||||
-- Log files are automatically created in log/ subdirectory
|
||||
@@ -30,20 +30,23 @@ SET PAUSE OFF
|
||||
-- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS;
|
||||
|
||||
PROMPT =========================================================================
|
||||
PROMPT MARS-1005: C2D MPEC Data Export to External Tables (One-Time Migration)
|
||||
PROMPT MARS-1005: OU_TOP Historical Data Export to ODS Bucket (One-Time Migration)
|
||||
PROMPT =========================================================================
|
||||
PROMPT
|
||||
PROMPT This script will export 3 C2D MPEC tables to OCI buckets:
|
||||
PROMPT This script will export 6 OU_TOP LEGACY tables to OCI DATA bucket:
|
||||
PROMPT
|
||||
PROMPT TARGET: ODS Bucket (CSV format):
|
||||
PROMPT - MPEC_ADMIN
|
||||
PROMPT - MPEC_CONTENT
|
||||
PROMPT - MPEC_CONTENT_CRITERION
|
||||
PROMPT TARGET: DATA Bucket / ODS area (CSV format):
|
||||
PROMPT - OU_TOP.LEGACY_ALLOTMENT -> ODS/TOP/TOP_ALLOTMENT
|
||||
PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_HEADER -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_HEADER
|
||||
PROMPT - OU_TOP.LEGACY_ALLOTMENT_MODIFICATION_ITEM -> ODS/TOP/TOP_ALLOTMENT_MODIFICATION_ITEM
|
||||
PROMPT - OU_TOP.LEGACY_ANNOUNCEMENT -> ODS/TOP/TOP_ANNOUNCEMENT
|
||||
PROMPT - OU_TOP.LEGACY_FBL_ITEM -> ODS/TOP/TOP_FULLBIDLIST_ITEM
|
||||
PROMPT - OU_TOP.LEGACY_FULLBID_ARRAY_COMPILED -> ODS/TOP/TOP_FULLBID_ARRAY_COMPILED
|
||||
PROMPT
|
||||
PROMPT Key Features:
|
||||
PROMPT - Files registered in A_SOURCE_FILE_RECEIVED for tracking
|
||||
PROMPT - Template table column order matching (CT_ET_TEMPLATES.C2D_MPEC_*)
|
||||
PROMPT - ODS/C2D bucket path structure
|
||||
PROMPT - Files registered in A_SOURCE_FILE_RECEIVED with PROCESS_NAME = 'MARS-1005'
|
||||
PROMPT - Template table column order matching (CT_ET_TEMPLATES.TOP_*)
|
||||
PROMPT - ODS/TOP bucket path structure
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Confirm installation with user
|
||||
|
||||
@@ -92,3 +92,8 @@ cd .\MARS_Packages\REL02_POST\MARS-1409
|
||||
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1409.sql"
|
||||
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1409.sql"
|
||||
7z a -pMojeSuperHaslo#123 -mhe=on M1409_arch.7z MARS-1409
|
||||
|
||||
cd .\MARS_Packages\REL03\MARS-1005
|
||||
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars1005.sql"
|
||||
echo 'yes' | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars1005.sql"
|
||||
7z a -pMojeSuperHaslo#123 -mhe=on M1005_arch.7z MARS-1005
|
||||
Reference in New Issue
Block a user