admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(MARS-835): Update export scripts to support HIST-only strategy, including verification and rollback adjustments

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-24 19:18:16 +01:00
parent ca5d8b320c
commit 04d4f6ac02
5 changed files with 76 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql
View File

@@ -1,125 +1,31 @@
--=============================================================================================================================
-- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY)
-- MARS-835: Export Group 1 - HIST Only (DEBT, DEBT_DAILY)
--=============================================================================================================================
-- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet)
-- Purpose: Export ALL data to HIST bucket (Parquet with Hive-style partitioning)
-- Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
-- Excludes legacy columns not required in new structure
-- USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats
-- USES: DATA_EXPORTER v2.12.0 with pTemplateTableName for column order and date formats
-- Author: Grzegorz Michalski
-- Date: 2025-12-17
-- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName)
-- Updated: 2026-02-24 (Changed to HIST-only export, no DATA bucket split)
-- Related: MARS-835 - CSDB Data Export
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"
PROMPT ========================================================================
PROMPT Exporting CSDB.DEBT - Split DATA + HIST
PROMPT Exporting CSDB.DEBT - HIST Only
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION
PROMPT ========================================================================
-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
-- Export ALL data to HIST bucket (Parquet)
-- NEW v2.12.0: Per-column date format handling with template table, full data range
BEGIN
-- Get bucket URI for DATA bucket
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/';
-- Count existing files
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'; -- Exclude directories
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
-- List existing files
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
-- Count records in external table
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
pSchemaName => 'OU_CSDB',
pTableName => 'LEGACY_DEBT',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'DATA',
pFolderName => 'ODS/CSDB/CSDB_DEBT',
pMinDate => &cutoff_date,
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
pProcessName => 'MARS-835', -- Process identifier for tracking
pJobClass => 'high' -- Oracle Scheduler job class for resource management
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
END;
/
-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...');
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (ALL data)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -128,7 +34,8 @@ BEGIN
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
pMaxDate => &cutoff_date,
pMinDate => DATE '1900-01-01', -- Include all historical data
pMaxDate => DATE '9999-12-31', -- Include all future dates
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
pJobClass => 'high' -- Oracle Scheduler job class for resource management
@@ -139,110 +46,18 @@ END;
/
PROMPT ========================================================================
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - HIST Only
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS,
PROMPT ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS
PROMPT ========================================================================
-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
-- Export ALL data to HIST bucket (Parquet)
-- NEW v2.12.0: Per-column date format handling with template table, full data range
BEGIN
-- Get bucket URI for DATA bucket
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/';
-- Count existing files
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'; -- Exclude directories
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
-- List existing files
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
-- Count records in external table
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
pSchemaName => 'OU_CSDB',
pTableName => 'LEGACY_DEBT_DAILY',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'DATA',
pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
pMinDate => &cutoff_date,
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
pProcessName => 'MARS-835', -- Process identifier for tracking
pJobClass => 'high' -- Oracle Scheduler job class for resource management
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
END;
/
-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...');
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (ALL data)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -251,7 +66,8 @@ BEGIN
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
pMaxDate => &cutoff_date,
pMinDate => DATE '1900-01-01', -- Include all historical data
pMaxDate => DATE '9999-12-31', -- Include all future dates
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
pJobClass => 'high' -- Oracle Scheduler job class for resource management
@@ -264,8 +80,8 @@ END;
PROMPT ========================================================================
PROMPT Group 1 Export Completed
PROMPT ========================================================================
PROMPT - LEGACY_DEBT: DATA + HIST exported
PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported
PROMPT - LEGACY_DEBT: HIST exported (ALL data)
PROMPT - LEGACY_DEBT_DAILY: HIST exported (ALL data)
PROMPT ========================================================================
--=============================================================================================================================

84
MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 03_MARS_835_verify_exports.sql
-- Purpose: Verify exported files exist in DATA and HIST buckets after export
-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/
-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,17 +14,14 @@ SET VERIFY OFF;
SET LINESIZE 200;
PROMPT =====================================================================================
PROMPT MARS-835 Verification: Listing exported files in DATA and HIST buckets
PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy)
PROMPT =====================================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
vTotalDataFiles NUMBER := 0;
vTotalHistFiles NUMBER := 0;
vTotalDataSize NUMBER := 0;
vTotalHistSize NUMBER := 0;
TYPE t_folder_info IS RECORD (
@@ -33,25 +31,18 @@ DECLARE
);
TYPE t_folder_list IS TABLE OF t_folder_info;
vDataFolders t_folder_list;
vHistFolders t_folder_list;
BEGIN
-- Get bucket URIs and credential from FILE_MANAGER
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential from FILE_MANAGER
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
DBMS_OUTPUT.PUT_LINE('');
-- Initialize folder lists
vDataFolders := t_folder_list(
t_folder_info('ODS/CSDB/CSDB_DEBT/', 'DEBT', 'CSV'),
t_folder_info('ODS/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'CSV')
);
-- Initialize folder list (all tables in HIST)
-- Initialize folder list (all 6 tables in HIST)
vHistFolders := t_folder_list(
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'),
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'),
@@ -62,49 +53,7 @@ BEGIN
);
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket Exports (CSV format - last 6 months)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check DATA bucket exports
FOR i IN 1..vDataFolders.COUNT LOOP
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Table: ' || vDataFolders(i).table_name || ' (' || vDataFolders(i).expected_format || ')');
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i).folder_name);
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
BEGIN
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || vDataFolders(i).folder_name
))
WHERE object_name LIKE '%.csv'
ORDER BY created DESC
) LOOP
vFileCount := vFileCount + 1;
vTotalDataFiles := vTotalDataFiles + 1;
vTotalDataSize := vTotalDataSize + rec.bytes;
DBMS_OUTPUT.PUT_LINE(' [' || vFileCount || '] ' || rec.object_name ||
' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' [ERROR] No CSV files found - Export may have failed!');
ELSE
DBMS_OUTPUT.PUT_LINE(' [SUCCESS] Found ' || vFileCount || ' CSV file(s)');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE(' [ERROR] Cannot access folder - ' || SQLERRM);
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning)');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check HIST bucket exports
@@ -155,24 +104,19 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Export Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('DATA Bucket (CSV):');
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalDataFiles);
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalDataSize/1024/1024/1024, 2) || ' GB');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 2 (DEBT, DEBT_DAILY - last 6 months)');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet):');
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):');
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalHistFiles || '+');
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables with historical data)');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables exported to HIST)');
DBMS_OUTPUT.PUT_LINE('');
IF vTotalDataFiles >= 2 AND vTotalHistFiles >= 6 THEN
IF vTotalHistFiles >= 6 THEN
DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL');
DBMS_OUTPUT.PUT_LINE(' Files found in both DATA and HIST buckets');
DBMS_OUTPUT.PUT_LINE(' Files found in HIST bucket for all tables');
DBMS_OUTPUT.PUT_LINE(' Proceed to record count verification (Step 4)');
ELSIF vTotalDataFiles = 0 AND vTotalHistFiles = 0 THEN
ELSIF vTotalHistFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED');
DBMS_OUTPUT.PUT_LINE(' No files found in either bucket');
DBMS_OUTPUT.PUT_LINE(' No files found in HIST bucket');
DBMS_OUTPUT.PUT_LINE(' Review export logs for errors');
ELSE
DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected');

105
MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 04_MARS_835_verify_record_counts.sql
-- Purpose: Verify record counts match between source tables and exported data
-- Purpose: Verify record counts match between source tables and exported data (HIST-only)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Verification: Compare OU_CSDB source tables with ODS external tables
-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only)
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,28 +14,23 @@ SET VERIFY OFF;
SET LINESIZE 200;
PROMPT =====================================================================================
PROMPT MARS-835 Record Count Verification
PROMPT MARS-835 Record Count Verification (HIST-only strategy)
PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT Comparing source table counts with HIST external table counts
PROMPT =====================================================================================
DECLARE
TYPE t_table_info IS RECORD (
source_schema VARCHAR2(50),
source_table VARCHAR2(100),
data_external_table VARCHAR2(100),
hist_external_table VARCHAR2(100),
has_data_export BOOLEAN,
has_hist_export BOOLEAN
hist_external_table VARCHAR2(100)
);
TYPE t_table_list IS TABLE OF t_table_info;
vTables t_table_list;
vSourceCount NUMBER;
vDataCount NUMBER;
vHistCount NUMBER;
vTotalSourceCount NUMBER := 0;
vTotalDataCount NUMBER := 0;
vTotalHistCount NUMBER := 0;
vMismatchCount NUMBER := 0;
vSql VARCHAR2(4000);
@@ -42,18 +38,18 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
DBMS_OUTPUT.PUT_LINE('');
-- Initialize table list with export configuration
-- Initialize table list (all tables HIST-only)
vTables := t_table_list(
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ODS', 'ODS.CSDB_DEBT_ARCHIVE', TRUE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ODS', 'ODS.CSDB_DEBT_DAILY_ARCHIVE', TRUE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', NULL, 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', NULL, 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', NULL, 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE)
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE')
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP
@@ -70,31 +66,6 @@ BEGIN
CONTINUE;
END;
-- Get DATA external table count (if applicable)
IF vTables(i).has_data_export THEN
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).data_external_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vDataCount;
vTotalDataCount := vTotalDataCount + vDataCount;
EXCEPTION
WHEN OTHERS THEN
-- If source table is empty (0 records), no files were exported
-- External table returns error, treat as 0
-- Acceptable error codes:
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vDataCount := 0;
ELSE
vDataCount := -1;
END IF;
END;
ELSE
vDataCount := NULL;
END IF;
-- Get HIST external table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table;
BEGIN
@@ -119,18 +90,8 @@ BEGIN
-- Display results
DECLARE
vStatus VARCHAR2(20);
vDataDisplay VARCHAR2(17);
vHistDisplay VARCHAR2(17);
BEGIN
-- Format DATA count display
IF vDataCount IS NULL THEN
vDataDisplay := 'N/A';
ELSIF vDataCount = -1 THEN
vDataDisplay := 'ERROR';
ELSE
vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999');
END IF;
-- Format HIST count display
IF vHistCount = -1 THEN
vHistDisplay := 'ERROR';
@@ -138,35 +99,20 @@ BEGIN
vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
END IF;
-- Determine status
IF vTables(i).has_data_export THEN
-- Split export: check DATA + HIST = SOURCE
IF (vDataCount + vHistCount) = vSourceCount THEN
vStatus := 'PASS';
ELSIF vDataCount = -1 OR vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
-- Determine status (HIST only: check HIST = SOURCE)
IF vHistCount = vSourceCount THEN
vStatus := 'PASS';
ELSIF vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
-- HIST only: check HIST = SOURCE
IF vHistCount = vSourceCount THEN
vStatus := 'PASS';
ELSIF vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
LPAD(vDataDisplay, 15) ||
LPAD(vHistDisplay, 15) || ' ' ||
vStatus
);
@@ -177,18 +123,16 @@ BEGIN
DBMS_OUTPUT.PUT_LINE(
RPAD('TOTALS', 24) ||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)');
DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 THEN
@@ -209,7 +153,6 @@ BEGIN
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (may not exist yet)');
DBMS_OUTPUT.PUT_LINE(' N/A - Not applicable (table not exported to DATA)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
EXCEPTION

85
MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql
View File

@@ -1,68 +1,34 @@
--=============================================================================================================================
-- MARS-835 ROLLBACK: Delete Group 1 Exported Files (DEBT, DEBT_DAILY)
--=============================================================================================================================
-- Purpose: Delete exported CSV and Parquet files from DATA and HIST buckets
-- Purpose: Delete exported Parquet files from HIST bucket (ARCHIVE only)
-- WARNING: This will permanently delete exported data files!
-- Author: Grzegorz Michalski
-- Date: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only rollback, no DATA bucket)
-- Related: MARS-835 - CSDB Data Export Rollback
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting DEBT exported files
PROMPT ROLLBACK: Deleting DEBT exported files from HIST
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT/
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URIs and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
))
WHERE object_name LIKE 'LEGACY_DEBT%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
FOR rec IN (
@@ -99,58 +65,23 @@ END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files from HIST
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URIs and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
))
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
FOR rec IN (

73
MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 99_MARS_835_verify_rollback.sql
-- Purpose: Verify all exported files have been deleted from DATA and HIST buckets
-- Purpose: Verify all exported files have been deleted from HIST bucket (HIST-only strategy)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Verification: Confirm complete rollback (no CSDB files remaining)
-- Verification: Confirm complete rollback (no CSDB files remaining in HIST)
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -19,33 +20,23 @@ PROMPT Checking that all CSDB export files have been deleted
PROMPT =====================================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vDataFileCount NUMBER := 0;
vHistFileCount NUMBER := 0;
vTotalFiles NUMBER := 0;
TYPE t_folder_list IS TABLE OF VARCHAR2(200);
vDataFolders t_folder_list;
vHistFolders t_folder_list;
BEGIN
-- Get bucket URIs
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('ROLLBACK VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
DBMS_OUTPUT.PUT_LINE('');
-- Initialize folder lists
vDataFolders := t_folder_list(
'ODS/CSDB/CSDB_DEBT/',
'ODS/CSDB/CSDB_DEBT_DAILY/'
);
-- Initialize folder list (all 6 tables in HIST)
-- Initialize folder list (all 6 tables in HIST)
vHistFolders := t_folder_list(
'ARCHIVE/CSDB/CSDB_DEBT/',
'ARCHIVE/CSDB/CSDB_DEBT_DAILY/',
@@ -55,47 +46,6 @@ BEGIN
'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/'
);
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (should be empty)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check DATA bucket
FOR i IN 1..vDataFolders.COUNT LOOP
DECLARE
vCount NUMBER := 0;
BEGIN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i));
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || vDataFolders(i)
))
WHERE object_name LIKE '%.csv'
) LOOP
vCount := vCount + 1;
vDataFileCount := vDataFileCount + 1;
DBMS_OUTPUT.PUT_LINE(' [FOUND] ' || rec.object_name);
END LOOP;
IF vCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' [OK] No CSV files found');
ELSE
DBMS_OUTPUT.PUT_LINE(' [INFO] Found ' || vCount || ' file(s) - may be pre-existing files from before installation');
END IF;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' [OK] Folder does not exist or is empty');
ELSE
DBMS_OUTPUT.PUT_LINE(' [ERROR] ' || SQLERRM);
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (should be empty)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
@@ -139,24 +89,21 @@ BEGIN
END;
END LOOP;
vTotalFiles := vDataFileCount + vHistFileCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Rollback Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('DATA bucket files remaining: ' || vDataFileCount);
DBMS_OUTPUT.PUT_LINE('HIST bucket files remaining: ' || vHistFileCount || '+');
DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles || '+');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('');
IF vTotalFiles = 0 THEN
IF vHistFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('[PASSED] ROLLBACK VERIFICATION PASSED');
DBMS_OUTPUT.PUT_LINE(' All CSDB export files have been deleted or were not created');
DBMS_OUTPUT.PUT_LINE(' Buckets are clean and ready for re-export if needed');
DBMS_OUTPUT.PUT_LINE(' HIST bucket is clean and ready for re-export if needed');
ELSE
DBMS_OUTPUT.PUT_LINE('[INFO] ROLLBACK VERIFICATION COMPLETED');
DBMS_OUTPUT.PUT_LINE(' Found ' || vTotalFiles || '+ file(s) remaining in buckets');
DBMS_OUTPUT.PUT_LINE(' Found ' || vHistFileCount || '+ file(s) remaining in HIST bucket');
DBMS_OUTPUT.PUT_LINE(' NOTE: These may be pre-existing files from before installation.');
DBMS_OUTPUT.PUT_LINE(' Rollback only deletes files created during this export operation.');
DBMS_OUTPUT.PUT_LINE(' If needed, manually verify and clean up remaining files.');