admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge develop into main - DATA_EXPORTER v2.14.0 optimization

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-26 20:39:33 +01:00
parent 60b218d211 c68d5bfe2c
commit e9d4056451
16 changed files with 264 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql
View File

@@ -25,6 +25,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported');
@@ -67,6 +69,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported');

2
MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql
View File

@@ -30,6 +30,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER',
pParallelDegree => 4,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported');
@@ -51,6 +52,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM',
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported');

3
MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql
View File

@@ -25,6 +25,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM',
pParallelDegree => 2,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported');
@@ -67,6 +69,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 2,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported');

2
MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql
View File

@@ -30,6 +30,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES',
pParallelDegree => 8,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported');
@@ -51,6 +52,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER',
pParallelDegree => 2,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported');

2
MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql
View File

@@ -26,6 +26,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER',
pParallelDegree => 2,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported');
@@ -47,6 +48,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM',
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported');

2
MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql
View File

@@ -30,6 +30,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER',
pParallelDegree => 4,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported');
@@ -51,6 +52,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM',
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported');

5
MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql
View File

@@ -25,6 +25,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM',
pParallelDegree => 4,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported');
@@ -66,8 +68,7 @@ BEGIN
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 2,
pJobClass => 'high'
pParallelDegree => 2, pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported');
EXCEPTION

2
MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql
View File

@@ -25,6 +25,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_HEADER',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM',
pParallelDegree => 1,
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_ITEM',
pJobClass => 'high'
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported');

14
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql
View File

@@ -26,7 +26,7 @@ END;
/
CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
CHUNK_ID NUMBER PRIMARY KEY,
CHUNK_ID NUMBER NOT NULL,
TASK_NAME VARCHAR2(100) NOT NULL,
YEAR_VALUE VARCHAR2(4) NOT NULL,
MONTH_VALUE VARCHAR2(2) NOT NULL,
@@ -47,14 +47,16 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL,
ERROR_MESSAGE VARCHAR2(4000),
EXPORT_TIMESTAMP TIMESTAMP,
CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL
CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL,
CONSTRAINT PK_PARALLEL_EXPORT_CHUNKS PRIMARY KEY (TASK_NAME, CHUNK_ID)
);
CREATE INDEX IX_PARALLEL_CHUNKS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(TASK_NAME);
-- Index for status-based queries (e.g., WHERE STATUS = 'FAILED' AND TASK_NAME = ?)
CREATE INDEX IX_PARALLEL_CHUNKS_STATUS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(STATUS, TASK_NAME);
COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Unique chunk identifier (partition number)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name for cleanup';
COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions. PK: (TASK_NAME, CHUNK_ID) ensures session isolation for concurrent exports.';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Chunk identifier within task (partition number) - unique per TASK_NAME, not globally';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name - session isolation key, part of composite PK with CHUNK_ID';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.YEAR_VALUE IS 'Partition year (YYYY)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MONTH_VALUE IS 'Partition month (MM)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.SCHEMA_NAME IS 'Schema owning the source table';

214
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb
View File

@@ -18,34 +18,104 @@ AS
----------------------------------------------------------------------------------------------------
/**
* Deletes export file from OCI bucket if it exists (used for cleanup before retry)
* Silently ignores if file doesn't exist (ORA-20404)
* Deletes ALL files matching specific file pattern before retry export
* Critical for preventing data duplication when DBMS_CLOUD.EXPORT_DATA fails mid-process
*
* Problem: Export fails after creating partial file(s), retry creates new _2, _3 suffixed files
* Solution: Delete ALL files matching the base filename pattern before retry
*
* Pattern matching strategy:
* - Parquet: folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/*.parquet (folder-level safe - each chunk has own partition folder)
* - CSV: folder/TABLENAME_202411*.csv (file-level pattern - multiple chunks share same folder!)
*
* CRITICAL for parallel processing:
* - Parquet chunks are isolated by partition folder structure (safe to delete folder/*)
* - CSV chunks share flat folder structure - MUST use file-specific pattern (TABLENAME_YYYYMM*)
* to avoid deleting files from other parallel chunks in same folder
**/
PROCEDURE DELETE_FAILED_EXPORT_FILE(
pFileUri IN VARCHAR2,
pCredentialName IN VARCHAR2,
pParameters IN VARCHAR2
) IS
vBucketUri VARCHAR2(4000);
vFolderPath VARCHAR2(4000);
vFileName VARCHAR2(1000);
vFileNamePattern VARCHAR2(1000);
vSlashPos NUMBER;
vDotPos NUMBER;
vFilesDeleted NUMBER := 0;
BEGIN
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to delete potentially corrupted file: ' || pFileUri, 'DEBUG', pParameters);
-- Extract components from URI
-- Example Parquet: https://.../bucket/folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/202411.parquet
-- Example CSV: https://.../bucket/folder/TABLENAME_202411.csv
DBMS_CLOUD.DELETE_OBJECT(
credential_name => pCredentialName,
object_uri => pFileUri
);
-- Find last slash before filename
vSlashPos := INSTR(pFileUri, '/', -1);
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted existing file (cleanup before retry): ' || pFileUri, 'INFO', pParameters);
EXCEPTION
WHEN OTHERS THEN
-- Object not found is OK (file doesn't exist)
IF SQLCODE = -20404 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('File does not exist (OK): ' || pFileUri, 'DEBUG', pParameters);
ELSE
-- Log but don't fail - export will attempt anyway
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete file (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
END IF;
END;
IF vSlashPos > 0 THEN
-- Extract filename from URI (after last slash)
vFileName := SUBSTR(pFileUri, vSlashPos + 1);
-- Extract folder path (before last slash)
vFolderPath := SUBSTR(pFileUri, 1, vSlashPos - 1);
-- Find bucket URI (protocol + namespace + bucket name)
-- Bucket URI ends after /o/ in OCI Object Storage URLs
vBucketUri := SUBSTR(pFileUri, 1, INSTR(pFileUri, '/o/') + 2);
-- Extract relative folder path (after bucket)
vFolderPath := SUBSTR(vFolderPath, LENGTH(vBucketUri) + 1);
-- Create file pattern by removing extension
-- Oracle adds suffixes BEFORE extension: file.csv -> file_1_timestamp.csv
-- Pattern: file* matches file_1_timestamp.csv, file_2_timestamp.csv
vDotPos := INSTR(vFileName, '.', -1);
IF vDotPos > 0 THEN
vFileNamePattern := SUBSTR(vFileName, 1, vDotPos - 1) || '%';
ELSE
vFileNamePattern := vFileName || '%';
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup before retry - Pattern: ' || vFolderPath || '/' || vFileNamePattern, 'DEBUG', pParameters);
-- List and delete ALL files matching pattern
-- CRITICAL: Uses file-specific pattern for CSV chunk isolation in shared folder
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => pCredentialName,
location_uri => vBucketUri
))
WHERE object_name LIKE vFolderPath || '/' || vFileNamePattern
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => pCredentialName,
object_uri => vBucketUri || rec.object_name
);
vFilesDeleted := vFilesDeleted + 1;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted partial file ' || vFilesDeleted || ': ' || rec.object_name, 'DEBUG', pParameters);
EXCEPTION
WHEN OTHERS THEN
-- Log but continue - don't fail entire cleanup
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete ' || rec.object_name || ': ' || SQLERRM, 'WARNING', pParameters);
END;
END LOOP;
IF vFilesDeleted > 0 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup completed: Deleted ' || vFilesDeleted || ' partial file(s) from previous failed export', 'INFO', pParameters);
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('No existing files to clean up (pattern match: ' || vFileNamePattern || ')', 'DEBUG', pParameters);
END IF;
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cannot parse file URI for cleanup: ' || pFileUri, 'WARNING', pParameters);
END IF;
EXCEPTION
WHEN OTHERS THEN
-- Don't fail export if cleanup fails - log and continue
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cleanup failed (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
END DELETE_FAILED_EXPORT_FILE;
----------------------------------------------------------------------------------------------------
@@ -415,6 +485,8 @@ AS
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: '||pFormat||')', 'DEBUG', pParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
-- Construct the URI based on format
IF pFormat = 'PARQUET' THEN
-- Parquet: Use Hive-style partitioning
@@ -425,6 +497,7 @@ AS
'PARTITION_MONTH=' || sanitizeFilename(pMonth) || '/' ||
sanitizeFilename(pYear) || sanitizeFilename(pMonth) || '.parquet';
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters);
-- Delete potentially corrupted file from previous failed attempt
@@ -445,6 +518,7 @@ AS
sanitizeFilename(vFileName);
ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('CSV maxfilesize: ' || pMaxFileSize || ' bytes (' || ROUND(pMaxFileSize/1048576, 2) || ' MB)', 'DEBUG', pParameters);
-- Delete potentially corrupted file from previous failed attempt
-- This prevents Oracle from creating _1 suffixed files on retry
@@ -472,8 +546,7 @@ AS
RAISE_APPLICATION_ERROR(-20001, 'Unsupported format: ' || pFormat || '. Use PARQUET or CSV.');
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: ' || pFormat || ')', 'DEBUG', pParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || pYear || '/' || pMonth, 'DEBUG', pParameters);
END EXPORT_SINGLE_PARTITION;
----------------------------------------------------------------------------------------------------
@@ -485,7 +558,8 @@ AS
**/
PROCEDURE EXPORT_PARTITION_PARALLEL (
pStartId IN NUMBER,
pEndId IN NUMBER
pEndId IN NUMBER,
pTaskName IN VARCHAR2 DEFAULT NULL
) IS
vYear VARCHAR2(4);
vMonth VARCHAR2(2);
@@ -502,9 +576,12 @@ AS
vFileBaseName VARCHAR2(1000);
vMaxFileSize NUMBER;
vJobClass VARCHAR2(128);
vTaskName VARCHAR2(128);
vParameters VARCHAR2(4000);
BEGIN
-- Retrieve chunk context from global temporary table
-- Retrieve chunk context from A_PARALLEL_EXPORT_CHUNKS table
-- CRITICAL: Filter by CHUNK_ID and TASK_NAME for precise session isolation
-- pTaskName parameter passed from RUN_TASK ensures deterministic single-row retrieval
SELECT
YEAR_VALUE,
MONTH_VALUE,
@@ -520,7 +597,8 @@ AS
FORMAT_TYPE,
FILE_BASE_NAME,
MAX_FILE_SIZE,
JOB_CLASS
JOB_CLASS,
TASK_NAME
INTO
vYear,
vMonth,
@@ -536,18 +614,22 @@ AS
vFormat,
vFileBaseName,
vMaxFileSize,
vJobClass
vJobClass,
vTaskName
FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
WHERE CHUNK_ID = pStartId;
WHERE CHUNK_ID = pStartId
AND TASK_NAME = pTaskName;
vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId;
vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId || ', TaskName: ' || vTaskName;
ENV_MANAGER.LOG_PROCESS_EVENT('Starting parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
-- Mark chunk as PROCESSING
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'PROCESSING',
ERROR_MESSAGE = NULL
WHERE CHUNK_ID = pStartId;
WHERE CHUNK_ID = pStartId
AND TASK_NAME = vTaskName;
COMMIT;
-- Call the worker procedure
@@ -570,26 +652,30 @@ AS
);
-- Mark chunk as COMPLETED
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'COMPLETED',
EXPORT_TIMESTAMP = SYSTIMESTAMP,
ERROR_MESSAGE = NULL
WHERE CHUNK_ID = pStartId;
WHERE CHUNK_ID = pStartId
AND TASK_NAME = vTaskName;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('Completed parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
EXCEPTION
WHEN OTHERS THEN
-- Capture error details in variable (SQLERRM cannot be used directly in SQL)
vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || ', TaskName: ' || vTaskName || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
-- Mark chunk as FAILED with error message
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
-- Use vgMsgTmp variable instead of SQLERRM directly (Oracle limitation in SQL context)
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'FAILED',
ERROR_MESSAGE = SUBSTR(vgMsgTmp, 1, 4000)
WHERE CHUNK_ID = pStartId;
WHERE CHUNK_ID = pStartId
AND TASK_NAME = vTaskName;
COMMIT;
RAISE;
@@ -1056,8 +1142,8 @@ AS
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
FOR i IN 1 .. vPartitions.COUNT LOOP
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id)
USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
@@ -1066,33 +1152,34 @@ AS
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING')
WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
-- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
-- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
END LOOP;
COMMIT;
-- Log chunk statistics
-- Log chunk statistics (session-safe: only count chunks for THIS task)
DECLARE
vPendingCount NUMBER;
vFailedCount NUMBER;
BEGIN
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
END;
-- Create parallel task
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
-- Define chunks by number range (1 to partition count)
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
-- Define chunks using SQL query to ensure TASK_NAME isolation
-- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
-- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
task_name => vTaskName,
table_owner => 'CT_MRDS',
table_name => 'A_PARALLEL_EXPORT_CHUNKS',
table_column => 'CHUNK_ID',
chunk_size => 1 -- Each partition is one chunk
sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
by_rowid => FALSE
);
-- Execute task in parallel
@@ -1101,7 +1188,7 @@ AS
IF pJobClass IS NOT NULL THEN
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree,
job_class => pJobClass
@@ -1109,7 +1196,7 @@ AS
ELSE
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree
);
@@ -1360,8 +1447,8 @@ AS
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
FOR i IN 1 .. vPartitions.COUNT LOOP
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id)
USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
@@ -1370,33 +1457,34 @@ AS
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING')
WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
-- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
-- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
END LOOP;
COMMIT;
-- Log chunk statistics
-- Log chunk statistics (session-safe: only count chunks for THIS task)
DECLARE
vPendingCount NUMBER;
vFailedCount NUMBER;
BEGIN
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
END;
-- Create parallel task
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
-- Define chunks by number range (1 to partition count)
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
-- Define chunks using SQL query to ensure TASK_NAME isolation
-- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
-- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
task_name => vTaskName,
table_owner => 'CT_MRDS',
table_name => 'A_PARALLEL_EXPORT_CHUNKS',
table_column => 'CHUNK_ID',
chunk_size => 1 -- Each partition is one chunk
sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
by_rowid => FALSE
);
-- Execute task in parallel
@@ -1405,7 +1493,7 @@ AS
IF pJobClass IS NOT NULL THEN
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree,
job_class => pJobClass
@@ -1413,7 +1501,7 @@ AS
ELSE
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree
);

18
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg
View File

@@ -9,17 +9,17 @@ AS
**/
-- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.11.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00';
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.14.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-25 09:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) ||
'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) ||
'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) ||
'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) ||
'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10);
'v2.14.0 (2026-02-25): OPTIMIZATION - Added pTaskName parameter to EXPORT_PARTITION_PARALLEL for deterministic filtering. Replaced FETCH FIRST 1 ROW ONLY safeguard with precise WHERE CHUNK_ID AND TASK_NAME filter. Eliminates ORDER BY overhead and provides cleaner session isolation.' || CHR(10) ||
'v2.13.1 (2026-02-25): CRITICAL FIX - Added START_ID and END_ID aliasses in CREATE_CHUNKS_BY_SQL to avoid ORA-00960 ambiguous column naming error.' || CHR(10) ||
'v2.13.0 (2026-02-25): CRITICAL SESSION ISOLATION FIX - Changed CREATE_CHUNKS_BY_NUMBER_COL to CREATE_CHUNKS_BY_SQL with TASK_NAME filter (fixes ORA-01422 in concurrent sessions). Added ORDER BY CREATED_DATE DESC FETCH FIRST 1 ROW safeguard to EXPORT_PARTITION_PARALLEL SELECT. Composite PK (TASK_NAME, CHUNK_ID) now fully functional.' || CHR(10) ||
'v2.12.0 (2026-02-24): CRITICAL FIX - Rewritten DELETE_FAILED_EXPORT_FILE to use file-specific pattern matching (prevents deleting parallel CSV chunks in shared folder). Added vQuery logging before DBMS_CLOUD calls. Added CSV maxfilesize logging.' || CHR(10) ||
'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgMsgTmp VARCHAR2(32000);
@@ -54,10 +54,12 @@ AS
* but should NOT be called directly by external code.
* @param pStartId - Chunk start ID (CHUNK_ID from A_PARALLEL_EXPORT_CHUNKS table)
* @param pEndId - Chunk end ID (same as pStartId for single-row chunks)
* @param pTaskName - Task name for session isolation (optional, DEFAULT NULL for backward compatibility)
**/
PROCEDURE EXPORT_PARTITION_PARALLEL (
pStartId IN NUMBER,
pEndId IN NUMBER
pEndId IN NUMBER,
pTaskName IN VARCHAR2 DEFAULT NULL
);
---------------------------------------------------------------------------------------------------------------------------

224
MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql
View File

@@ -1,125 +1,31 @@
--=============================================================================================================================
-- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY)
-- MARS-835: Export Group 1 - HIST Only (DEBT, DEBT_DAILY)
--=============================================================================================================================
-- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet)
-- Purpose: Export ALL data to HIST bucket (Parquet with Hive-style partitioning)
-- Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
-- Excludes legacy columns not required in new structure
-- USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats
-- USES: DATA_EXPORTER v2.12.0 with pTemplateTableName for column order and date formats
-- Author: Grzegorz Michalski
-- Date: 2025-12-17
-- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName)
-- Updated: 2026-02-24 (Changed to HIST-only export, no DATA bucket split)
-- Related: MARS-835 - CSDB Data Export
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"
PROMPT ========================================================================
PROMPT Exporting CSDB.DEBT - Split DATA + HIST
PROMPT Exporting CSDB.DEBT - HIST Only
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION
PROMPT ========================================================================
-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
-- Export ALL data to HIST bucket (Parquet)
-- NEW v2.12.0: Per-column date format handling with template table, full data range
BEGIN
-- Get bucket URI for DATA bucket
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/';
-- Count existing files
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'; -- Exclude directories
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
-- List existing files
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
-- Count records in external table
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
pSchemaName => 'OU_CSDB',
pTableName => 'LEGACY_DEBT',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'DATA',
pFolderName => 'ODS/CSDB/CSDB_DEBT',
pMinDate => &cutoff_date,
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
pProcessName => 'MARS-835', -- Process identifier for tracking
pJobClass => 'high' -- Oracle Scheduler job class for resource management
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
END;
/
-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...');
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (ALL data)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -128,7 +34,8 @@ BEGIN
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
pMaxDate => &cutoff_date,
pMinDate => DATE '1900-01-01', -- Include all historical data
pMaxDate => DATE '9999-12-31', -- Include all future dates
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
pJobClass => 'high' -- Oracle Scheduler job class for resource management
@@ -139,110 +46,18 @@ END;
/
PROMPT ========================================================================
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - HIST Only
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS,
PROMPT ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS
PROMPT ========================================================================
-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
vFileCount NUMBER := 0;
vRecordCount NUMBER := 0;
vLocationUri VARCHAR2(1000);
-- Export ALL data to HIST bucket (Parquet)
-- NEW v2.12.0: Per-column date format handling with template table, full data range
BEGIN
-- Get bucket URI for DATA bucket
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/';
-- Count existing files
SELECT COUNT(*)
INTO vFileCount
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'; -- Exclude directories
IF vFileCount > 0 THEN
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
DBMS_OUTPUT.PUT_LINE('');
-- List existing files
DBMS_OUTPUT.PUT_LINE('Existing files:');
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vLocationUri
))
WHERE object_name NOT LIKE '%/'
ORDER BY object_name
) LOOP
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
END LOOP;
-- Count records in external table
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
END;
DBMS_OUTPUT.PUT_LINE('===============================================================================');
DBMS_OUTPUT.PUT_LINE('');
ELSE
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
DBMS_OUTPUT.PUT_LINE('');
END IF;
END;
/
-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
pSchemaName => 'OU_CSDB',
pTableName => 'LEGACY_DEBT_DAILY',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'DATA',
pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
pMinDate => &cutoff_date,
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
pProcessName => 'MARS-835', -- Process identifier for tracking
pJobClass => 'high' -- Oracle Scheduler job class for resource management
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
END;
/
-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...');
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (ALL data)...');
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -251,7 +66,8 @@ BEGIN
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
pMaxDate => &cutoff_date,
pMinDate => DATE '1900-01-01', -- Include all historical data
pMaxDate => DATE '9999-12-31', -- Include all future dates
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
pJobClass => 'high' -- Oracle Scheduler job class for resource management
@@ -264,8 +80,8 @@ END;
PROMPT ========================================================================
PROMPT Group 1 Export Completed
PROMPT ========================================================================
PROMPT - LEGACY_DEBT: DATA + HIST exported
PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported
PROMPT - LEGACY_DEBT: HIST exported (ALL data)
PROMPT - LEGACY_DEBT_DAILY: HIST exported (ALL data)
PROMPT ========================================================================
--=============================================================================================================================

84
MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 03_MARS_835_verify_exports.sql
-- Purpose: Verify exported files exist in DATA and HIST buckets after export
-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/
-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,17 +14,14 @@ SET VERIFY OFF;
SET LINESIZE 200;
PROMPT =====================================================================================
PROMPT MARS-835 Verification: Listing exported files in DATA and HIST buckets
PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy)
PROMPT =====================================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
vTotalDataFiles NUMBER := 0;
vTotalHistFiles NUMBER := 0;
vTotalDataSize NUMBER := 0;
vTotalHistSize NUMBER := 0;
TYPE t_folder_info IS RECORD (
@@ -33,25 +31,18 @@ DECLARE
);
TYPE t_folder_list IS TABLE OF t_folder_info;
vDataFolders t_folder_list;
vHistFolders t_folder_list;
BEGIN
-- Get bucket URIs and credential from FILE_MANAGER
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential from FILE_MANAGER
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
DBMS_OUTPUT.PUT_LINE('');
-- Initialize folder lists
vDataFolders := t_folder_list(
t_folder_info('ODS/CSDB/CSDB_DEBT/', 'DEBT', 'CSV'),
t_folder_info('ODS/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'CSV')
);
-- Initialize folder list (all tables in HIST)
-- Initialize folder list (all 6 tables in HIST)
vHistFolders := t_folder_list(
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'),
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'),
@@ -62,49 +53,7 @@ BEGIN
);
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket Exports (CSV format - last 6 months)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check DATA bucket exports
FOR i IN 1..vDataFolders.COUNT LOOP
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Table: ' || vDataFolders(i).table_name || ' (' || vDataFolders(i).expected_format || ')');
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i).folder_name);
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
BEGIN
FOR rec IN (
SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || vDataFolders(i).folder_name
))
WHERE object_name LIKE '%.csv'
ORDER BY created DESC
) LOOP
vFileCount := vFileCount + 1;
vTotalDataFiles := vTotalDataFiles + 1;
vTotalDataSize := vTotalDataSize + rec.bytes;
DBMS_OUTPUT.PUT_LINE(' [' || vFileCount || '] ' || rec.object_name ||
' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
END LOOP;
IF vFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' [ERROR] No CSV files found - Export may have failed!');
ELSE
DBMS_OUTPUT.PUT_LINE(' [SUCCESS] Found ' || vFileCount || ' CSV file(s)');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE(' [ERROR] Cannot access folder - ' || SQLERRM);
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning)');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check HIST bucket exports
@@ -155,24 +104,19 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Export Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('DATA Bucket (CSV):');
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalDataFiles);
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalDataSize/1024/1024/1024, 2) || ' GB');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 2 (DEBT, DEBT_DAILY - last 6 months)');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet):');
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):');
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalHistFiles || '+');
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables with historical data)');
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables exported to HIST)');
DBMS_OUTPUT.PUT_LINE('');
IF vTotalDataFiles >= 2 AND vTotalHistFiles >= 6 THEN
IF vTotalHistFiles >= 6 THEN
DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL');
DBMS_OUTPUT.PUT_LINE(' Files found in both DATA and HIST buckets');
DBMS_OUTPUT.PUT_LINE(' Files found in HIST bucket for all tables');
DBMS_OUTPUT.PUT_LINE(' Proceed to record count verification (Step 4)');
ELSIF vTotalDataFiles = 0 AND vTotalHistFiles = 0 THEN
ELSIF vTotalHistFiles = 0 THEN
DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED');
DBMS_OUTPUT.PUT_LINE(' No files found in either bucket');
DBMS_OUTPUT.PUT_LINE(' No files found in HIST bucket');
DBMS_OUTPUT.PUT_LINE(' Review export logs for errors');
ELSE
DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected');

105
MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 04_MARS_835_verify_record_counts.sql
-- Purpose: Verify record counts match between source tables and exported data
-- Purpose: Verify record counts match between source tables and exported data (HIST-only)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Verification: Compare OU_CSDB source tables with ODS external tables
-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only)
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,28 +14,23 @@ SET VERIFY OFF;
SET LINESIZE 200;
PROMPT =====================================================================================
PROMPT MARS-835 Record Count Verification
PROMPT MARS-835 Record Count Verification (HIST-only strategy)
PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT Comparing source table counts with HIST external table counts
PROMPT =====================================================================================
DECLARE
TYPE t_table_info IS RECORD (
source_schema VARCHAR2(50),
source_table VARCHAR2(100),
data_external_table VARCHAR2(100),
hist_external_table VARCHAR2(100),
has_data_export BOOLEAN,
has_hist_export BOOLEAN
hist_external_table VARCHAR2(100)
);
TYPE t_table_list IS TABLE OF t_table_info;
vTables t_table_list;
vSourceCount NUMBER;
vDataCount NUMBER;
vHistCount NUMBER;
vTotalSourceCount NUMBER := 0;
vTotalDataCount NUMBER := 0;
vTotalHistCount NUMBER := 0;
vMismatchCount NUMBER := 0;
vSql VARCHAR2(4000);
@@ -42,18 +38,18 @@ BEGIN
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
DBMS_OUTPUT.PUT_LINE('');
-- Initialize table list with export configuration
-- Initialize table list (all tables HIST-only)
vTables := t_table_list(
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ODS', 'ODS.CSDB_DEBT_ARCHIVE', TRUE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ODS', 'ODS.CSDB_DEBT_DAILY_ARCHIVE', TRUE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', NULL, 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', NULL, 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', NULL, 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE', FALSE, TRUE),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE)
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'),
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE')
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP
@@ -70,31 +66,6 @@ BEGIN
CONTINUE;
END;
-- Get DATA external table count (if applicable)
IF vTables(i).has_data_export THEN
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).data_external_table;
BEGIN
EXECUTE IMMEDIATE vSql INTO vDataCount;
vTotalDataCount := vTotalDataCount + vDataCount;
EXCEPTION
WHEN OTHERS THEN
-- If source table is empty (0 records), no files were exported
-- External table returns error, treat as 0
-- Acceptable error codes:
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vDataCount := 0;
ELSE
vDataCount := -1;
END IF;
END;
ELSE
vDataCount := NULL;
END IF;
-- Get HIST external table count
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table;
BEGIN
@@ -119,18 +90,8 @@ BEGIN
-- Display results
DECLARE
vStatus VARCHAR2(20);
vDataDisplay VARCHAR2(17);
vHistDisplay VARCHAR2(17);
BEGIN
-- Format DATA count display
IF vDataCount IS NULL THEN
vDataDisplay := 'N/A';
ELSIF vDataCount = -1 THEN
vDataDisplay := 'ERROR';
ELSE
vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999');
END IF;
-- Format HIST count display
IF vHistCount = -1 THEN
vHistDisplay := 'ERROR';
@@ -138,35 +99,20 @@ BEGIN
vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
END IF;
-- Determine status
IF vTables(i).has_data_export THEN
-- Split export: check DATA + HIST = SOURCE
IF (vDataCount + vHistCount) = vSourceCount THEN
vStatus := 'PASS';
ELSIF vDataCount = -1 OR vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
-- Determine status (HIST only: check HIST = SOURCE)
IF vHistCount = vSourceCount THEN
vStatus := 'PASS';
ELSIF vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
-- HIST only: check HIST = SOURCE
IF vHistCount = vSourceCount THEN
vStatus := 'PASS';
ELSIF vHistCount = -1 THEN
vStatus := 'ERROR';
vMismatchCount := vMismatchCount + 1;
ELSE
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
vStatus := 'MISMATCH';
vMismatchCount := vMismatchCount + 1;
END IF;
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
LPAD(vDataDisplay, 15) ||
LPAD(vHistDisplay, 15) || ' ' ||
vStatus
);
@@ -177,18 +123,16 @@ BEGIN
DBMS_OUTPUT.PUT_LINE(
RPAD('TOTALS', 24) ||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)');
DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 THEN
@@ -209,7 +153,6 @@ BEGIN
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (may not exist yet)');
DBMS_OUTPUT.PUT_LINE(' N/A - Not applicable (table not exported to DATA)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
EXCEPTION

85
MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql
View File

@@ -1,68 +1,34 @@
--=============================================================================================================================
-- MARS-835 ROLLBACK: Delete Group 1 Exported Files (DEBT, DEBT_DAILY)
--=============================================================================================================================
-- Purpose: Delete exported CSV and Parquet files from DATA and HIST buckets
-- Purpose: Delete exported Parquet files from HIST bucket (ARCHIVE only)
-- WARNING: This will permanently delete exported data files!
-- Author: Grzegorz Michalski
-- Date: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only rollback, no DATA bucket)
-- Related: MARS-835 - CSDB Data Export Rollback
--=============================================================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting DEBT exported files
PROMPT ROLLBACK: Deleting DEBT exported files from HIST
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT/
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URIs and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
))
WHERE object_name LIKE 'LEGACY_DEBT%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
FOR rec IN (
@@ -99,58 +65,23 @@ END;
/
PROMPT ========================================================================
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files from HIST
PROMPT ========================================================================
PROMPT WARNING: This will delete files from:
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/
PROMPT ========================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vFileCount NUMBER := 0;
BEGIN
-- Get bucket URIs and credential
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI and credential
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
))
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => vCredentialName,
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
);
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
vFileCount := vFileCount + 1;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
ELSE
RAISE;
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
vFileCount := 0;
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
FOR rec IN (

73
MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql
View File

@@ -1,10 +1,11 @@
-- =====================================================================================
-- Script: 99_MARS_835_verify_rollback.sql
-- Purpose: Verify all exported files have been deleted from DATA and HIST buckets
-- Purpose: Verify all exported files have been deleted from HIST bucket (HIST-only strategy)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Verification: Confirm complete rollback (no CSDB files remaining)
-- Verification: Confirm complete rollback (no CSDB files remaining in HIST)
-- =====================================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -19,33 +20,23 @@ PROMPT Checking that all CSDB export files have been deleted
PROMPT =====================================================================================
DECLARE
vDataBucketUri VARCHAR2(500);
vHistBucketUri VARCHAR2(500);
vCredentialName VARCHAR2(100);
vDataFileCount NUMBER := 0;
vHistFileCount NUMBER := 0;
vTotalFiles NUMBER := 0;
TYPE t_folder_list IS TABLE OF VARCHAR2(200);
vDataFolders t_folder_list;
vHistFolders t_folder_list;
BEGIN
-- Get bucket URIs
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
-- Get bucket URI
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
DBMS_OUTPUT.PUT_LINE('ROLLBACK VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
DBMS_OUTPUT.PUT_LINE('');
-- Initialize folder lists
vDataFolders := t_folder_list(
'ODS/CSDB/CSDB_DEBT/',
'ODS/CSDB/CSDB_DEBT_DAILY/'
);
-- Initialize folder list (all 6 tables in HIST)
-- Initialize folder list (all 6 tables in HIST)
vHistFolders := t_folder_list(
'ARCHIVE/CSDB/CSDB_DEBT/',
'ARCHIVE/CSDB/CSDB_DEBT_DAILY/',
@@ -55,47 +46,6 @@ BEGIN
'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/'
);
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (should be empty)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-- Check DATA bucket
FOR i IN 1..vDataFolders.COUNT LOOP
DECLARE
vCount NUMBER := 0;
BEGIN
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i));
FOR rec IN (
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => vCredentialName,
location_uri => vDataBucketUri || vDataFolders(i)
))
WHERE object_name LIKE '%.csv'
) LOOP
vCount := vCount + 1;
vDataFileCount := vDataFileCount + 1;
DBMS_OUTPUT.PUT_LINE(' [FOUND] ' || rec.object_name);
END LOOP;
IF vCount = 0 THEN
DBMS_OUTPUT.PUT_LINE(' [OK] No CSV files found');
ELSE
DBMS_OUTPUT.PUT_LINE(' [INFO] Found ' || vCount || ' file(s) - may be pre-existing files from before installation');
END IF;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
DBMS_OUTPUT.PUT_LINE(' [OK] Folder does not exist or is empty');
ELSE
DBMS_OUTPUT.PUT_LINE(' [ERROR] ' || SQLERRM);
END IF;
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (should be empty)');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
@@ -139,24 +89,21 @@ BEGIN
END;
END LOOP;
vTotalFiles := vDataFileCount + vHistFileCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Rollback Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('DATA bucket files remaining: ' || vDataFileCount);
DBMS_OUTPUT.PUT_LINE('HIST bucket files remaining: ' || vHistFileCount || '+');
DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles || '+');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('');
IF vTotalFiles = 0 THEN
IF vHistFileCount = 0 THEN
DBMS_OUTPUT.PUT_LINE('[PASSED] ROLLBACK VERIFICATION PASSED');
DBMS_OUTPUT.PUT_LINE(' All CSDB export files have been deleted or were not created');
DBMS_OUTPUT.PUT_LINE(' Buckets are clean and ready for re-export if needed');
DBMS_OUTPUT.PUT_LINE(' HIST bucket is clean and ready for re-export if needed');
ELSE
DBMS_OUTPUT.PUT_LINE('[INFO] ROLLBACK VERIFICATION COMPLETED');
DBMS_OUTPUT.PUT_LINE(' Found ' || vTotalFiles || '+ file(s) remaining in buckets');
DBMS_OUTPUT.PUT_LINE(' Found ' || vHistFileCount || '+ file(s) remaining in HIST bucket');
DBMS_OUTPUT.PUT_LINE(' NOTE: These may be pre-existing files from before installation.');
DBMS_OUTPUT.PUT_LINE(' Rollback only deletes files created during this export operation.');
DBMS_OUTPUT.PUT_LINE(' If needed, manually verify and clean up remaining files.');