diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql index 1d8ca40..699e608 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql @@ -25,6 +25,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported'); @@ -46,6 +47,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported'); @@ -67,6 +69,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql index 7294564..3f4a206 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql @@ -30,6 +30,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER', pParallelDegree => 4, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported'); @@ -51,6 +52,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM', pParallelDegree => 16, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql index f86e25f..2c7cd69 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql @@ -25,6 +25,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported'); @@ -46,6 +47,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM', pParallelDegree => 2, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported'); @@ -67,6 +69,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER', pParallelDegree => 2, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql index fef4bf9..ae81aed 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql @@ -30,6 +30,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES', pParallelDegree => 8, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported'); @@ -51,6 +52,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER', pParallelDegree => 2, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql index 6d31e2d..8b91728 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql @@ -26,6 +26,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER', pParallelDegree => 2, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported'); @@ -47,6 +48,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM', pParallelDegree => 16, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql index 7e0202c..f1cbb14 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql @@ -30,6 +30,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER', pParallelDegree => 4, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported'); @@ -51,6 +52,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM', pParallelDegree => 16, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql index cc512ba..1391512 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql @@ -25,6 +25,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported'); @@ -46,6 +47,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM', pParallelDegree => 4, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported'); @@ -66,8 +68,7 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER', - pParallelDegree => 2, - pJobClass => 'high' + pParallelDegree => 2, pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported'); EXCEPTION diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql index 8150372..c7fe059 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql @@ -25,6 +25,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_HEADER', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported'); @@ -46,6 +47,7 @@ BEGIN pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM', pParallelDegree => 1, + pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_ITEM', pJobClass => 'high' ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql index bd37230..298ac97 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql @@ -26,7 +26,7 @@ END; / CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS ( - CHUNK_ID NUMBER PRIMARY KEY, + CHUNK_ID NUMBER NOT NULL, TASK_NAME VARCHAR2(100) NOT NULL, YEAR_VALUE VARCHAR2(4) NOT NULL, MONTH_VALUE VARCHAR2(2) NOT NULL, @@ -47,14 +47,16 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS ( STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL, ERROR_MESSAGE VARCHAR2(4000), EXPORT_TIMESTAMP TIMESTAMP, - CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL + CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL, + CONSTRAINT PK_PARALLEL_EXPORT_CHUNKS PRIMARY KEY (TASK_NAME, CHUNK_ID) ); -CREATE INDEX IX_PARALLEL_CHUNKS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(TASK_NAME); +-- Index for status-based queries (e.g., WHERE STATUS = 'FAILED' AND TASK_NAME = ?) +CREATE INDEX IX_PARALLEL_CHUNKS_STATUS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(STATUS, TASK_NAME); -COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions'; -COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Unique chunk identifier (partition number)'; -COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name for cleanup'; +COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions. PK: (TASK_NAME, CHUNK_ID) ensures session isolation for concurrent exports.'; +COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Chunk identifier within task (partition number) - unique per TASK_NAME, not globally'; +COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name - session isolation key, part of composite PK with CHUNK_ID'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.YEAR_VALUE IS 'Partition year (YYYY)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MONTH_VALUE IS 'Partition month (MM)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.SCHEMA_NAME IS 'Schema owning the source table'; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb index 05d97d8..13771b1 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb @@ -18,34 +18,104 @@ AS ---------------------------------------------------------------------------------------------------- /** - * Deletes export file from OCI bucket if it exists (used for cleanup before retry) - * Silently ignores if file doesn't exist (ORA-20404) + * Deletes ALL files matching specific file pattern before retry export + * Critical for preventing data duplication when DBMS_CLOUD.EXPORT_DATA fails mid-process + * + * Problem: Export fails after creating partial file(s), retry creates new _2, _3 suffixed files + * Solution: Delete ALL files matching the base filename pattern before retry + * + * Pattern matching strategy: + * - Parquet: folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/*.parquet (folder-level safe - each chunk has own partition folder) + * - CSV: folder/TABLENAME_202411*.csv (file-level pattern - multiple chunks share same folder!) + * + * CRITICAL for parallel processing: + * - Parquet chunks are isolated by partition folder structure (safe to delete folder/*) + * - CSV chunks share flat folder structure - MUST use file-specific pattern (TABLENAME_YYYYMM*) + * to avoid deleting files from other parallel chunks in same folder **/ PROCEDURE DELETE_FAILED_EXPORT_FILE( pFileUri IN VARCHAR2, pCredentialName IN VARCHAR2, pParameters IN VARCHAR2 ) IS + vBucketUri VARCHAR2(4000); + vFolderPath VARCHAR2(4000); + vFileName VARCHAR2(1000); + vFileNamePattern VARCHAR2(1000); + vSlashPos NUMBER; + vDotPos NUMBER; + vFilesDeleted NUMBER := 0; BEGIN - BEGIN - ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to delete potentially corrupted file: ' || pFileUri, 'DEBUG', pParameters); + -- Extract components from URI + -- Example Parquet: https://.../bucket/folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/202411.parquet + -- Example CSV: https://.../bucket/folder/TABLENAME_202411.csv + + -- Find last slash before filename + vSlashPos := INSTR(pFileUri, '/', -1); + + IF vSlashPos > 0 THEN + -- Extract filename from URI (after last slash) + vFileName := SUBSTR(pFileUri, vSlashPos + 1); - DBMS_CLOUD.DELETE_OBJECT( - credential_name => pCredentialName, - object_uri => pFileUri - ); + -- Extract folder path (before last slash) + vFolderPath := SUBSTR(pFileUri, 1, vSlashPos - 1); - ENV_MANAGER.LOG_PROCESS_EVENT('Deleted existing file (cleanup before retry): ' || pFileUri, 'INFO', pParameters); - EXCEPTION - WHEN OTHERS THEN - -- Object not found is OK (file doesn't exist) - IF SQLCODE = -20404 THEN - ENV_MANAGER.LOG_PROCESS_EVENT('File does not exist (OK): ' || pFileUri, 'DEBUG', pParameters); - ELSE - -- Log but don't fail - export will attempt anyway - ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete file (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters); - END IF; - END; + -- Find bucket URI (protocol + namespace + bucket name) + -- Bucket URI ends after /o/ in OCI Object Storage URLs + vBucketUri := SUBSTR(pFileUri, 1, INSTR(pFileUri, '/o/') + 2); + + -- Extract relative folder path (after bucket) + vFolderPath := SUBSTR(vFolderPath, LENGTH(vBucketUri) + 1); + + -- Create file pattern by removing extension + -- Oracle adds suffixes BEFORE extension: file.csv -> file_1_timestamp.csv + -- Pattern: file* matches file_1_timestamp.csv, file_2_timestamp.csv + vDotPos := INSTR(vFileName, '.', -1); + IF vDotPos > 0 THEN + vFileNamePattern := SUBSTR(vFileName, 1, vDotPos - 1) || '%'; + ELSE + vFileNamePattern := vFileName || '%'; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup before retry - Pattern: ' || vFolderPath || '/' || vFileNamePattern, 'DEBUG', pParameters); + + -- List and delete ALL files matching pattern + -- CRITICAL: Uses file-specific pattern for CSV chunk isolation in shared folder + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => pCredentialName, + location_uri => vBucketUri + )) + WHERE object_name LIKE vFolderPath || '/' || vFileNamePattern + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => pCredentialName, + object_uri => vBucketUri || rec.object_name + ); + + vFilesDeleted := vFilesDeleted + 1; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted partial file ' || vFilesDeleted || ': ' || rec.object_name, 'DEBUG', pParameters); + EXCEPTION + WHEN OTHERS THEN + -- Log but continue - don't fail entire cleanup + ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete ' || rec.object_name || ': ' || SQLERRM, 'WARNING', pParameters); + END; + END LOOP; + + IF vFilesDeleted > 0 THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup completed: Deleted ' || vFilesDeleted || ' partial file(s) from previous failed export', 'INFO', pParameters); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('No existing files to clean up (pattern match: ' || vFileNamePattern || ')', 'DEBUG', pParameters); + END IF; + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cannot parse file URI for cleanup: ' || pFileUri, 'WARNING', pParameters); + END IF; + EXCEPTION + WHEN OTHERS THEN + -- Don't fail export if cleanup fails - log and continue + ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cleanup failed (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters); END DELETE_FAILED_EXPORT_FILE; ---------------------------------------------------------------------------------------------------- @@ -415,6 +485,8 @@ AS AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'') AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')'; + ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: '||pFormat||')', 'DEBUG', pParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters); -- Construct the URI based on format IF pFormat = 'PARQUET' THEN -- Parquet: Use Hive-style partitioning @@ -425,6 +497,7 @@ AS 'PARTITION_MONTH=' || sanitizeFilename(pMonth) || '/' || sanitizeFilename(pYear) || sanitizeFilename(pMonth) || '.parquet'; + ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters); -- Delete potentially corrupted file from previous failed attempt @@ -445,6 +518,7 @@ AS sanitizeFilename(vFileName); ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('CSV maxfilesize: ' || pMaxFileSize || ' bytes (' || ROUND(pMaxFileSize/1048576, 2) || ' MB)', 'DEBUG', pParameters); -- Delete potentially corrupted file from previous failed attempt -- This prevents Oracle from creating _1 suffixed files on retry @@ -472,8 +546,7 @@ AS RAISE_APPLICATION_ERROR(-20001, 'Unsupported format: ' || pFormat || '. Use PARQUET or CSV.'); END IF; - ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: ' || pFormat || ')', 'DEBUG', pParameters); - ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || pYear || '/' || pMonth, 'DEBUG', pParameters); END EXPORT_SINGLE_PARTITION; ---------------------------------------------------------------------------------------------------- @@ -485,7 +558,8 @@ AS **/ PROCEDURE EXPORT_PARTITION_PARALLEL ( pStartId IN NUMBER, - pEndId IN NUMBER + pEndId IN NUMBER, + pTaskName IN VARCHAR2 DEFAULT NULL ) IS vYear VARCHAR2(4); vMonth VARCHAR2(2); @@ -502,9 +576,12 @@ AS vFileBaseName VARCHAR2(1000); vMaxFileSize NUMBER; vJobClass VARCHAR2(128); + vTaskName VARCHAR2(128); vParameters VARCHAR2(4000); BEGIN - -- Retrieve chunk context from global temporary table + -- Retrieve chunk context from A_PARALLEL_EXPORT_CHUNKS table + -- CRITICAL: Filter by CHUNK_ID and TASK_NAME for precise session isolation + -- pTaskName parameter passed from RUN_TASK ensures deterministic single-row retrieval SELECT YEAR_VALUE, MONTH_VALUE, @@ -520,7 +597,8 @@ AS FORMAT_TYPE, FILE_BASE_NAME, MAX_FILE_SIZE, - JOB_CLASS + JOB_CLASS, + TASK_NAME INTO vYear, vMonth, @@ -536,18 +614,22 @@ AS vFormat, vFileBaseName, vMaxFileSize, - vJobClass + vJobClass, + vTaskName FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS - WHERE CHUNK_ID = pStartId; + WHERE CHUNK_ID = pStartId + AND TASK_NAME = pTaskName; - vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId; + vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId || ', TaskName: ' || vTaskName; ENV_MANAGER.LOG_PROCESS_EVENT('Starting parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters); -- Mark chunk as PROCESSING + -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS SET STATUS = 'PROCESSING', ERROR_MESSAGE = NULL - WHERE CHUNK_ID = pStartId; + WHERE CHUNK_ID = pStartId + AND TASK_NAME = vTaskName; COMMIT; -- Call the worker procedure @@ -570,26 +652,30 @@ AS ); -- Mark chunk as COMPLETED + -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS SET STATUS = 'COMPLETED', EXPORT_TIMESTAMP = SYSTIMESTAMP, ERROR_MESSAGE = NULL - WHERE CHUNK_ID = pStartId; + WHERE CHUNK_ID = pStartId + AND TASK_NAME = vTaskName; COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('Completed parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters); EXCEPTION WHEN OTHERS THEN -- Capture error details in variable (SQLERRM cannot be used directly in SQL) - vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; + vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || ', TaskName: ' || vTaskName || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); -- Mark chunk as FAILED with error message + -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation -- Use vgMsgTmp variable instead of SQLERRM directly (Oracle limitation in SQL context) UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS SET STATUS = 'FAILED', ERROR_MESSAGE = SUBSTR(vgMsgTmp, 1, 4000) - WHERE CHUNK_ID = pStartId; + WHERE CHUNK_ID = pStartId + AND TASK_NAME = vTaskName; COMMIT; RAISE; @@ -1056,8 +1142,8 @@ AS -- Populate chunks table (insert new chunks, preserve FAILED chunks for retry) FOR i IN 1 .. vPartitions.COUNT LOOP MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t - USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s - ON (t.CHUNK_ID = s.chunk_id) + USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s + ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name) WHEN NOT MATCHED THEN INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, @@ -1066,33 +1152,34 @@ AS vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING') WHEN MATCHED THEN - UPDATE SET TASK_NAME = vTaskName, - STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, + -- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID) + -- This handles retry scenario: reset FAILED chunks to PENDING for re-processing + UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END; END LOOP; COMMIT; - -- Log chunk statistics + -- Log chunk statistics (session-safe: only count chunks for THIS task) DECLARE vPendingCount NUMBER; vFailedCount NUMBER; BEGIN - SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING'; - SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED'; + SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName; + SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName; - ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters); END; -- Create parallel task DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName); - -- Define chunks by number range (1 to partition count) - DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL( + -- Define chunks using SQL query to ensure TASK_NAME isolation + -- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions + -- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming + DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL( task_name => vTaskName, - table_owner => 'CT_MRDS', - table_name => 'A_PARALLEL_EXPORT_CHUNKS', - table_column => 'CHUNK_ID', - chunk_size => 1 -- Each partition is one chunk + sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID', + by_rowid => FALSE ); -- Execute task in parallel @@ -1101,7 +1188,7 @@ AS IF pJobClass IS NOT NULL THEN DBMS_PARALLEL_EXECUTE.RUN_TASK( task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;', language_flag => DBMS_SQL.NATIVE, parallel_level => pParallelDegree, job_class => pJobClass @@ -1109,7 +1196,7 @@ AS ELSE DBMS_PARALLEL_EXECUTE.RUN_TASK( task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;', language_flag => DBMS_SQL.NATIVE, parallel_level => pParallelDegree ); @@ -1360,8 +1447,8 @@ AS -- Populate chunks table (insert new chunks, preserve FAILED chunks for retry) FOR i IN 1 .. vPartitions.COUNT LOOP MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t - USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s - ON (t.CHUNK_ID = s.chunk_id) + USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s + ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name) WHEN NOT MATCHED THEN INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, @@ -1370,33 +1457,34 @@ AS vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING') WHEN MATCHED THEN - UPDATE SET TASK_NAME = vTaskName, - STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, + -- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID) + -- This handles retry scenario: reset FAILED chunks to PENDING for re-processing + UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END; END LOOP; COMMIT; - -- Log chunk statistics + -- Log chunk statistics (session-safe: only count chunks for THIS task) DECLARE vPendingCount NUMBER; vFailedCount NUMBER; BEGIN - SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING'; - SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED'; + SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName; + SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName; - ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters); END; -- Create parallel task DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName); - -- Define chunks by number range (1 to partition count) - DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL( + -- Define chunks using SQL query to ensure TASK_NAME isolation + -- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions + -- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming + DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL( task_name => vTaskName, - table_owner => 'CT_MRDS', - table_name => 'A_PARALLEL_EXPORT_CHUNKS', - table_column => 'CHUNK_ID', - chunk_size => 1 -- Each partition is one chunk + sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID', + by_rowid => FALSE ); -- Execute task in parallel @@ -1405,7 +1493,7 @@ AS IF pJobClass IS NOT NULL THEN DBMS_PARALLEL_EXECUTE.RUN_TASK( task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;', language_flag => DBMS_SQL.NATIVE, parallel_level => pParallelDegree, job_class => pJobClass @@ -1413,7 +1501,7 @@ AS ELSE DBMS_PARALLEL_EXECUTE.RUN_TASK( task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;', language_flag => DBMS_SQL.NATIVE, parallel_level => pParallelDegree ); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg index a723db1..e8498b4 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg @@ -9,17 +9,17 @@ AS **/ -- Package Version Information - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.11.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.14.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-25 09:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (last 3-5 changes) VERSION_HISTORY CONSTANT VARCHAR2(4000) := - 'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) || - 'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) || - 'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) || - 'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) || - 'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10); + 'v2.14.0 (2026-02-25): OPTIMIZATION - Added pTaskName parameter to EXPORT_PARTITION_PARALLEL for deterministic filtering. Replaced FETCH FIRST 1 ROW ONLY safeguard with precise WHERE CHUNK_ID AND TASK_NAME filter. Eliminates ORDER BY overhead and provides cleaner session isolation.' || CHR(10) || + 'v2.13.1 (2026-02-25): CRITICAL FIX - Added START_ID and END_ID aliasses in CREATE_CHUNKS_BY_SQL to avoid ORA-00960 ambiguous column naming error.' || CHR(10) || + 'v2.13.0 (2026-02-25): CRITICAL SESSION ISOLATION FIX - Changed CREATE_CHUNKS_BY_NUMBER_COL to CREATE_CHUNKS_BY_SQL with TASK_NAME filter (fixes ORA-01422 in concurrent sessions). Added ORDER BY CREATED_DATE DESC FETCH FIRST 1 ROW safeguard to EXPORT_PARTITION_PARALLEL SELECT. Composite PK (TASK_NAME, CHUNK_ID) now fully functional.' || CHR(10) || + 'v2.12.0 (2026-02-24): CRITICAL FIX - Rewritten DELETE_FAILED_EXPORT_FILE to use file-specific pattern matching (prevents deleting parallel CSV chunks in shared folder). Added vQuery logging before DBMS_CLOUD calls. Added CSV maxfilesize logging.' || CHR(10) || + 'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10); cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); vgMsgTmp VARCHAR2(32000); @@ -54,10 +54,12 @@ AS * but should NOT be called directly by external code. * @param pStartId - Chunk start ID (CHUNK_ID from A_PARALLEL_EXPORT_CHUNKS table) * @param pEndId - Chunk end ID (same as pStartId for single-row chunks) + * @param pTaskName - Task name for session isolation (optional, DEFAULT NULL for backward compatibility) **/ PROCEDURE EXPORT_PARTITION_PARALLEL ( pStartId IN NUMBER, - pEndId IN NUMBER + pEndId IN NUMBER, + pTaskName IN VARCHAR2 DEFAULT NULL ); --------------------------------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql index 235e387..964f9f8 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql @@ -1,125 +1,31 @@ --============================================================================================================================= --- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY) +-- MARS-835: Export Group 1 - HIST Only (DEBT, DEBT_DAILY) --============================================================================================================================= --- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet) +-- Purpose: Export ALL data to HIST bucket (Parquet with Hive-style partitioning) -- Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY -- Excludes legacy columns not required in new structure --- USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats +-- USES: DATA_EXPORTER v2.12.0 with pTemplateTableName for column order and date formats -- Author: Grzegorz Michalski -- Date: 2025-12-17 --- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName) +-- Updated: 2026-02-24 (Changed to HIST-only export, no DATA bucket split) -- Related: MARS-835 - CSDB Data Export --============================================================================================================================= SET SERVEROUTPUT ON SIZE UNLIMITED SET TIMING ON -DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')" - PROMPT ======================================================================== -PROMPT Exporting CSDB.DEBT - Split DATA + HIST +PROMPT Exporting CSDB.DEBT - HIST Only PROMPT ======================================================================== -PROMPT Last 6 months to DATA bucket (CSV format) -PROMPT Older data to HIST bucket (Parquet with partitioning) +PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning) PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION PROMPT ======================================================================== --- PRE-EXPORT CHECK: List existing files and count records -DECLARE - vFileCount NUMBER := 0; - vRecordCount NUMBER := 0; - vLocationUri VARCHAR2(1000); +-- Export ALL data to HIST bucket (Parquet) +-- NEW v2.12.0: Per-column date format handling with template table, full data range BEGIN - -- Get bucket URI for DATA bucket - vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/'; - - -- Count existing files - SELECT COUNT(*) - INTO vFileCount - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'OCI$RESOURCE_PRINCIPAL', - location_uri => vLocationUri - )) - WHERE object_name NOT LIKE '%/'; -- Exclude directories - - IF vFileCount > 0 THEN - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket'); - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri); - DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount); - DBMS_OUTPUT.PUT_LINE(''); - - -- List existing files - DBMS_OUTPUT.PUT_LINE('Existing files:'); - FOR rec IN ( - SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'OCI$RESOURCE_PRINCIPAL', - location_uri => vLocationUri - )) - WHERE object_name NOT LIKE '%/' - ORDER BY object_name - ) LOOP - DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')'); - END LOOP; - - -- Count records in external table - BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount; - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE('>>>'); - DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount); - DBMS_OUTPUT.PUT_LINE('>>>'); - DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table'); - DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); - END; - - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE(''); - ELSE - DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean'); - DBMS_OUTPUT.PUT_LINE(''); - END IF; -END; -/ - --- Export recent data to DATA bucket (CSV) --- NEW v2.4.0: Per-column date format handling with template table for column order -BEGIN - DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...'); - DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT'); - - CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE( - pSchemaName => 'OU_CSDB', - pTableName => 'LEGACY_DEBT', - pKeyColumnName => 'A_ETL_LOAD_SET_FK', - pBucketArea => 'DATA', - pFolderName => 'ODS/CSDB/CSDB_DEBT', - pMinDate => &cutoff_date, - pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) - pParallelDegree => 16, - pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT', - pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) - pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) - pProcessName => 'MARS-835', -- Process identifier for tracking - pJobClass => 'high' -- Oracle Scheduler job class for resource management - ); - - DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order'); -END; -/ - --- Export historical data to HIST bucket (Parquet) --- NEW v2.4.0: Per-column date format handling with template table -BEGIN - DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...'); + DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (ALL data)...'); DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT'); CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE( @@ -128,7 +34,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT', - pMaxDate => &cutoff_date, + pMinDate => DATE '1900-01-01', -- Include all historical data + pMaxDate => DATE '9999-12-31', -- Include all future dates pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT', pJobClass => 'high' -- Oracle Scheduler job class for resource management @@ -139,110 +46,18 @@ END; / PROMPT ======================================================================== -PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST +PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - HIST Only PROMPT ======================================================================== -PROMPT Last 6 months to DATA bucket (CSV format) -PROMPT Older data to HIST bucket (Parquet with partitioning) +PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning) PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS, PROMPT ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS PROMPT ======================================================================== --- PRE-EXPORT CHECK: List existing files and count records -DECLARE - vFileCount NUMBER := 0; - vRecordCount NUMBER := 0; - vLocationUri VARCHAR2(1000); +-- Export ALL data to HIST bucket (Parquet) +-- NEW v2.12.0: Per-column date format handling with template table, full data range BEGIN - -- Get bucket URI for DATA bucket - vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/'; - - -- Count existing files - SELECT COUNT(*) - INTO vFileCount - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'OCI$RESOURCE_PRINCIPAL', - location_uri => vLocationUri - )) - WHERE object_name NOT LIKE '%/'; -- Exclude directories - - IF vFileCount > 0 THEN - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket'); - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri); - DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount); - DBMS_OUTPUT.PUT_LINE(''); - - -- List existing files - DBMS_OUTPUT.PUT_LINE('Existing files:'); - FOR rec IN ( - SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'OCI$RESOURCE_PRINCIPAL', - location_uri => vLocationUri - )) - WHERE object_name NOT LIKE '%/' - ORDER BY object_name - ) LOOP - DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')'); - END LOOP; - - -- Count records in external table - BEGIN - EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount; - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE('>>>'); - DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount); - DBMS_OUTPUT.PUT_LINE('>>>'); - DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------'); - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table'); - DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM); - END; - - DBMS_OUTPUT.PUT_LINE('==============================================================================='); - DBMS_OUTPUT.PUT_LINE(''); - ELSE - DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean'); - DBMS_OUTPUT.PUT_LINE(''); - END IF; -END; -/ - --- Export recent data to DATA bucket (CSV) --- NEW v2.4.0: Per-column date format handling with template table for column order -BEGIN - DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...'); - DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY'); - - CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE( - pSchemaName => 'OU_CSDB', - pTableName => 'LEGACY_DEBT_DAILY', - pKeyColumnName => 'A_ETL_LOAD_SET_FK', - pBucketArea => 'DATA', - pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY', - pMinDate => &cutoff_date, - pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) - pParallelDegree => 16, - pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY', - pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) - pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) - pProcessName => 'MARS-835', -- Process identifier for tracking - pJobClass => 'high' -- Oracle Scheduler job class for resource management - ); - - DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order'); -END; -/ - --- Export historical data to HIST bucket (Parquet) --- NEW v2.4.0: Per-column date format handling with template table -BEGIN - DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...'); + DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (ALL data)...'); DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY'); CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE( @@ -251,7 +66,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY', - pMaxDate => &cutoff_date, + pMinDate => DATE '1900-01-01', -- Include all historical data + pMaxDate => DATE '9999-12-31', -- Include all future dates pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY', pJobClass => 'high' -- Oracle Scheduler job class for resource management @@ -264,8 +80,8 @@ END; PROMPT ======================================================================== PROMPT Group 1 Export Completed PROMPT ======================================================================== -PROMPT - LEGACY_DEBT: DATA + HIST exported -PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported +PROMPT - LEGACY_DEBT: HIST exported (ALL data) +PROMPT - LEGACY_DEBT_DAILY: HIST exported (ALL data) PROMPT ======================================================================== --============================================================================================================================= diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql index d8af1d8..c6173bc 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql @@ -1,10 +1,11 @@ -- ===================================================================================== -- Script: 03_MARS_835_verify_exports.sql --- Purpose: Verify exported files exist in DATA and HIST buckets after export +-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy) -- Author: Grzegorz Michalski -- Created: 2025-12-17 +-- Updated: 2026-02-24 (Changed to HIST-only verification) -- MARS Issue: MARS-835 --- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/ +-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/ -- ===================================================================================== SET SERVEROUTPUT ON SIZE UNLIMITED; @@ -13,17 +14,14 @@ SET VERIFY OFF; SET LINESIZE 200; PROMPT ===================================================================================== -PROMPT MARS-835 Verification: Listing exported files in DATA and HIST buckets +PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy) PROMPT ===================================================================================== DECLARE - vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); vFileCount NUMBER := 0; - vTotalDataFiles NUMBER := 0; vTotalHistFiles NUMBER := 0; - vTotalDataSize NUMBER := 0; vTotalHistSize NUMBER := 0; TYPE t_folder_info IS RECORD ( @@ -33,25 +31,18 @@ DECLARE ); TYPE t_folder_list IS TABLE OF t_folder_info; - vDataFolders t_folder_list; vHistFolders t_folder_list; BEGIN - -- Get bucket URIs and credential from FILE_MANAGER - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + -- Get bucket URI and credential from FILE_MANAGER vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3')); - DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri); DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri); DBMS_OUTPUT.PUT_LINE(''); - -- Initialize folder lists - vDataFolders := t_folder_list( - t_folder_info('ODS/CSDB/CSDB_DEBT/', 'DEBT', 'CSV'), - t_folder_info('ODS/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'CSV') - ); - + -- Initialize folder list (all tables in HIST) + -- Initialize folder list (all 6 tables in HIST) vHistFolders := t_folder_list( t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'), t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'), @@ -62,49 +53,7 @@ BEGIN ); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket Exports (CSV format - last 6 months)'); - DBMS_OUTPUT.PUT_LINE('====================================================================================='); - - -- Check DATA bucket exports - FOR i IN 1..vDataFolders.COUNT LOOP - vFileCount := 0; - - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('Table: ' || vDataFolders(i).table_name || ' (' || vDataFolders(i).expected_format || ')'); - DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i).folder_name); - DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------'); - - BEGIN - FOR rec IN ( - SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || vDataFolders(i).folder_name - )) - WHERE object_name LIKE '%.csv' - ORDER BY created DESC - ) LOOP - vFileCount := vFileCount + 1; - vTotalDataFiles := vTotalDataFiles + 1; - vTotalDataSize := vTotalDataSize + rec.bytes; - DBMS_OUTPUT.PUT_LINE(' [' || vFileCount || '] ' || rec.object_name || - ' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date); - END LOOP; - - IF vFileCount = 0 THEN - DBMS_OUTPUT.PUT_LINE(' [ERROR] No CSV files found - Export may have failed!'); - ELSE - DBMS_OUTPUT.PUT_LINE(' [SUCCESS] Found ' || vFileCount || ' CSV file(s)'); - END IF; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(' [ERROR] Cannot access folder - ' || SQLERRM); - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning)'); + DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); -- Check HIST bucket exports @@ -155,24 +104,19 @@ BEGIN DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Export Verification Summary'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('DATA Bucket (CSV):'); - DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalDataFiles); - DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalDataSize/1024/1024/1024, 2) || ' GB'); - DBMS_OUTPUT.PUT_LINE(' - Expected tables: 2 (DEBT, DEBT_DAILY - last 6 months)'); - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet):'); + DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):'); DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalHistFiles || '+'); DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)'); - DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables with historical data)'); + DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables exported to HIST)'); DBMS_OUTPUT.PUT_LINE(''); - IF vTotalDataFiles >= 2 AND vTotalHistFiles >= 6 THEN + IF vTotalHistFiles >= 6 THEN DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL'); - DBMS_OUTPUT.PUT_LINE(' Files found in both DATA and HIST buckets'); + DBMS_OUTPUT.PUT_LINE(' Files found in HIST bucket for all tables'); DBMS_OUTPUT.PUT_LINE(' Proceed to record count verification (Step 4)'); - ELSIF vTotalDataFiles = 0 AND vTotalHistFiles = 0 THEN + ELSIF vTotalHistFiles = 0 THEN DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED'); - DBMS_OUTPUT.PUT_LINE(' No files found in either bucket'); + DBMS_OUTPUT.PUT_LINE(' No files found in HIST bucket'); DBMS_OUTPUT.PUT_LINE(' Review export logs for errors'); ELSE DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql index 88781c5..a0dd0e4 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql @@ -1,10 +1,11 @@ -- ===================================================================================== -- Script: 04_MARS_835_verify_record_counts.sql --- Purpose: Verify record counts match between source tables and exported data +-- Purpose: Verify record counts match between source tables and exported data (HIST-only) -- Author: Grzegorz Michalski -- Created: 2025-12-17 +-- Updated: 2026-02-24 (Changed to HIST-only verification) -- MARS Issue: MARS-835 --- Verification: Compare OU_CSDB source tables with ODS external tables +-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only) -- ===================================================================================== SET SERVEROUTPUT ON SIZE UNLIMITED; @@ -13,28 +14,23 @@ SET VERIFY OFF; SET LINESIZE 200; PROMPT ===================================================================================== -PROMPT MARS-835 Record Count Verification +PROMPT MARS-835 Record Count Verification (HIST-only strategy) PROMPT ===================================================================================== -PROMPT Comparing source table counts with exported external table counts +PROMPT Comparing source table counts with HIST external table counts PROMPT ===================================================================================== DECLARE TYPE t_table_info IS RECORD ( source_schema VARCHAR2(50), source_table VARCHAR2(100), - data_external_table VARCHAR2(100), - hist_external_table VARCHAR2(100), - has_data_export BOOLEAN, - has_hist_export BOOLEAN + hist_external_table VARCHAR2(100) ); TYPE t_table_list IS TABLE OF t_table_info; vTables t_table_list; vSourceCount NUMBER; - vDataCount NUMBER; vHistCount NUMBER; vTotalSourceCount NUMBER := 0; - vTotalDataCount NUMBER := 0; vTotalHistCount NUMBER := 0; vMismatchCount NUMBER := 0; vSql VARCHAR2(4000); @@ -42,18 +38,18 @@ BEGIN DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS')); DBMS_OUTPUT.PUT_LINE(''); - -- Initialize table list with export configuration + -- Initialize table list (all tables HIST-only) vTables := t_table_list( - t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ODS', 'ODS.CSDB_DEBT_ARCHIVE', TRUE, TRUE), - t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ODS', 'ODS.CSDB_DEBT_DAILY_ARCHIVE', TRUE, TRUE), - t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', NULL, 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE', FALSE, TRUE), - t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', NULL, 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE', FALSE, TRUE), - t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', NULL, 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE', FALSE, TRUE), - t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE) + t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'), + t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'), + t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'), + t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'), + t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'), + t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE') ); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status'); + DBMS_OUTPUT.PUT_LINE('Table Name Source Count HIST Count Status'); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); FOR i IN 1..vTables.COUNT LOOP @@ -70,31 +66,6 @@ BEGIN CONTINUE; END; - -- Get DATA external table count (if applicable) - IF vTables(i).has_data_export THEN - vSql := 'SELECT COUNT(*) FROM ' || vTables(i).data_external_table; - BEGIN - EXECUTE IMMEDIATE vSql INTO vDataCount; - vTotalDataCount := vTotalDataCount + vDataCount; - EXCEPTION - WHEN OTHERS THEN - -- If source table is empty (0 records), no files were exported - -- External table returns error, treat as 0 - -- Acceptable error codes: - -- ORA-29913: error in executing ODCIEXTTABLEOPEN callout - -- ORA-29400: data cartridge error - -- KUP-13023: nothing matched wildcard query (no files in bucket) - -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty - IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN - vDataCount := 0; - ELSE - vDataCount := -1; - END IF; - END; - ELSE - vDataCount := NULL; - END IF; - -- Get HIST external table count vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table; BEGIN @@ -119,18 +90,8 @@ BEGIN -- Display results DECLARE vStatus VARCHAR2(20); - vDataDisplay VARCHAR2(17); vHistDisplay VARCHAR2(17); BEGIN - -- Format DATA count display - IF vDataCount IS NULL THEN - vDataDisplay := 'N/A'; - ELSIF vDataCount = -1 THEN - vDataDisplay := 'ERROR'; - ELSE - vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999'); - END IF; - -- Format HIST count display IF vHistCount = -1 THEN vHistDisplay := 'ERROR'; @@ -138,35 +99,20 @@ BEGIN vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999'); END IF; - -- Determine status - IF vTables(i).has_data_export THEN - -- Split export: check DATA + HIST = SOURCE - IF (vDataCount + vHistCount) = vSourceCount THEN - vStatus := 'PASS'; - ELSIF vDataCount = -1 OR vHistCount = -1 THEN - vStatus := 'ERROR'; - vMismatchCount := vMismatchCount + 1; - ELSE - vStatus := 'MISMATCH'; - vMismatchCount := vMismatchCount + 1; - END IF; + -- Determine status (HIST only: check HIST = SOURCE) + IF vHistCount = vSourceCount THEN + vStatus := 'PASS'; + ELSIF vHistCount = -1 THEN + vStatus := 'ERROR'; + vMismatchCount := vMismatchCount + 1; ELSE - -- HIST only: check HIST = SOURCE - IF vHistCount = vSourceCount THEN - vStatus := 'PASS'; - ELSIF vHistCount = -1 THEN - vStatus := 'ERROR'; - vMismatchCount := vMismatchCount + 1; - ELSE - vStatus := 'MISMATCH'; - vMismatchCount := vMismatchCount + 1; - END IF; + vStatus := 'MISMATCH'; + vMismatchCount := vMismatchCount + 1; END IF; DBMS_OUTPUT.PUT_LINE( RPAD(vTables(i).source_table, 24) || LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) || - LPAD(vDataDisplay, 15) || LPAD(vHistDisplay, 15) || ' ' || vStatus ); @@ -177,18 +123,16 @@ BEGIN DBMS_OUTPUT.PUT_LINE( RPAD('TOTALS', 24) || LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) || - LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) || LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15) ); DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------'); DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary'); + DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999')); - DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)'); - DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)'); + DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)'); DBMS_OUTPUT.PUT_LINE(''); IF vMismatchCount = 0 THEN @@ -209,7 +153,6 @@ BEGIN DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)'); DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files'); DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (may not exist yet)'); - DBMS_OUTPUT.PUT_LINE(' N/A - Not applicable (table not exported to DATA)'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); EXCEPTION diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql index b5e283c..1b0fb70 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql @@ -1,68 +1,34 @@ --============================================================================================================================= -- MARS-835 ROLLBACK: Delete Group 1 Exported Files (DEBT, DEBT_DAILY) --============================================================================================================================= --- Purpose: Delete exported CSV and Parquet files from DATA and HIST buckets +-- Purpose: Delete exported Parquet files from HIST bucket (ARCHIVE only) -- WARNING: This will permanently delete exported data files! -- Author: Grzegorz Michalski -- Date: 2025-12-17 +-- Updated: 2026-02-24 (Changed to HIST-only rollback, no DATA bucket) -- Related: MARS-835 - CSDB Data Export Rollback --============================================================================================================================= SET SERVEROUTPUT ON SIZE UNLIMITED PROMPT ======================================================================== -PROMPT ROLLBACK: Deleting DEBT exported files +PROMPT ROLLBACK: Deleting DEBT exported files from HIST PROMPT ======================================================================== PROMPT WARNING: This will delete files from: -PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT/ PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/ PROMPT ======================================================================== DECLARE - vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); vFileCount NUMBER := 0; BEGIN - -- Get bucket URIs and credential - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + -- Get bucket URI and credential vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket'); - - -- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' - )) - WHERE object_name LIKE 'LEGACY_DEBT%' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - RAISE; - END IF; - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))'); - DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)'); - vFileCount := 0; + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS'); -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS FOR rec IN ( @@ -99,58 +65,23 @@ END; / PROMPT ======================================================================== -PROMPT ROLLBACK: Deleting DEBT_DAILY exported files +PROMPT ROLLBACK: Deleting DEBT_DAILY exported files from HIST PROMPT ======================================================================== PROMPT WARNING: This will delete files from: -PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/ PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/ PROMPT ======================================================================== DECLARE - vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); vFileCount NUMBER := 0; BEGIN - -- Get bucket URIs and credential - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + -- Get bucket URI and credential vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket'); - - -- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' - )) - WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - RAISE; - END IF; - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))'); - DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...'); - DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)'); - vFileCount := 0; + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS'); -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS FOR rec IN ( diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql index d08a4ea..c372d30 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql @@ -1,10 +1,11 @@ -- ===================================================================================== -- Script: 99_MARS_835_verify_rollback.sql --- Purpose: Verify all exported files have been deleted from DATA and HIST buckets +-- Purpose: Verify all exported files have been deleted from HIST bucket (HIST-only strategy) -- Author: Grzegorz Michalski -- Created: 2025-12-17 +-- Updated: 2026-02-24 (Changed to HIST-only verification) -- MARS Issue: MARS-835 --- Verification: Confirm complete rollback (no CSDB files remaining) +-- Verification: Confirm complete rollback (no CSDB files remaining in HIST) -- ===================================================================================== SET SERVEROUTPUT ON SIZE UNLIMITED; @@ -19,33 +20,23 @@ PROMPT Checking that all CSDB export files have been deleted PROMPT ===================================================================================== DECLARE - vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); - vDataFileCount NUMBER := 0; vHistFileCount NUMBER := 0; - vTotalFiles NUMBER := 0; TYPE t_folder_list IS TABLE OF VARCHAR2(200); - vDataFolders t_folder_list; vHistFolders t_folder_list; BEGIN - -- Get bucket URIs - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + -- Get bucket URI vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; DBMS_OUTPUT.PUT_LINE('ROLLBACK VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3')); - DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri); DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri); DBMS_OUTPUT.PUT_LINE(''); - -- Initialize folder lists - vDataFolders := t_folder_list( - 'ODS/CSDB/CSDB_DEBT/', - 'ODS/CSDB/CSDB_DEBT_DAILY/' - ); - + -- Initialize folder list (all 6 tables in HIST) + -- Initialize folder list (all 6 tables in HIST) vHistFolders := t_folder_list( 'ARCHIVE/CSDB/CSDB_DEBT/', 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/', @@ -55,47 +46,6 @@ BEGIN 'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/' ); - DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (should be empty)'); - DBMS_OUTPUT.PUT_LINE('====================================================================================='); - - -- Check DATA bucket - FOR i IN 1..vDataFolders.COUNT LOOP - DECLARE - vCount NUMBER := 0; - BEGIN - DBMS_OUTPUT.PUT_LINE(''); - DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i)); - - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || vDataFolders(i) - )) - WHERE object_name LIKE '%.csv' - ) LOOP - vCount := vCount + 1; - vDataFileCount := vDataFileCount + 1; - DBMS_OUTPUT.PUT_LINE(' [FOUND] ' || rec.object_name); - END LOOP; - - IF vCount = 0 THEN - DBMS_OUTPUT.PUT_LINE(' [OK] No CSV files found'); - ELSE - DBMS_OUTPUT.PUT_LINE(' [INFO] Found ' || vCount || ' file(s) - may be pre-existing files from before installation'); - END IF; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' [OK] Folder does not exist or is empty'); - ELSE - DBMS_OUTPUT.PUT_LINE(' [ERROR] ' || SQLERRM); - END IF; - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (should be empty)'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); @@ -139,24 +89,21 @@ BEGIN END; END LOOP; - vTotalFiles := vDataFileCount + vHistFileCount; - DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE('====================================================================================='); DBMS_OUTPUT.PUT_LINE('Rollback Verification Summary'); DBMS_OUTPUT.PUT_LINE('====================================================================================='); - DBMS_OUTPUT.PUT_LINE('DATA bucket files remaining: ' || vDataFileCount); DBMS_OUTPUT.PUT_LINE('HIST bucket files remaining: ' || vHistFileCount || '+'); - DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles || '+'); + DBMS_OUTPUT.PUT_LINE(''); DBMS_OUTPUT.PUT_LINE(''); - IF vTotalFiles = 0 THEN + IF vHistFileCount = 0 THEN DBMS_OUTPUT.PUT_LINE('[PASSED] ROLLBACK VERIFICATION PASSED'); DBMS_OUTPUT.PUT_LINE(' All CSDB export files have been deleted or were not created'); - DBMS_OUTPUT.PUT_LINE(' Buckets are clean and ready for re-export if needed'); + DBMS_OUTPUT.PUT_LINE(' HIST bucket is clean and ready for re-export if needed'); ELSE DBMS_OUTPUT.PUT_LINE('[INFO] ROLLBACK VERIFICATION COMPLETED'); - DBMS_OUTPUT.PUT_LINE(' Found ' || vTotalFiles || '+ file(s) remaining in buckets'); + DBMS_OUTPUT.PUT_LINE(' Found ' || vHistFileCount || '+ file(s) remaining in HIST bucket'); DBMS_OUTPUT.PUT_LINE(' NOTE: These may be pre-existing files from before installation.'); DBMS_OUTPUT.PUT_LINE(' Rollback only deletes files created during this export operation.'); DBMS_OUTPUT.PUT_LINE(' If needed, manually verify and clean up remaining files.');