admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Develop

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-02 11:13:24 +01:00
parent ecd833f682
commit ffcb288afa
62 changed files with 8463 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826-PREHOOK/new_version/DATA_EXPORTER.pkg
View File

@@ -16,7 +16,6 @@ AS
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY' || CHR(10) ||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10) ||
'v1.0.0 (2025-09-15): Initial implementation within FILE_MANAGER package' || CHR(10);

9
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql
View File

@@ -23,7 +23,8 @@ BEGIN
pTableName => 'LEGACY_ADHOC_ADJ_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER'
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported');
EXCEPTION
@@ -42,7 +43,8 @@ BEGIN
pTableName => 'LEGACY_ADHOC_ADJ_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM'
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported');
EXCEPTION
@@ -61,7 +63,8 @@ BEGIN
pTableName => 'LEGACY_ADHOC_ADJ_ITEM_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER'
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported');
EXCEPTION

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql
View File

@@ -28,7 +28,8 @@ BEGIN
pTableName => 'LEGACY_BALANCESHEET_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER'
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER',
pParallelDegree => 4
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported');
EXCEPTION
@@ -47,7 +48,8 @@ BEGIN
pTableName => 'LEGACY_BALANCESHEET_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM'
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM',
pParallelDegree => 16
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported');
EXCEPTION

9
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql
View File

@@ -23,7 +23,8 @@ BEGIN
pTableName => 'LEGACY_CSM_ADJ_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER'
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported');
EXCEPTION
@@ -42,7 +43,8 @@ BEGIN
pTableName => 'LEGACY_CSM_ADJ_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM'
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM',
pParallelDegree => 2
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported');
EXCEPTION
@@ -61,7 +63,8 @@ BEGIN
pTableName => 'LEGACY_CSM_ADJ_ITEM_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER'
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 2
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported');
EXCEPTION

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql
View File

@@ -28,7 +28,8 @@ BEGIN
pTableName => 'LEGACY_STANDING_FACILITY',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES'
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES',
pParallelDegree => 8
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported');
EXCEPTION
@@ -47,7 +48,8 @@ BEGIN
pTableName => 'LEGACY_STANDING_FACILITY_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER'
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER',
pParallelDegree => 2
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported');
EXCEPTION

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql
View File

@@ -24,7 +24,8 @@ BEGIN
pTableName => 'LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER'
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER',
pParallelDegree => 2
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported');
EXCEPTION
@@ -43,7 +44,8 @@ BEGIN
pTableName => 'LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM'
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM',
pParallelDegree => 16
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported');
EXCEPTION

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/06_MARS_826_export_FORECAST_tables.sql
View File

@@ -28,7 +28,8 @@ BEGIN
pTableName => 'LEGACY_FORECAST_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER'
pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER',
pParallelDegree => 4
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported');
EXCEPTION
@@ -47,7 +48,8 @@ BEGIN
pTableName => 'LEGACY_FORECAST_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM'
pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM',
pParallelDegree => 16
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported');
EXCEPTION

9
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql
View File

@@ -23,7 +23,8 @@ BEGIN
pTableName => 'LEGACY_QR_ADJ_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER'
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported');
EXCEPTION
@@ -42,7 +43,8 @@ BEGIN
pTableName => 'LEGACY_QR_ADJ_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM'
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM',
pParallelDegree => 4
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported');
EXCEPTION
@@ -61,7 +63,8 @@ BEGIN
pTableName => 'LEGACY_QR_ADJ_ITEM_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER'
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER',
pParallelDegree => 2
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported');
EXCEPTION

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-826/08_MARS_826_export_TTS_tables.sql
View File

@@ -23,7 +23,8 @@ BEGIN
pTableName => 'LEGACY_TTS_HEADER',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER'
pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported');
EXCEPTION
@@ -42,7 +43,8 @@ BEGIN
pTableName => 'LEGACY_TTS_ITEM',
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM'
pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM',
pParallelDegree => 1
);
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported');
EXCEPTION

82
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/README.md
View File

@@ -1,82 +0,0 @@
# MARS-835-PREHOOK: Parallel Processing for DATA_EXPORTER
## Overview
Implements parallel partition processing for DATA_EXPORTER package using **DBMS_PARALLEL_EXECUTE** framework.
## Changes Summary
### ENV_MANAGER v3.1.0 → v3.2.0
- Added `CODE_INVALID_PARALLEL_DEGREE` (-20110) error code
- Added `CODE_PARALLEL_EXECUTION_FAILED` (-20111) error code
- Added corresponding message constants and exception declarations
### DATA_EXPORTER v2.2.0 → v2.3.0
- Added `pParallelDegree` parameter to `EXPORT_TABLE_DATA_BY_DATE` (default: 1, range: 1-16)
- Added `pParallelDegree` parameter to `EXPORT_TABLE_DATA_TO_CSV_BY_DATE` (default: 1, range: 1-16)
- Implemented `EXPORT_PARTITION_PARALLEL` callback procedure for DBMS_PARALLEL_EXECUTE
- Created global temporary table `A_PARALLEL_EXPORT_CHUNKS` for chunk management
- Sequential processing when `pParallelDegree = 1` (default - safest option)
- Parallel processing via DBMS_PARALLEL_EXECUTE when `pParallelDegree > 1`
- Automatic error detection and reporting through `USER_PARALLEL_EXECUTE_CHUNKS`
## Installation
### Prerequisites
- Oracle Database 23ai or higher (DBMS_PARALLEL_EXECUTE support)
- ADMIN privileges for table creation
- CT_MRDS schema for package deployment
### Installation Command
```powershell
cd .\MARS_Packages\REL01_POST_DEACTIVATION\MARS-835-PREHOOK
echo "YES" | sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars835_prehook.sql"
```
## Usage Examples
### Parallel Export (8 threads)
```sql
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
pSchemaName => 'OU_TOP',
pTableName => 'AGGREGATED_ALLOTMENT',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'ARCHIVE',
pFolderName => 'parallel_export',
pMinDate => DATE '2020-01-01',
pMaxDate => SYSDATE,
pParallelDegree => 8
);
END;
/
```
### Sequential Export
```sql
BEGIN
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
pSchemaName => 'OU_TOP',
pTableName => 'AGGREGATED_ALLOTMENT',
pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY',
pBucketArea => 'DATA',
pFolderName => 'sequential_export',
pParallelDegree => 1 -- Sequential
);
END;
/
```
## Test Results
✅ Installation successful
✅ ENV_MANAGER v3.2.0 compiled
✅ DATA_EXPORTER v2.3.0 compiled
✅ Zero partition handling works correctly
✅ DBMS_PARALLEL_EXECUTE framework verified
## Rollback
```powershell
sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars835_prehook.sql"
```
## Author
Grzegorz Michalski - 2025-12-20

6
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql
View File

@@ -43,6 +43,9 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
FILE_BASE_NAME VARCHAR2(1000),
TEMPLATE_TABLE_NAME VARCHAR2(200),
MAX_FILE_SIZE NUMBER DEFAULT 104857600 NOT NULL,
STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL,
ERROR_MESSAGE VARCHAR2(4000),
EXPORT_TIMESTAMP TIMESTAMP,
CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL
);
@@ -66,4 +69,7 @@ COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FORMAT_TYPE IS 'Export format
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FILE_BASE_NAME IS 'Base filename for CSV exports (NULL for Parquet)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TEMPLATE_TABLE_NAME IS 'Template table name for per-column date format configuration (e.g., CT_ET_TEMPLATES.TABLE_NAME)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MAX_FILE_SIZE IS 'Maximum file size in bytes for CSV exports only (e.g., 104857600 = 100MB, 1073741824 = 1GB) - default 100MB (104857600). NOTE: Not applicable for PARQUET format (Oracle limitation)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.STATUS IS 'Chunk processing status: PENDING (not started), PROCESSING (in progress), COMPLETED (success), FAILED (error) - allows retry of failed partitions only';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.ERROR_MESSAGE IS 'Error message if chunk processing failed (STATUS = FAILED)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.EXPORT_TIMESTAMP IS 'Timestamp when chunk export was completed (STATUS = COMPLETED)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CREATED_DATE IS 'Timestamp when chunk was created';

173
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb
View File

@@ -17,6 +17,39 @@ AS
----------------------------------------------------------------------------------------------------
/**
* Deletes export file from OCI bucket if it exists (used for cleanup before retry)
* Silently ignores if file doesn't exist (ORA-20404)
**/
PROCEDURE DELETE_FAILED_EXPORT_FILE(
pFileUri IN VARCHAR2,
pCredentialName IN VARCHAR2,
pParameters IN VARCHAR2
) IS
BEGIN
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to delete potentially corrupted file: ' || pFileUri, 'DEBUG', pParameters);
DBMS_CLOUD.DELETE_OBJECT(
credential_name => pCredentialName,
object_uri => pFileUri
);
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted existing file (cleanup before retry): ' || pFileUri, 'INFO', pParameters);
EXCEPTION
WHEN OTHERS THEN
-- Object not found is OK (file doesn't exist)
IF SQLCODE = -20404 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('File does not exist (OK): ' || pFileUri, 'DEBUG', pParameters);
ELSE
-- Log but don't fail - export will attempt anyway
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete file (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
END IF;
END;
END DELETE_FAILED_EXPORT_FILE;
----------------------------------------------------------------------------------------------------
/**
* Builds query with TO_CHAR for date/timestamp columns using per-column formats
* Retrieves format for each date column from FILE_MANAGER.GET_DATE_FORMAT
@@ -394,6 +427,10 @@ AS
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters);
-- Delete potentially corrupted file from previous failed attempt
-- This prevents Oracle from creating _1 suffixed files on retry
DELETE_FAILED_EXPORT_FILE(vUri, pCredentialName, pParameters);
DBMS_CLOUD.EXPORT_DATA(
credential_name => pCredentialName,
file_uri_list => vUri,
@@ -409,6 +446,10 @@ AS
ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters);
-- Delete potentially corrupted file from previous failed attempt
-- This prevents Oracle from creating _1 suffixed files on retry
DELETE_FAILED_EXPORT_FILE(vUri, pCredentialName, pParameters);
-- Use json_object() for CSV export with maxfilesize in bytes (Oracle requirement)
-- Oracle maxfilesize: min 10MB (10485760), max 1GB (1073741824), default 10MB
-- NOTE: maxfilesize must be NUMBER (bytes), not string like '1000M'
@@ -499,6 +540,13 @@ AS
vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId;
ENV_MANAGER.LOG_PROCESS_EVENT('Starting parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
-- Mark chunk as PROCESSING
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'PROCESSING',
ERROR_MESSAGE = NULL
WHERE CHUNK_ID = pStartId;
COMMIT;
-- Call the worker procedure
EXPORT_SINGLE_PARTITION(
pSchemaName => vSchemaName,
@@ -518,11 +566,29 @@ AS
pParameters => vParameters
);
-- Mark chunk as COMPLETED
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'COMPLETED',
EXPORT_TIMESTAMP = SYSTIMESTAMP,
ERROR_MESSAGE = NULL
WHERE CHUNK_ID = pStartId;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('Completed parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
EXCEPTION
WHEN OTHERS THEN
-- Capture error details in variable (SQLERRM cannot be used directly in SQL)
vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
-- Mark chunk as FAILED with error message
-- Use vgMsgTmp variable instead of SQLERRM directly (Oracle limitation in SQL context)
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
SET STATUS = 'FAILED',
ERROR_MESSAGE = SUBSTR(vgMsgTmp, 1, 4000)
WHERE CHUNK_ID = pStartId;
COMMIT;
RAISE;
END EXPORT_PARTITION_PARALLEL;
@@ -798,23 +864,50 @@ AS
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Using parallel processing with ' || pParallelDegree || ' threads', 'INFO', vParameters);
-- Clear any existing chunks from previous runs (TRUNCATE to avoid PK violations)
EXECUTE IMMEDIATE 'TRUNCATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS';
-- Clean up old completed chunks (>24 hours) to prevent table bloat
-- CRITICAL: Do NOT delete chunks from other active sessions (same-day tasks)
-- This prevents race conditions when multiple exports run simultaneously
DELETE FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
WHERE STATUS = 'COMPLETED'
AND CREATED_DATE < SYSTIMESTAMP - INTERVAL '1' DAY;
COMMIT;
-- Populate chunks table
ENV_MANAGER.LOG_PROCESS_EVENT('Cleared old COMPLETED chunks (>24h). Active session chunks preserved.', 'DEBUG', vParameters);
-- This prevents re-exporting successfully completed partitions
DELETE FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'COMPLETED';
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('Cleared COMPLETED chunks. FAILED chunks retained for retry.', 'DEBUG', vParameters);
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
FOR i IN 1 .. vPartitions.COUNT LOOP
INSERT INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE
) VALUES (
i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600
);
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id)
WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS)
VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, 'PENDING')
WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
END LOOP;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('Populated ' || vPartitions.COUNT || ' chunks for parallel export', 'DEBUG', vParameters);
-- Log chunk statistics
DECLARE
vPendingCount NUMBER;
vFailedCount NUMBER;
BEGIN
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
END;
-- Create parallel task
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
@@ -856,7 +949,8 @@ AS
-- Clean up task
DBMS_PARALLEL_EXECUTE.DROP_TASK(task_name => vTaskName);
-- Clean up chunks for this task
-- Clean up chunks for THIS specific task only (session-safe)
-- CRITICAL: Use TASK_NAME filter to avoid deleting chunks from other active sessions
DELETE FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = vTaskName;
COMMIT;
@@ -1055,23 +1149,45 @@ AS
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Using parallel processing with ' || pParallelDegree || ' threads', 'INFO', vParameters);
-- Clear any existing chunks from previous runs (TRUNCATE to avoid PK violations)
EXECUTE IMMEDIATE 'TRUNCATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS';
-- Clean up old completed chunks (>24 hours) to prevent table bloat
-- CRITICAL: Do NOT delete chunks from other active sessions (same-day tasks)
-- This prevents race conditions when multiple CSV exports run simultaneously
DELETE FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
WHERE STATUS = 'COMPLETED'
AND CREATED_DATE < SYSTIMESTAMP - INTERVAL '1' DAY;
COMMIT;
-- Populate chunks table
ENV_MANAGER.LOG_PROCESS_EVENT('Cleared old COMPLETED chunks (>24h). Active session chunks preserved.', 'DEBUG', vParameters);
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
FOR i IN 1 .. vPartitions.COUNT LOOP
INSERT INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE
) VALUES (
i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize
);
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
ON (t.CHUNK_ID = s.chunk_id)
WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS)
VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, 'PENDING')
WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
END LOOP;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('Populated ' || vPartitions.COUNT || ' chunks for parallel CSV export', 'DEBUG', vParameters);
-- Log chunk statistics
DECLARE
vPendingCount NUMBER;
vFailedCount NUMBER;
BEGIN
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
END;
-- Create parallel task
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
@@ -1113,7 +1229,8 @@ AS
-- Clean up task
DBMS_PARALLEL_EXECUTE.DROP_TASK(task_name => vTaskName);
-- Clean up chunks for this task
-- Clean up chunks for THIS specific task only (session-safe)
-- CRITICAL: Use TASK_NAME filter to avoid deleting chunks from other active CSV sessions
DELETE FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = vTaskName;
COMMIT;

8
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg
View File

@@ -9,12 +9,16 @@ AS
**/
-- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.5.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2026-01-26 13:30:00';
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.6.3';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2026-01-28 19:30:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.6.3 (2026-01-28): COMPILATION FIX - Resolved ORA-00904 error in EXPORT_PARTITION_PARALLEL. SQLERRM and DBMS_UTILITY.FORMAT_ERROR_BACKTRACE cannot be used directly in SQL UPDATE statements. Now properly assigned to vgMsgTmp variable before UPDATE.' || CHR(10) ||
'v2.6.2 (2026-01-28): CRITICAL FIX - Race condition when multiple exports run simultaneously. Changed DELETE to filter by age (>24h) instead of deleting all COMPLETED chunks. Prevents concurrent sessions from deleting each other chunks. Session-safe cleanup with TASK_NAME filtering. Enables true parallel execution of multiple export jobs.' || CHR(10) ||
'v2.6.1 (2026-01-28): Added DELETE_FAILED_EXPORT_FILE procedure to clean up partial/corrupted files before retry. When partition fails mid-export, partial file is deleted before retry to prevent Oracle from creating _1 suffixed duplicates. Ensures clean retry without orphaned files in OCI bucket.' || CHR(10) ||
'v2.6.0 (2026-01-28): CRITICAL FIX - Added STATUS tracking to A_PARALLEL_EXPORT_CHUNKS table to prevent data duplication on retry. System now restarts ONLY failed partitions instead of re-exporting all data. Added ERROR_MESSAGE and EXPORT_TIMESTAMP columns for better error handling and monitoring. Prevents duplicate file creation when parallel tasks fail (e.g., 22 partitions with 16 threads, 3 failures no longer duplicates 19 successful exports).' || CHR(10) ||
'v2.5.0 (2026-01-26): Added recorddelimiter parameter with CRLF (CHR(13)||CHR(10)) for CSV exports to ensure Windows-compatible line endings. Improves cross-platform compatibility when CSV files are opened in Windows applications (Notepad, Excel).' || CHR(10) ||
'v2.4.0 (2026-01-11): Added pTemplateTableName parameter for per-column date format configuration. Implements dynamic query building with TO_CHAR for each date/timestamp column using FILE_MANAGER.GET_DATE_FORMAT. Supports 3-tier hierarchy: column-specific, template DEFAULT, global fallback. Eliminates single dateformat limitation of DBMS_CLOUD.EXPORT_DATA.' || CHR(10) ||
'v2.3.0 (2025-12-20): Added parallel partition processing using DBMS_PARALLEL_EXECUTE. New pParallelDegree parameter (1-16, default 1) for EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures. Each year/month partition processed in separate thread for improved performance.' || CHR(10) ||

1196
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/rollback_version/v2.5.0/DATA_EXPORTER.pkb Normal file
View File

File diff suppressed because it is too large Load Diff

214
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/rollback_version/v2.5.0/DATA_EXPORTER.pkg Normal file
View File

@@ -0,0 +1,214 @@
create or replace PACKAGE CT_MRDS.DATA_EXPORTER
AUTHID CURRENT_USER
AS
/**
* Data Export Package: Provides comprehensive data export capabilities to various formats (CSV, Parquet)
* with support for cloud storage integration via Oracle Cloud Infrastructure (OCI).
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.5.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2026-01-26 13:30:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.5.0 (2026-01-26): Added recorddelimiter parameter with CRLF (CHR(13)||CHR(10)) for CSV exports to ensure Windows-compatible line endings. Improves cross-platform compatibility when CSV files are opened in Windows applications (Notepad, Excel).' || CHR(10) ||
'v2.4.0 (2026-01-11): Added pTemplateTableName parameter for per-column date format configuration. Implements dynamic query building with TO_CHAR for each date/timestamp column using FILE_MANAGER.GET_DATE_FORMAT. Supports 3-tier hierarchy: column-specific, template DEFAULT, global fallback. Eliminates single dateformat limitation of DBMS_CLOUD.EXPORT_DATA.' || CHR(10) ||
'v2.3.0 (2025-12-20): Added parallel partition processing using DBMS_PARALLEL_EXECUTE. New pParallelDegree parameter (1-16, default 1) for EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures. Each year/month partition processed in separate thread for improved performance.' || CHR(10) ||
'v2.2.0 (2025-12-19): DRY refactoring - extracted shared helper functions (sanitizeFilename, VALIDATE_TABLE_AND_COLUMNS, GET_PARTITIONS, EXPORT_SINGLE_PARTITION worker procedure). Reduced code duplication by ~400 lines. Prepared architecture for v2.3.0 parallel processing.' || CHR(10) ||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgMsgTmp VARCHAR2(32000);
---------------------------------------------------------------------------------------------------------------------------
-- TYPE DEFINITIONS FOR PARTITION HANDLING
---------------------------------------------------------------------------------------------------------------------------
/**
* Record type for year/month partition information
**/
TYPE partition_rec IS RECORD (
year VARCHAR2(4),
month VARCHAR2(2)
);
/**
* Table type for collection of partition records
**/
TYPE partition_tab IS TABLE OF partition_rec;
---------------------------------------------------------------------------------------------------------------------------
-- INTERNAL PARALLEL PROCESSING CALLBACK
---------------------------------------------------------------------------------------------------------------------------
/**
* @name EXPORT_PARTITION_PARALLEL
* @desc Internal callback procedure for DBMS_PARALLEL_EXECUTE.
* Processes single partition (year/month) chunk in parallel task.
* Called by DBMS_PARALLEL_EXECUTE framework for each chunk.
* This procedure is PUBLIC because DBMS_PARALLEL_EXECUTE requires it,
* but should NOT be called directly by external code.
* @param pStartId - Chunk start ID (CHUNK_ID from A_PARALLEL_EXPORT_CHUNKS table)
* @param pEndId - Chunk end ID (same as pStartId for single-row chunks)
**/
PROCEDURE EXPORT_PARTITION_PARALLEL (
pStartId IN NUMBER,
pEndId IN NUMBER
);
---------------------------------------------------------------------------------------------------------------------------
-- MAIN EXPORT PROCEDURES
---------------------------------------------------------------------------------------------------------------------------
/**
* @name EXPORT_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data into CSV file on OCI infrustructure.
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* @example
* begin
* DATA_EXPORTER.EXPORT_TABLE_DATA(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'csv_exports'
* );
* end;
**/
PROCEDURE EXPORT_TABLE_DATA (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
/**
* @name EXPORT_TABLE_DATA_BY_DATE
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data into PARQUET files on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
* Validates that all columns in pColumnList exist in the target table.
* Automatically adds 'T.' prefix to column names in pColumnList.
* Supports parallel partition processing via pParallelDegree parameter (default 1, range 1-16).
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* @example
* begin
* DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'parquet_exports',
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
* pMinDate => DATE '2024-01-01',
* pMaxDate => SYSDATE,
* pParallelDegree => 8 -- Optional, default 1, range 1-16
* );
* end;
**/
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pParallelDegree IN NUMBER default 1,
pTemplateTableName IN VARCHAR2 default NULL,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
/**
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
* @desc Exports data to separate CSV files partitioned by year and month.
* Creates one CSV file for each year/month combination found in the data.
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY as EXPORT_TABLE_DATA_BY_DATE,
* but exports to CSV format instead of Parquet.
* Supports parallel partition processing via pParallelDegree parameter (1-16).
* File naming pattern: {pFileName}_YYYYMM.csv or {TABLENAME}_YYYYMM.csv (if pFileName is NULL)
* @example
* begin
* -- With custom filename
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'exports',
* pFileName => 'my_export.csv',
* pMinDate => DATE '2024-01-01',
* pMaxDate => SYSDATE,
* pParallelDegree => 8 -- Optional, default 1, range 1-16
* );
*
* -- With auto-generated filename (based on table name only)
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
* pSchemaName => 'OU_TOP',
* pTableName => 'AGGREGATED_ALLOTMENT',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'ARCHIVE',
* pFolderName => 'exports',
* pMinDate => DATE '2025-09-01',
* pMaxDate => DATE '2025-09-17'
* );
* -- This will create files like: AGGREGATED_ALLOTMENT_202509.csv, etc.
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* end;
**/
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pFileName IN VARCHAR2 DEFAULT NULL,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pParallelDegree IN NUMBER default 1,
pTemplateTableName IN VARCHAR2 default NULL,
pMaxFileSize IN NUMBER default 104857600,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
---------------------------------------------------------------------------------------------------------------------------
-- VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* Returns the current package version number
* return: Version string in format X.Y.Z (e.g., '2.1.0')
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* Returns comprehensive build information including version, date, and author
* return: Formatted string with complete build details
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* Returns the version history with recent changes
* return: Multi-line string with version history
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END;
/

22
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835/01_MARS_835_install_step1.sql
View File

@@ -14,7 +14,7 @@
SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON
DEFINE cutoff_date = "ADD_MONTHS(SYSDATE, -6)"
DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"
PROMPT ========================================================================
PROMPT Exporting CSDB.DEBT - Split DATA + HIST
@@ -69,7 +69,11 @@ BEGIN
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
@@ -100,7 +104,7 @@ BEGIN
pFolderName => 'ODS/CSDB/CSDB_DEBT',
pMinDate => &cutoff_date,
pMaxDate => SYSDATE,
pParallelDegree => 8,
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
pMaxFileSize => 104857600 -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
);
@@ -122,7 +126,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
pMaxDate => &cutoff_date,
pParallelDegree => 8,
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT'
);
@@ -184,7 +188,11 @@ BEGIN
BEGIN
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
DBMS_OUTPUT.PUT_LINE('>>>');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('');
@@ -215,7 +223,7 @@ BEGIN
pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
pMinDate => &cutoff_date,
pMaxDate => SYSDATE,
pParallelDegree => 8,
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
pMaxFileSize => 104857600 -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
);
@@ -237,7 +245,7 @@ BEGIN
pBucketArea => 'ARCHIVE',
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
pMaxDate => &cutoff_date,
pParallelDegree => 8,
pParallelDegree => 16,
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY'
);

65
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835/04_MARS_835_verify_record_counts.sql
View File

@@ -52,9 +52,9 @@ BEGIN
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE)
);
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
FOR i IN 1..vTables.COUNT LOOP
-- Get source table count
@@ -78,7 +78,18 @@ BEGIN
vTotalDataCount := vTotalDataCount + vDataCount;
EXCEPTION
WHEN OTHERS THEN
vDataCount := -1;
-- If source table is empty (0 records), no files were exported
-- External table returns error, treat as 0
-- Acceptable error codes:
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vDataCount := 0;
ELSE
vDataCount := -1;
END IF;
END;
ELSE
vDataCount := NULL;
@@ -91,14 +102,25 @@ BEGIN
vTotalHistCount := vTotalHistCount + vHistCount;
EXCEPTION
WHEN OTHERS THEN
vHistCount := -1;
-- If source table is empty (0 records), no files were exported
-- External table returns error, treat as 0
-- Acceptable error codes:
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-- ORA-29400: data cartridge error
-- KUP-13023: nothing matched wildcard query (no files in bucket)
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
vHistCount := 0;
ELSE
vHistCount := -1;
END IF;
END;
-- Display results
DECLARE
vStatus VARCHAR2(20);
vDataDisplay VARCHAR2(15);
vHistDisplay VARCHAR2(15);
vDataDisplay VARCHAR2(17);
vHistDisplay VARCHAR2(17);
BEGIN
-- Format DATA count display
IF vDataCount IS NULL THEN
@@ -106,14 +128,14 @@ BEGIN
ELSIF vDataCount = -1 THEN
vDataDisplay := 'ERROR';
ELSE
vDataDisplay := TO_CHAR(vDataCount, '999,999,999');
vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999');
END IF;
-- Format HIST count display
IF vHistCount = -1 THEN
vHistDisplay := 'ERROR';
ELSE
vHistDisplay := TO_CHAR(vHistCount, '999,999,999');
vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
END IF;
-- Determine status
@@ -143,27 +165,30 @@ BEGIN
DBMS_OUTPUT.PUT_LINE(
RPAD(vTables(i).source_table, 24) ||
LPAD(TO_CHAR(vSourceCount, '999,999,999'), 13) ||
LPAD(vDataDisplay, 13) ||
LPAD(vHistDisplay, 13) || ' ' ||
LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
LPAD(vDataDisplay, 15) ||
LPAD(vHistDisplay, 15) || ' ' ||
vStatus
);
END;
END LOOP;
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('TOTALS:' || LPAD(TO_CHAR(vTotalSourceCount, '999,999,999'), 17) ||
LPAD(TO_CHAR(vTotalDataCount, '999,999,999'), 13) ||
LPAD(TO_CHAR(vTotalHistCount, '999,999,999'), 13));
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE(
RPAD('TOTALS', 24) ||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) ||
LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
);
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
DBMS_OUTPUT.PUT_LINE('');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '999,999,999') || ' (last 6 months)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '999,999,999') || ' (historical + full exports)');
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)');
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)');
DBMS_OUTPUT.PUT_LINE('');
IF vMismatchCount = 0 THEN