Merge develop into main - DATA_EXPORTER v2.14.0 optimization

MARS_Packages/REL01_ADDITIONS/MARS-826/01_MARS_826_export_ADHOC_ADJ_tables.sql

@@ -25,6 +25,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported');
@@ -67,6 +69,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/02_MARS_826_export_BALANCESHEET_tables.sql

@@ -30,6 +30,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER',
         pParallelDegree => 4,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported');
@@ -51,6 +52,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM',
         pParallelDegree => 16,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/03_MARS_826_export_CSM_ADJ_tables.sql

@@ -25,6 +25,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM',
         pParallelDegree => 2,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported');
@@ -67,6 +69,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER',
         pParallelDegree => 2,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/04_MARS_826_export_STANDING_FACILITY_tables.sql

@@ -30,6 +30,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES',
         pParallelDegree => 8,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported');
@@ -51,6 +52,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER',
         pParallelDegree => 2,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/05_MARS_826_export_MRR_IND_CURRENT_ACCOUNT_tables.sql

@@ -26,6 +26,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER',
         pParallelDegree => 2,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported');
@@ -47,6 +48,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM',
         pParallelDegree => 16,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/06_MARS_826_export_FORECAST_tables.sql

@@ -30,6 +30,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER',
         pParallelDegree => 4,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported');
@@ -51,6 +52,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM',
         pParallelDegree => 16,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported');

MARS_Packages/REL01_ADDITIONS/MARS-826/07_MARS_826_export_QR_ADJ_tables.sql

@@ -25,6 +25,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM',
         pParallelDegree => 4,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported');
@@ -66,8 +68,7 @@ BEGIN
         pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER',
-        pParallelDegree => 2,
+        pParallelDegree => 2,        pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM_HEADER',        pJobClass => 'high'
-        pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported');
 EXCEPTION

MARS_Packages/REL01_ADDITIONS/MARS-826/08_MARS_826_export_TTS_tables.sql

@@ -25,6 +25,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_HEADER',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported');
@@ -46,6 +47,7 @@ BEGIN
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM',
         pParallelDegree => 1,
+        pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_ITEM',
         pJobClass => 'high'
     );
     DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported');

MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql

@@ -26,7 +26,7 @@ END;
 /
 
 CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
-    CHUNK_ID          NUMBER          PRIMARY KEY,
+    CHUNK_ID          NUMBER          NOT NULL,
     TASK_NAME         VARCHAR2(100)   NOT NULL,
     YEAR_VALUE        VARCHAR2(4)     NOT NULL,
     MONTH_VALUE       VARCHAR2(2)     NOT NULL,
@@ -47,14 +47,16 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
     STATUS              VARCHAR2(30)    DEFAULT 'PENDING' NOT NULL,
     ERROR_MESSAGE       VARCHAR2(4000),
     EXPORT_TIMESTAMP    TIMESTAMP,
-    CREATED_DATE        TIMESTAMP       DEFAULT SYSTIMESTAMP NOT NULL
+    CREATED_DATE        TIMESTAMP       DEFAULT SYSTIMESTAMP NOT NULL,
+    CONSTRAINT PK_PARALLEL_EXPORT_CHUNKS PRIMARY KEY (TASK_NAME, CHUNK_ID)
 );
 
-CREATE INDEX IX_PARALLEL_CHUNKS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(TASK_NAME);
+-- Index for status-based queries (e.g., WHERE STATUS = 'FAILED' AND TASK_NAME = ?)
+CREATE INDEX IX_PARALLEL_CHUNKS_STATUS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(STATUS, TASK_NAME);
 
-COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions';
+COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions. PK: (TASK_NAME, CHUNK_ID) ensures session isolation for concurrent exports.';
-COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Unique chunk identifier (partition number)';
+COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Chunk identifier within task (partition number) - unique per TASK_NAME, not globally';
-COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name for cleanup';
+COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name - session isolation key, part of composite PK with CHUNK_ID';
 COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.YEAR_VALUE IS 'Partition year (YYYY)';
 COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MONTH_VALUE IS 'Partition month (MM)';
 COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.SCHEMA_NAME IS 'Schema owning the source table';

MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb

@@ -18,34 +18,104 @@ AS
    ----------------------------------------------------------------------------------------------------
 
    /**
-      * Deletes export file from OCI bucket if it exists (used for cleanup before retry)
+      * Deletes ALL files matching specific file pattern before retry export
-      * Silently ignores if file doesn't exist (ORA-20404)
+      * Critical for preventing data duplication when DBMS_CLOUD.EXPORT_DATA fails mid-process
+      * 
+      * Problem: Export fails after creating partial file(s), retry creates new _2, _3 suffixed files
+      * Solution: Delete ALL files matching the base filename pattern before retry
+      * 
+      * Pattern matching strategy:
+      * - Parquet: folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/*.parquet (folder-level safe - each chunk has own partition folder)
+      * - CSV: folder/TABLENAME_202411*.csv (file-level pattern - multiple chunks share same folder!)
+      * 
+      * CRITICAL for parallel processing:
+      * - Parquet chunks are isolated by partition folder structure (safe to delete folder/*)
+      * - CSV chunks share flat folder structure - MUST use file-specific pattern (TABLENAME_YYYYMM*)
+      *   to avoid deleting files from other parallel chunks in same folder
    **/
    PROCEDURE DELETE_FAILED_EXPORT_FILE(
       pFileUri        IN VARCHAR2,
       pCredentialName IN VARCHAR2,
       pParameters     IN VARCHAR2
    ) IS
+      vBucketUri        VARCHAR2(4000);
+      vFolderPath       VARCHAR2(4000);
+      vFileName         VARCHAR2(1000);
+      vFileNamePattern  VARCHAR2(1000);
+      vSlashPos         NUMBER;
+      vDotPos           NUMBER;
+      vFilesDeleted     NUMBER := 0;
    BEGIN
-      BEGIN
+      -- Extract components from URI
-         ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to delete potentially corrupted file: ' || pFileUri, 'DEBUG', pParameters);
+      -- Example Parquet: https://.../bucket/folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/202411.parquet
+      -- Example CSV:     https://.../bucket/folder/TABLENAME_202411.csv
+      
+      -- Find last slash before filename
+      vSlashPos := INSTR(pFileUri, '/', -1);
+      
+      IF vSlashPos > 0 THEN
+         -- Extract filename from URI (after last slash)
+         vFileName := SUBSTR(pFileUri, vSlashPos + 1);
          
-         DBMS_CLOUD.DELETE_OBJECT(
+         -- Extract folder path (before last slash)
-            credential_name => pCredentialName,
+         vFolderPath := SUBSTR(pFileUri, 1, vSlashPos - 1);
-            object_uri => pFileUri
-         );
          
-         ENV_MANAGER.LOG_PROCESS_EVENT('Deleted existing file (cleanup before retry): ' || pFileUri, 'INFO', pParameters);
+         -- Find bucket URI (protocol + namespace + bucket name)
-      EXCEPTION
+         -- Bucket URI ends after /o/ in OCI Object Storage URLs
-         WHEN OTHERS THEN
+         vBucketUri := SUBSTR(pFileUri, 1, INSTR(pFileUri, '/o/') + 2);
-            -- Object not found is OK (file doesn't exist)
+         
-            IF SQLCODE = -20404 THEN
+         -- Extract relative folder path (after bucket)
-               ENV_MANAGER.LOG_PROCESS_EVENT('File does not exist (OK): ' || pFileUri, 'DEBUG', pParameters);
+         vFolderPath := SUBSTR(vFolderPath, LENGTH(vBucketUri) + 1);
-            ELSE
+         
-               -- Log but don't fail - export will attempt anyway
+         -- Create file pattern by removing extension
-               ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete file (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
+         -- Oracle adds suffixes BEFORE extension: file.csv -> file_1_timestamp.csv
-            END IF;
+         -- Pattern: file* matches file_1_timestamp.csv, file_2_timestamp.csv
-      END;
+         vDotPos := INSTR(vFileName, '.', -1);
+         IF vDotPos > 0 THEN
+            vFileNamePattern := SUBSTR(vFileName, 1, vDotPos - 1) || '%';
+         ELSE
+            vFileNamePattern := vFileName || '%';
+         END IF;
+         
+         ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup before retry - Pattern: ' || vFolderPath || '/' || vFileNamePattern, 'DEBUG', pParameters);
+         
+         -- List and delete ALL files matching pattern
+         -- CRITICAL: Uses file-specific pattern for CSV chunk isolation in shared folder
+         FOR rec IN (
+            SELECT object_name
+            FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
+               credential_name => pCredentialName,
+               location_uri => vBucketUri
+            ))
+            WHERE object_name LIKE vFolderPath || '/' || vFileNamePattern
+         ) LOOP
+            BEGIN
+               DBMS_CLOUD.DELETE_OBJECT(
+                  credential_name => pCredentialName,
+                  object_uri => vBucketUri || rec.object_name
+               );
+               
+               vFilesDeleted := vFilesDeleted + 1;
+               ENV_MANAGER.LOG_PROCESS_EVENT('Deleted partial file ' || vFilesDeleted || ': ' || rec.object_name, 'DEBUG', pParameters);
+            EXCEPTION
+               WHEN OTHERS THEN
+                  -- Log but continue - don't fail entire cleanup
+                  ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete ' || rec.object_name || ': ' || SQLERRM, 'WARNING', pParameters);
+            END;
+         END LOOP;
+         
+         IF vFilesDeleted > 0 THEN
+            ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup completed: Deleted ' || vFilesDeleted || ' partial file(s) from previous failed export', 'INFO', pParameters);
+         ELSE
+            ENV_MANAGER.LOG_PROCESS_EVENT('No existing files to clean up (pattern match: ' || vFileNamePattern || ')', 'DEBUG', pParameters);
+         END IF;
+      ELSE
+         ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cannot parse file URI for cleanup: ' || pFileUri, 'WARNING', pParameters);
+      END IF;
+   EXCEPTION
+      WHEN OTHERS THEN
+         -- Don't fail export if cleanup fails - log and continue
+         ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cleanup failed (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
    END DELETE_FAILED_EXPORT_FILE;
 
    ----------------------------------------------------------------------------------------------------
@@ -415,6 +485,8 @@ AS
                     AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
                     AND L.LOAD_START <  TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
 
+      ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: '||pFormat||')', 'DEBUG', pParameters);
+      ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
       -- Construct the URI based on format
       IF pFormat = 'PARQUET' THEN
          -- Parquet: Use Hive-style partitioning
@@ -425,6 +497,7 @@ AS
                  'PARTITION_MONTH=' || sanitizeFilename(pMonth) || '/' ||
                  sanitizeFilename(pYear) || sanitizeFilename(pMonth) || '.parquet';
          
+         
          ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters);
          
          -- Delete potentially corrupted file from previous failed attempt
@@ -445,6 +518,7 @@ AS
                  sanitizeFilename(vFileName);
          
          ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters);
+         ENV_MANAGER.LOG_PROCESS_EVENT('CSV maxfilesize: ' || pMaxFileSize || ' bytes (' || ROUND(pMaxFileSize/1048576, 2) || ' MB)', 'DEBUG', pParameters);
          
          -- Delete potentially corrupted file from previous failed attempt
          -- This prevents Oracle from creating _1 suffixed files on retry
@@ -472,8 +546,7 @@ AS
          RAISE_APPLICATION_ERROR(-20001, 'Unsupported format: ' || pFormat || '. Use PARQUET or CSV.');
       END IF;
 
-      ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: ' || pFormat || ')', 'DEBUG', pParameters);
+      ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || pYear || '/' || pMonth, 'DEBUG', pParameters);
-      ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
    END EXPORT_SINGLE_PARTITION;
 
    ----------------------------------------------------------------------------------------------------
@@ -485,7 +558,8 @@ AS
    **/
    PROCEDURE EXPORT_PARTITION_PARALLEL (
       pStartId IN NUMBER,
-      pEndId IN NUMBER
+      pEndId IN NUMBER,
+      pTaskName IN VARCHAR2 DEFAULT NULL
    ) IS
       vYear            VARCHAR2(4);
       vMonth           VARCHAR2(2);
@@ -502,9 +576,12 @@ AS
       vFileBaseName    VARCHAR2(1000);
       vMaxFileSize     NUMBER;
       vJobClass        VARCHAR2(128);
+      vTaskName        VARCHAR2(128);
       vParameters      VARCHAR2(4000);
    BEGIN
-      -- Retrieve chunk context from global temporary table
+      -- Retrieve chunk context from A_PARALLEL_EXPORT_CHUNKS table
+      -- CRITICAL: Filter by CHUNK_ID and TASK_NAME for precise session isolation
+      -- pTaskName parameter passed from RUN_TASK ensures deterministic single-row retrieval
       SELECT 
          YEAR_VALUE, 
          MONTH_VALUE, 
@@ -520,7 +597,8 @@ AS
          FORMAT_TYPE, 
          FILE_BASE_NAME,
          MAX_FILE_SIZE,
-         JOB_CLASS
+         JOB_CLASS,
+         TASK_NAME
       INTO 
          vYear, 
          vMonth, 
@@ -536,18 +614,22 @@ AS
          vFormat, 
          vFileBaseName,
          vMaxFileSize,
-         vJobClass
+         vJobClass,
+         vTaskName
       FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
-      WHERE CHUNK_ID = pStartId;
+      WHERE CHUNK_ID = pStartId
+        AND TASK_NAME = pTaskName;
 
-      vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId;
+      vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId || ', TaskName: ' || vTaskName;
       ENV_MANAGER.LOG_PROCESS_EVENT('Starting parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
 
       -- Mark chunk as PROCESSING
+      -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
       UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
       SET STATUS = 'PROCESSING',
           ERROR_MESSAGE = NULL
-      WHERE CHUNK_ID = pStartId;
+      WHERE CHUNK_ID = pStartId
+        AND TASK_NAME = vTaskName;
       COMMIT;
 
       -- Call the worker procedure
@@ -570,26 +652,30 @@ AS
       );
 
       -- Mark chunk as COMPLETED
+      -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
       UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
       SET STATUS = 'COMPLETED',
           EXPORT_TIMESTAMP = SYSTIMESTAMP,
           ERROR_MESSAGE = NULL
-      WHERE CHUNK_ID = pStartId;
+      WHERE CHUNK_ID = pStartId
+        AND TASK_NAME = vTaskName;
       COMMIT;
 
       ENV_MANAGER.LOG_PROCESS_EVENT('Completed parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
    EXCEPTION
       WHEN OTHERS THEN
          -- Capture error details in variable (SQLERRM cannot be used directly in SQL)
-         vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
+         vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || ', TaskName: ' || vTaskName || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
          ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
          
          -- Mark chunk as FAILED with error message
+         -- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
          -- Use vgMsgTmp variable instead of SQLERRM directly (Oracle limitation in SQL context)
          UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
          SET STATUS = 'FAILED',
              ERROR_MESSAGE = SUBSTR(vgMsgTmp, 1, 4000)
-         WHERE CHUNK_ID = pStartId;
+         WHERE CHUNK_ID = pStartId
+           AND TASK_NAME = vTaskName;
          COMMIT;
          
          RAISE;
@@ -1056,8 +1142,8 @@ AS
             -- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
             FOR i IN 1 .. vPartitions.COUNT LOOP
                MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
-               USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
+               USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
-               ON (t.CHUNK_ID = s.chunk_id)
+               ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
                WHEN NOT MATCHED THEN
                   INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
                           BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
@@ -1066,33 +1152,34 @@ AS
                           vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
                           pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING')
                WHEN MATCHED THEN
-                  UPDATE SET TASK_NAME = vTaskName,
+                  -- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
-                             STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
+                  -- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
+                  UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
                              ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
             END LOOP;
             COMMIT;
             
-            -- Log chunk statistics
+            -- Log chunk statistics (session-safe: only count chunks for THIS task)
             DECLARE
                vPendingCount NUMBER;
                vFailedCount NUMBER;
             BEGIN
-               SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
+               SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
-               SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
+               SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
                
-               ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
+               ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
             END;
             
             -- Create parallel task
             DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
             
-            -- Define chunks by number range (1 to partition count)
+            -- Define chunks using SQL query to ensure TASK_NAME isolation
-            DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
+            -- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
+            -- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
+            DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
                task_name    => vTaskName,
-               table_owner  => 'CT_MRDS',
+               sql_stmt     => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
-               table_name   => 'A_PARALLEL_EXPORT_CHUNKS',
+               by_rowid     => FALSE
-               table_column => 'CHUNK_ID',
-               chunk_size   => 1  -- Each partition is one chunk
             );
             
             -- Execute task in parallel
@@ -1101,7 +1188,7 @@ AS
             IF pJobClass IS NOT NULL THEN
                DBMS_PARALLEL_EXECUTE.RUN_TASK(
                   task_name      => vTaskName,
-                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
+                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
                   language_flag  => DBMS_SQL.NATIVE,
                   parallel_level => pParallelDegree,
                   job_class      => pJobClass
@@ -1109,7 +1196,7 @@ AS
             ELSE
                DBMS_PARALLEL_EXECUTE.RUN_TASK(
                   task_name      => vTaskName,
-                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
+                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
                   language_flag  => DBMS_SQL.NATIVE,
                   parallel_level => pParallelDegree
                );
@@ -1360,8 +1447,8 @@ AS
             -- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
             FOR i IN 1 .. vPartitions.COUNT LOOP
                MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
-               USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
+               USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
-               ON (t.CHUNK_ID = s.chunk_id)
+               ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
                WHEN NOT MATCHED THEN
                   INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
                           BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
@@ -1370,33 +1457,34 @@ AS
                           vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
                           pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING')
                WHEN MATCHED THEN
-                  UPDATE SET TASK_NAME = vTaskName,
+                  -- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
-                             STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
+                  -- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
+                  UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
                              ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
             END LOOP;
             COMMIT;
             
-            -- Log chunk statistics
+            -- Log chunk statistics (session-safe: only count chunks for THIS task)
             DECLARE
                vPendingCount NUMBER;
                vFailedCount NUMBER;
             BEGIN
-               SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
+               SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
-               SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
+               SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
                
-               ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
+               ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
             END;
             
             -- Create parallel task
             DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
             
-            -- Define chunks by number range (1 to partition count)
+            -- Define chunks using SQL query to ensure TASK_NAME isolation
-            DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
+            -- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
+            -- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
+            DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
                task_name    => vTaskName,
-               table_owner  => 'CT_MRDS',
+               sql_stmt     => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
-               table_name   => 'A_PARALLEL_EXPORT_CHUNKS',
+               by_rowid     => FALSE
-               table_column => 'CHUNK_ID',
-               chunk_size   => 1  -- Each partition is one chunk
             );
             
             -- Execute task in parallel
@@ -1405,7 +1493,7 @@ AS
             IF pJobClass IS NOT NULL THEN
                DBMS_PARALLEL_EXECUTE.RUN_TASK(
                   task_name      => vTaskName,
-                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
+                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
                   language_flag  => DBMS_SQL.NATIVE,
                   parallel_level => pParallelDegree,
                   job_class      => pJobClass
@@ -1413,7 +1501,7 @@ AS
             ELSE
                DBMS_PARALLEL_EXECUTE.RUN_TASK(
                   task_name      => vTaskName,
-                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
+                  sql_stmt       => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
                   language_flag  => DBMS_SQL.NATIVE,
                   parallel_level => pParallelDegree
                );

MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg

@@ -9,17 +9,17 @@ AS
    **/
 
    -- Package Version Information
-   PACKAGE_VERSION     CONSTANT VARCHAR2(10) := '2.11.0';
+   PACKAGE_VERSION     CONSTANT VARCHAR2(10) := '2.14.0';
-   PACKAGE_BUILD_DATE  CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00';
+   PACKAGE_BUILD_DATE  CONSTANT VARCHAR2(20) := '2026-02-25 09:00:00';
    PACKAGE_AUTHOR      CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
 
    -- Version History (last 3-5 changes)
    VERSION_HISTORY     CONSTANT VARCHAR2(4000) := 
-       'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) ||
+       'v2.14.0 (2026-02-25): OPTIMIZATION - Added pTaskName parameter to EXPORT_PARTITION_PARALLEL for deterministic filtering. Replaced FETCH FIRST 1 ROW ONLY safeguard with precise WHERE CHUNK_ID AND TASK_NAME filter. Eliminates ORDER BY overhead and provides cleaner session isolation.' || CHR(10) ||
-       'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) ||
+       'v2.13.1 (2026-02-25): CRITICAL FIX - Added START_ID and END_ID aliasses in CREATE_CHUNKS_BY_SQL to avoid ORA-00960 ambiguous column naming error.' || CHR(10) ||
-       'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) ||
+       'v2.13.0 (2026-02-25): CRITICAL SESSION ISOLATION FIX - Changed CREATE_CHUNKS_BY_NUMBER_COL to CREATE_CHUNKS_BY_SQL with TASK_NAME filter (fixes ORA-01422 in concurrent sessions). Added ORDER BY CREATED_DATE DESC FETCH FIRST 1 ROW safeguard to EXPORT_PARTITION_PARALLEL SELECT. Composite PK (TASK_NAME, CHUNK_ID) now fully functional.' || CHR(10) ||
-       'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) ||
+       'v2.12.0 (2026-02-24): CRITICAL FIX - Rewritten DELETE_FAILED_EXPORT_FILE to use file-specific pattern matching (prevents deleting parallel CSV chunks in shared folder). Added vQuery logging before DBMS_CLOUD calls. Added CSV maxfilesize logging.' || CHR(10) ||
-       'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10);
+       'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10);
 
    cgBL       CONSTANT  VARCHAR2(2)   := CHR(13)||CHR(10);
    vgMsgTmp             VARCHAR2(32000);
@@ -54,10 +54,12 @@ AS
       *           but should NOT be called directly by external code.
       * @param    pStartId - Chunk start ID (CHUNK_ID from A_PARALLEL_EXPORT_CHUNKS table)
       * @param    pEndId - Chunk end ID (same as pStartId for single-row chunks)
+      * @param    pTaskName - Task name for session isolation (optional, DEFAULT NULL for backward compatibility)
    **/
    PROCEDURE EXPORT_PARTITION_PARALLEL (
       pStartId IN NUMBER,
-      pEndId IN NUMBER
+      pEndId IN NUMBER,
+      pTaskName IN VARCHAR2 DEFAULT NULL
    );
 
    ---------------------------------------------------------------------------------------------------------------------------

MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql

@@ -1,125 +1,31 @@
 --=============================================================================================================================
--- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY)
+-- MARS-835: Export Group 1 - HIST Only (DEBT, DEBT_DAILY)
 --=============================================================================================================================
--- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet)
+-- Purpose: Export ALL data to HIST bucket (Parquet with Hive-style partitioning)
 --          Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
 --          Excludes legacy columns not required in new structure
---          USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats
+--          USES: DATA_EXPORTER v2.12.0 with pTemplateTableName for column order and date formats
 -- Author: Grzegorz Michalski
 -- Date: 2025-12-17
--- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName)
+-- Updated: 2026-02-24 (Changed to HIST-only export, no DATA bucket split)
 -- Related: MARS-835 - CSDB Data Export
 --=============================================================================================================================
 
 SET SERVEROUTPUT ON SIZE UNLIMITED
 SET TIMING ON
 
-DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"
-
 PROMPT ========================================================================
-PROMPT Exporting CSDB.DEBT - Split DATA + HIST
+PROMPT Exporting CSDB.DEBT - HIST Only
 PROMPT ========================================================================
-PROMPT Last 6 months to DATA bucket (CSV format)
+PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
-PROMPT Older data to HIST bucket (Parquet with partitioning)
 PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
 PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION
 PROMPT ========================================================================
 
--- PRE-EXPORT CHECK: List existing files and count records
+-- Export ALL data to HIST bucket (Parquet)
-DECLARE
+-- NEW v2.12.0: Per-column date format handling with template table, full data range
-    vFileCount NUMBER := 0;
-    vRecordCount NUMBER := 0;
-    vLocationUri VARCHAR2(1000);
 BEGIN
-    -- Get bucket URI for DATA bucket
+    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (ALL data)...');
-    vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/';
-    
-    -- Count existing files
-    SELECT COUNT(*)
-    INTO vFileCount
-    FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-        credential_name => 'OCI$RESOURCE_PRINCIPAL',
-        location_uri => vLocationUri
-    ))
-    WHERE object_name NOT LIKE '%/';  -- Exclude directories
-    
-    IF vFileCount > 0 THEN
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
-        DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
-        DBMS_OUTPUT.PUT_LINE('');
-        
-        -- List existing files
-        DBMS_OUTPUT.PUT_LINE('Existing files:');
-        FOR rec IN (
-            SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
-            FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-                credential_name => 'OCI$RESOURCE_PRINCIPAL',
-                location_uri => vLocationUri
-            ))
-            WHERE object_name NOT LIKE '%/'
-            ORDER BY object_name
-        ) LOOP
-            DBMS_OUTPUT.PUT_LINE('  - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
-        END LOOP;
-        
-        -- Count records in external table
-        BEGIN
-            EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
-            DBMS_OUTPUT.PUT_LINE('');
-            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
-            DBMS_OUTPUT.PUT_LINE('>>>');
-            DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
-            DBMS_OUTPUT.PUT_LINE('>>>');
-            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
-        EXCEPTION
-            WHEN OTHERS THEN
-                DBMS_OUTPUT.PUT_LINE('');
-                DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
-                DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
-        END;
-        
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('');
-    ELSE
-        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
-        DBMS_OUTPUT.PUT_LINE('');
-    END IF;
-END;
-/
-
--- Export recent data to DATA bucket (CSV)
--- NEW v2.4.0: Per-column date format handling with template table for column order
-BEGIN
-    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...');
-    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
-    
-    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
-        pSchemaName => 'OU_CSDB',
-        pTableName => 'LEGACY_DEBT',
-        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
-        pBucketArea => 'DATA',
-        pFolderName => 'ODS/CSDB/CSDB_DEBT',
-        pMinDate => &cutoff_date,
-        pMaxDate => DATE '9999-12-31',  -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
-        pParallelDegree => 16,
-        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
-        pMaxFileSize => 104857600,  -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
-        pRegisterExport => TRUE,  -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
-        pProcessName => 'MARS-835',                 -- Process identifier for tracking
-        pJobClass => 'high'  -- Oracle Scheduler job class for resource management
-    );
-    
-    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
-END;
-/
-
--- Export historical data to HIST bucket (Parquet)
--- NEW v2.4.0: Per-column date format handling with template table
-BEGIN
-    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...');
     DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
     
     CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -128,7 +34,8 @@ BEGIN
         pKeyColumnName => 'A_ETL_LOAD_SET_FK',
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
-        pMaxDate => &cutoff_date,
+        pMinDate => DATE '1900-01-01',  -- Include all historical data
+        pMaxDate => DATE '9999-12-31',  -- Include all future dates
         pParallelDegree => 16,
         pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
         pJobClass => 'high'  -- Oracle Scheduler job class for resource management
@@ -139,110 +46,18 @@ END;
 /
 
 PROMPT ========================================================================
-PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST
+PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - HIST Only
 PROMPT ========================================================================
-PROMPT Last 6 months to DATA bucket (CSV format)
+PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
-PROMPT Older data to HIST bucket (Parquet with partitioning)
 PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
 PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS,
 PROMPT                   ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS
 PROMPT ========================================================================
 
--- PRE-EXPORT CHECK: List existing files and count records
+-- Export ALL data to HIST bucket (Parquet)
-DECLARE
+-- NEW v2.12.0: Per-column date format handling with template table, full data range
-    vFileCount NUMBER := 0;
-    vRecordCount NUMBER := 0;
-    vLocationUri VARCHAR2(1000);
 BEGIN
-    -- Get bucket URI for DATA bucket
+    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (ALL data)...');
-    vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/';
-    
-    -- Count existing files
-    SELECT COUNT(*)
-    INTO vFileCount
-    FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-        credential_name => 'OCI$RESOURCE_PRINCIPAL',
-        location_uri => vLocationUri
-    ))
-    WHERE object_name NOT LIKE '%/';  -- Exclude directories
-    
-    IF vFileCount > 0 THEN
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
-        DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
-        DBMS_OUTPUT.PUT_LINE('');
-        
-        -- List existing files
-        DBMS_OUTPUT.PUT_LINE('Existing files:');
-        FOR rec IN (
-            SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
-            FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-                credential_name => 'OCI$RESOURCE_PRINCIPAL',
-                location_uri => vLocationUri
-            ))
-            WHERE object_name NOT LIKE '%/'
-            ORDER BY object_name
-        ) LOOP
-            DBMS_OUTPUT.PUT_LINE('  - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
-        END LOOP;
-        
-        -- Count records in external table
-        BEGIN
-            EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
-            DBMS_OUTPUT.PUT_LINE('');
-            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
-            DBMS_OUTPUT.PUT_LINE('>>>');
-            DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
-            DBMS_OUTPUT.PUT_LINE('>>>');
-            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
-        EXCEPTION
-            WHEN OTHERS THEN
-                DBMS_OUTPUT.PUT_LINE('');
-                DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
-                DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
-        END;
-        
-        DBMS_OUTPUT.PUT_LINE('===============================================================================');
-        DBMS_OUTPUT.PUT_LINE('');
-    ELSE
-        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
-        DBMS_OUTPUT.PUT_LINE('');
-    END IF;
-END;
-/
-
--- Export recent data to DATA bucket (CSV)
--- NEW v2.4.0: Per-column date format handling with template table for column order
-BEGIN
-    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...');
-    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
-    
-    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
-        pSchemaName => 'OU_CSDB',
-        pTableName => 'LEGACY_DEBT_DAILY',
-        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
-        pBucketArea => 'DATA',
-        pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
-        pMinDate => &cutoff_date,
-        pMaxDate => DATE '9999-12-31',  -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
-        pParallelDegree => 16,
-        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
-        pMaxFileSize => 104857600,  -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
-        pRegisterExport => TRUE,  -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
-        pProcessName => 'MARS-835',                 -- Process identifier for tracking
-        pJobClass => 'high'  -- Oracle Scheduler job class for resource management
-    );
-    
-    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
-END;
-/
-
--- Export historical data to HIST bucket (Parquet)
--- NEW v2.4.0: Per-column date format handling with template table
-BEGIN
-    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...');
     DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
     
     CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
@@ -251,7 +66,8 @@ BEGIN
         pKeyColumnName => 'A_ETL_LOAD_SET_FK',
         pBucketArea => 'ARCHIVE',
         pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
-        pMaxDate => &cutoff_date,
+        pMinDate => DATE '1900-01-01',  -- Include all historical data
+        pMaxDate => DATE '9999-12-31',  -- Include all future dates
         pParallelDegree => 16,
         pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
         pJobClass => 'high'  -- Oracle Scheduler job class for resource management
@@ -264,8 +80,8 @@ END;
 PROMPT ========================================================================
 PROMPT Group 1 Export Completed
 PROMPT ========================================================================
-PROMPT - LEGACY_DEBT: DATA + HIST exported
+PROMPT - LEGACY_DEBT: HIST exported (ALL data)
-PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported
+PROMPT - LEGACY_DEBT_DAILY: HIST exported (ALL data)
 PROMPT ========================================================================
 
 --=============================================================================================================================

MARS_Packages/REL01_ADDITIONS/MARS-835/03_MARS_835_verify_exports.sql

@@ -1,10 +1,11 @@
 -- =====================================================================================
 -- Script: 03_MARS_835_verify_exports.sql
--- Purpose: Verify exported files exist in DATA and HIST buckets after export
+-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy)
 -- Author: Grzegorz Michalski
 -- Created: 2025-12-17
+-- Updated: 2026-02-24 (Changed to HIST-only verification)
 -- MARS Issue: MARS-835
--- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/
+-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/
 -- =====================================================================================
 
 SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,17 +14,14 @@ SET VERIFY OFF;
 SET LINESIZE 200;
 
 PROMPT =====================================================================================
-PROMPT MARS-835 Verification: Listing exported files in DATA and HIST buckets
+PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy)
 PROMPT =====================================================================================
 
 DECLARE
-    vDataBucketUri VARCHAR2(500);
     vHistBucketUri VARCHAR2(500);
     vCredentialName VARCHAR2(100);
     vFileCount NUMBER := 0;
-    vTotalDataFiles NUMBER := 0;
     vTotalHistFiles NUMBER := 0;
-    vTotalDataSize NUMBER := 0;
     vTotalHistSize NUMBER := 0;
     
     TYPE t_folder_info IS RECORD (
@@ -33,25 +31,18 @@ DECLARE
     );
     TYPE t_folder_list IS TABLE OF t_folder_info;
     
-    vDataFolders t_folder_list;
     vHistFolders t_folder_list;
 BEGIN
-    -- Get bucket URIs and credential from FILE_MANAGER
+    -- Get bucket URI and credential from FILE_MANAGER
-    vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
     vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
     vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
     
     DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
-    DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
     DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
     DBMS_OUTPUT.PUT_LINE('');
     
-    -- Initialize folder lists
+    -- Initialize folder list (all tables in HIST)
-    vDataFolders := t_folder_list(
+    -- Initialize folder list (all 6 tables in HIST)
-        t_folder_info('ODS/CSDB/CSDB_DEBT/', 'DEBT', 'CSV'),
-        t_folder_info('ODS/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'CSV')
-    );
-    
     vHistFolders := t_folder_list(
         t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'),
         t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'),
@@ -62,49 +53,7 @@ BEGIN
     );
     
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket Exports (CSV format - last 6 months)');
+    DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)');
-    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    
-    -- Check DATA bucket exports
-    FOR i IN 1..vDataFolders.COUNT LOOP
-        vFileCount := 0;
-        
-        DBMS_OUTPUT.PUT_LINE('');
-        DBMS_OUTPUT.PUT_LINE('Table: ' || vDataFolders(i).table_name || ' (' || vDataFolders(i).expected_format || ')');
-        DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i).folder_name);
-        DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
-        
-        BEGIN
-            FOR rec IN (
-                SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
-                FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-                    credential_name => vCredentialName,
-                    location_uri => vDataBucketUri || vDataFolders(i).folder_name
-                ))
-                WHERE object_name LIKE '%.csv'
-                ORDER BY created DESC
-            ) LOOP
-                vFileCount := vFileCount + 1;
-                vTotalDataFiles := vTotalDataFiles + 1;
-                vTotalDataSize := vTotalDataSize + rec.bytes;
-                DBMS_OUTPUT.PUT_LINE('  [' || vFileCount || '] ' || rec.object_name || 
-                                   ' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
-            END LOOP;
-            
-            IF vFileCount = 0 THEN
-                DBMS_OUTPUT.PUT_LINE('  [ERROR] No CSV files found - Export may have failed!');
-            ELSE
-                DBMS_OUTPUT.PUT_LINE('  [SUCCESS] Found ' || vFileCount || ' CSV file(s)');
-            END IF;
-        EXCEPTION
-            WHEN OTHERS THEN
-                DBMS_OUTPUT.PUT_LINE('  [ERROR] Cannot access folder - ' || SQLERRM);
-        END;
-    END LOOP;
-    
-    DBMS_OUTPUT.PUT_LINE('');
-    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning)');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     
     -- Check HIST bucket exports
@@ -155,24 +104,19 @@ BEGIN
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     DBMS_OUTPUT.PUT_LINE('Export Verification Summary');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('DATA Bucket (CSV):');
+    DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):');
-    DBMS_OUTPUT.PUT_LINE('  - Total files: ' || vTotalDataFiles);
-    DBMS_OUTPUT.PUT_LINE('  - Total size: ' || ROUND(vTotalDataSize/1024/1024/1024, 2) || ' GB');
-    DBMS_OUTPUT.PUT_LINE('  - Expected tables: 2 (DEBT, DEBT_DAILY - last 6 months)');
-    DBMS_OUTPUT.PUT_LINE('');
-    DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet):');
     DBMS_OUTPUT.PUT_LINE('  - Total files: ' || vTotalHistFiles || '+');
     DBMS_OUTPUT.PUT_LINE('  - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)');
-    DBMS_OUTPUT.PUT_LINE('  - Expected tables: 6 (all CSDB tables with historical data)');
+    DBMS_OUTPUT.PUT_LINE('  - Expected tables: 6 (all CSDB tables exported to HIST)');
     DBMS_OUTPUT.PUT_LINE('');
     
-    IF vTotalDataFiles >= 2 AND vTotalHistFiles >= 6 THEN
+    IF vTotalHistFiles >= 6 THEN
         DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL');
-        DBMS_OUTPUT.PUT_LINE('  Files found in both DATA and HIST buckets');
+        DBMS_OUTPUT.PUT_LINE('  Files found in HIST bucket for all tables');
         DBMS_OUTPUT.PUT_LINE('  Proceed to record count verification (Step 4)');
-    ELSIF vTotalDataFiles = 0 AND vTotalHistFiles = 0 THEN
+    ELSIF vTotalHistFiles = 0 THEN
         DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED');
-        DBMS_OUTPUT.PUT_LINE('  No files found in either bucket');
+        DBMS_OUTPUT.PUT_LINE('  No files found in HIST bucket');
         DBMS_OUTPUT.PUT_LINE('  Review export logs for errors');
     ELSE
         DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected');

MARS_Packages/REL01_ADDITIONS/MARS-835/04_MARS_835_verify_record_counts.sql

@@ -1,10 +1,11 @@
 -- =====================================================================================
 -- Script: 04_MARS_835_verify_record_counts.sql
--- Purpose: Verify record counts match between source tables and exported data
+-- Purpose: Verify record counts match between source tables and exported data (HIST-only)
 -- Author: Grzegorz Michalski
 -- Created: 2025-12-17
+-- Updated: 2026-02-24 (Changed to HIST-only verification)
 -- MARS Issue: MARS-835
--- Verification: Compare OU_CSDB source tables with ODS external tables
+-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only)
 -- =====================================================================================
 
 SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -13,28 +14,23 @@ SET VERIFY OFF;
 SET LINESIZE 200;
 
 PROMPT =====================================================================================
-PROMPT MARS-835 Record Count Verification
+PROMPT MARS-835 Record Count Verification (HIST-only strategy)
 PROMPT =====================================================================================
-PROMPT Comparing source table counts with exported external table counts
+PROMPT Comparing source table counts with HIST external table counts
 PROMPT =====================================================================================
 
 DECLARE
     TYPE t_table_info IS RECORD (
         source_schema VARCHAR2(50),
         source_table VARCHAR2(100),
-        data_external_table VARCHAR2(100),
+        hist_external_table VARCHAR2(100)
-        hist_external_table VARCHAR2(100),
-        has_data_export BOOLEAN,
-        has_hist_export BOOLEAN
     );
     TYPE t_table_list IS TABLE OF t_table_info;
     
     vTables t_table_list;
     vSourceCount NUMBER;
-    vDataCount NUMBER;
     vHistCount NUMBER;
     vTotalSourceCount NUMBER := 0;
-    vTotalDataCount NUMBER := 0;
     vTotalHistCount NUMBER := 0;
     vMismatchCount NUMBER := 0;
     vSql VARCHAR2(4000);
@@ -42,18 +38,18 @@ BEGIN
     DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
     DBMS_OUTPUT.PUT_LINE('');
     
-    -- Initialize table list with export configuration
+    -- Initialize table list (all tables HIST-only)
     vTables := t_table_list(
-        t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ODS', 'ODS.CSDB_DEBT_ARCHIVE', TRUE, TRUE),
+        t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'),
-        t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ODS', 'ODS.CSDB_DEBT_DAILY_ARCHIVE', TRUE, TRUE),
+        t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'),
-        t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', NULL, 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE', FALSE, TRUE),
+        t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'),
-        t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', NULL, 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE', FALSE, TRUE),
+        t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'),
-        t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', NULL, 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE', FALSE, TRUE),
+        t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'),
-        t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE)
+        t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE')
     );
     
     DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
-    DBMS_OUTPUT.PUT_LINE('Table Name                Source Count      DATA Count      HIST Count   Status');
+    DBMS_OUTPUT.PUT_LINE('Table Name                Source Count      HIST Count   Status');
     DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
     
     FOR i IN 1..vTables.COUNT LOOP
@@ -70,31 +66,6 @@ BEGIN
                 CONTINUE;
         END;
         
-        -- Get DATA external table count (if applicable)
-        IF vTables(i).has_data_export THEN
-            vSql := 'SELECT COUNT(*) FROM ' || vTables(i).data_external_table;
-            BEGIN
-                EXECUTE IMMEDIATE vSql INTO vDataCount;
-                vTotalDataCount := vTotalDataCount + vDataCount;
-            EXCEPTION
-                WHEN OTHERS THEN
-                    -- If source table is empty (0 records), no files were exported
-                    -- External table returns error, treat as 0
-                    -- Acceptable error codes:
-                    --   ORA-29913: error in executing ODCIEXTTABLEOPEN callout
-                    --   ORA-29400: data cartridge error
-                    --   KUP-13023: nothing matched wildcard query (no files in bucket)
-                    -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
-                    IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
-                        vDataCount := 0;
-                    ELSE
-                        vDataCount := -1;
-                    END IF;
-            END;
-        ELSE
-            vDataCount := NULL;
-        END IF;
-        
         -- Get HIST external table count
         vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table;
         BEGIN
@@ -119,18 +90,8 @@ BEGIN
         -- Display results
         DECLARE
             vStatus VARCHAR2(20);
-            vDataDisplay VARCHAR2(17);
             vHistDisplay VARCHAR2(17);
         BEGIN
-            -- Format DATA count display
-            IF vDataCount IS NULL THEN
-                vDataDisplay := 'N/A';
-            ELSIF vDataCount = -1 THEN
-                vDataDisplay := 'ERROR';
-            ELSE
-                vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999');
-            END IF;
-            
             -- Format HIST count display
             IF vHistCount = -1 THEN
                 vHistDisplay := 'ERROR';
@@ -138,35 +99,20 @@ BEGIN
                 vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
             END IF;
             
-            -- Determine status
+            -- Determine status (HIST only: check HIST = SOURCE)
-            IF vTables(i).has_data_export THEN
+            IF vHistCount = vSourceCount THEN
-                -- Split export: check DATA + HIST = SOURCE
+                vStatus := 'PASS';
-                IF (vDataCount + vHistCount) = vSourceCount THEN
+            ELSIF vHistCount = -1 THEN
-                    vStatus := 'PASS';
+                vStatus := 'ERROR';
-                ELSIF vDataCount = -1 OR vHistCount = -1 THEN
+                vMismatchCount := vMismatchCount + 1;
-                    vStatus := 'ERROR';
-                    vMismatchCount := vMismatchCount + 1;
-                ELSE
-                    vStatus := 'MISMATCH';
-                    vMismatchCount := vMismatchCount + 1;
-                END IF;
             ELSE
-                -- HIST only: check HIST = SOURCE
+                vStatus := 'MISMATCH';
-                IF vHistCount = vSourceCount THEN
+                vMismatchCount := vMismatchCount + 1;
-                    vStatus := 'PASS';
-                ELSIF vHistCount = -1 THEN
-                    vStatus := 'ERROR';
-                    vMismatchCount := vMismatchCount + 1;
-                ELSE
-                    vStatus := 'MISMATCH';
-                    vMismatchCount := vMismatchCount + 1;
-                END IF;
             END IF;
             
             DBMS_OUTPUT.PUT_LINE(
                 RPAD(vTables(i).source_table, 24) ||
                 LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
-                LPAD(vDataDisplay, 15) ||
                 LPAD(vHistDisplay, 15) || '   ' ||
                 vStatus
             );
@@ -177,18 +123,16 @@ BEGIN
     DBMS_OUTPUT.PUT_LINE(
         RPAD('TOTALS', 24) ||
         LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
-        LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) ||
         LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
     );
     DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
     DBMS_OUTPUT.PUT_LINE('');
     
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
+    DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
-    DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)');
+    DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)');
-    DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)');
     DBMS_OUTPUT.PUT_LINE('');
     
     IF vMismatchCount = 0 THEN
@@ -209,7 +153,6 @@ BEGIN
     DBMS_OUTPUT.PUT_LINE('  MISMATCH   - Record counts differ (may be pre-existing files or export issue)');
     DBMS_OUTPUT.PUT_LINE('               Check pre-check results to identify pre-existing files');
     DBMS_OUTPUT.PUT_LINE('  ERROR      - Cannot access table (may not exist yet)');
-    DBMS_OUTPUT.PUT_LINE('  N/A        - Not applicable (table not exported to DATA)');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     
 EXCEPTION

MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql

@@ -1,68 +1,34 @@
 --=============================================================================================================================
 -- MARS-835 ROLLBACK: Delete Group 1 Exported Files (DEBT, DEBT_DAILY)
 --=============================================================================================================================
--- Purpose: Delete exported CSV and Parquet files from DATA and HIST buckets
+-- Purpose: Delete exported Parquet files from HIST bucket (ARCHIVE only)
 -- WARNING: This will permanently delete exported data files!
 -- Author: Grzegorz Michalski
 -- Date: 2025-12-17
+-- Updated: 2026-02-24 (Changed to HIST-only rollback, no DATA bucket)
 -- Related: MARS-835 - CSDB Data Export Rollback
 --=============================================================================================================================
 
 SET SERVEROUTPUT ON SIZE UNLIMITED
 
 PROMPT ========================================================================
-PROMPT ROLLBACK: Deleting DEBT exported files
+PROMPT ROLLBACK: Deleting DEBT exported files from HIST
 PROMPT ========================================================================
 PROMPT WARNING: This will delete files from:
-PROMPT   - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT/
 PROMPT   - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/
 PROMPT ========================================================================
 
 DECLARE
-    vDataBucketUri VARCHAR2(500);
     vHistBucketUri VARCHAR2(500);
     vCredentialName VARCHAR2(100);
     vFileCount NUMBER := 0;
 BEGIN
-    -- Get bucket URIs and credential
+    -- Get bucket URI and credential
-    vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
     vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
     vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
     
-    DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...');
-    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-    
-    -- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS
-    FOR rec IN (
-        SELECT object_name
-        FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-            credential_name => vCredentialName,
-            location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
-        ))
-        WHERE object_name LIKE 'LEGACY_DEBT%'
-    ) LOOP
-        BEGIN
-            DBMS_CLOUD.DELETE_OBJECT(
-                credential_name => vCredentialName,
-                object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
-            );
-            DBMS_OUTPUT.PUT_LINE('  Deleted: ' || rec.object_name);
-            vFileCount := vFileCount + 1;
-        EXCEPTION
-            WHEN OTHERS THEN
-                IF SQLCODE = -20404 THEN
-                    DBMS_OUTPUT.PUT_LINE('  Skipped (not found): ' || rec.object_name);
-                ELSE
-                    RAISE;
-                END IF;
-        END;
-    END LOOP;
-    
-    DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
-    
     DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...');
-    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
+    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS');
-    vFileCount := 0;
     
     -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
     FOR rec IN (
@@ -99,58 +65,23 @@ END;
 /
 
 PROMPT ========================================================================
-PROMPT ROLLBACK: Deleting DEBT_DAILY exported files
+PROMPT ROLLBACK: Deleting DEBT_DAILY exported files from HIST
 PROMPT ========================================================================
 PROMPT WARNING: This will delete files from:
-PROMPT   - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/
 PROMPT   - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/
 PROMPT ========================================================================
 
 DECLARE
-    vDataBucketUri VARCHAR2(500);
     vHistBucketUri VARCHAR2(500);
     vCredentialName VARCHAR2(100);
     vFileCount NUMBER := 0;
 BEGIN
-    -- Get bucket URIs and credential
+    -- Get bucket URI and credential
-    vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
     vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
     vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
     
-    DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...');
-    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
-    
-    -- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS
-    FOR rec IN (
-        SELECT object_name
-        FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-            credential_name => vCredentialName,
-            location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
-        ))
-        WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
-    ) LOOP
-        BEGIN
-            DBMS_CLOUD.DELETE_OBJECT(
-                credential_name => vCredentialName,
-                object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
-            );
-            DBMS_OUTPUT.PUT_LINE('  Deleted: ' || rec.object_name);
-            vFileCount := vFileCount + 1;
-        EXCEPTION
-            WHEN OTHERS THEN
-                IF SQLCODE = -20404 THEN
-                    DBMS_OUTPUT.PUT_LINE('  Skipped (not found): ' || rec.object_name);
-                ELSE
-                    RAISE;
-                END IF;
-        END;
-    END LOOP;
-    
-    DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
-    
     DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
-    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
+    DBMS_OUTPUT.PUT_LINE('  Using DBMS_CLOUD.LIST_OBJECTS');
-    vFileCount := 0;
     
     -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
     FOR rec IN (

MARS_Packages/REL01_ADDITIONS/MARS-835/99_MARS_835_verify_rollback.sql

@@ -1,10 +1,11 @@
 -- =====================================================================================
 -- Script: 99_MARS_835_verify_rollback.sql
--- Purpose: Verify all exported files have been deleted from DATA and HIST buckets
+-- Purpose: Verify all exported files have been deleted from HIST bucket (HIST-only strategy)
 -- Author: Grzegorz Michalski
 -- Created: 2025-12-17
+-- Updated: 2026-02-24 (Changed to HIST-only verification)
 -- MARS Issue: MARS-835
--- Verification: Confirm complete rollback (no CSDB files remaining)
+-- Verification: Confirm complete rollback (no CSDB files remaining in HIST)
 -- =====================================================================================
 
 SET SERVEROUTPUT ON SIZE UNLIMITED;
@@ -19,33 +20,23 @@ PROMPT Checking that all CSDB export files have been deleted
 PROMPT =====================================================================================
 
 DECLARE
-    vDataBucketUri VARCHAR2(500);
     vHistBucketUri VARCHAR2(500);
     vCredentialName VARCHAR2(100);
-    vDataFileCount NUMBER := 0;
     vHistFileCount NUMBER := 0;
-    vTotalFiles NUMBER := 0;
     
     TYPE t_folder_list IS TABLE OF VARCHAR2(200);
-    vDataFolders t_folder_list;
     vHistFolders t_folder_list;
 BEGIN
-    -- Get bucket URIs
+    -- Get bucket URI
-    vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
     vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
     vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
     
     DBMS_OUTPUT.PUT_LINE('ROLLBACK VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
-    DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
     DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
     DBMS_OUTPUT.PUT_LINE('');
     
-    -- Initialize folder lists
+    -- Initialize folder list (all 6 tables in HIST)
-    vDataFolders := t_folder_list(
+    -- Initialize folder list (all 6 tables in HIST)
-        'ODS/CSDB/CSDB_DEBT/',
-        'ODS/CSDB/CSDB_DEBT_DAILY/'
-    );
-    
     vHistFolders := t_folder_list(
         'ARCHIVE/CSDB/CSDB_DEBT/',
         'ARCHIVE/CSDB/CSDB_DEBT_DAILY/',
@@ -55,47 +46,6 @@ BEGIN
         'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/'
     );
     
-    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (should be empty)');
-    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    
-    -- Check DATA bucket
-    FOR i IN 1..vDataFolders.COUNT LOOP
-        DECLARE
-            vCount NUMBER := 0;
-        BEGIN
-            DBMS_OUTPUT.PUT_LINE('');
-            DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i));
-            
-            FOR rec IN (
-                SELECT object_name
-                FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
-                    credential_name => vCredentialName,
-                    location_uri => vDataBucketUri || vDataFolders(i)
-                ))
-                WHERE object_name LIKE '%.csv'
-            ) LOOP
-                vCount := vCount + 1;
-                vDataFileCount := vDataFileCount + 1;
-                DBMS_OUTPUT.PUT_LINE('  [FOUND] ' || rec.object_name);
-            END LOOP;
-            
-            IF vCount = 0 THEN
-                DBMS_OUTPUT.PUT_LINE('  [OK] No CSV files found');
-            ELSE
-                DBMS_OUTPUT.PUT_LINE('  [INFO] Found ' || vCount || ' file(s) - may be pre-existing files from before installation');
-            END IF;
-        EXCEPTION
-            WHEN OTHERS THEN
-                IF SQLCODE = -20404 THEN
-                    DBMS_OUTPUT.PUT_LINE('  [OK] Folder does not exist or is empty');
-                ELSE
-                    DBMS_OUTPUT.PUT_LINE('  [ERROR] ' || SQLERRM);
-                END IF;
-        END;
-    END LOOP;
-    
-    DBMS_OUTPUT.PUT_LINE('');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (should be empty)');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
@@ -139,24 +89,21 @@ BEGIN
         END;
     END LOOP;
     
-    vTotalFiles := vDataFileCount + vHistFileCount;
-    
     DBMS_OUTPUT.PUT_LINE('');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
     DBMS_OUTPUT.PUT_LINE('Rollback Verification Summary');
     DBMS_OUTPUT.PUT_LINE('=====================================================================================');
-    DBMS_OUTPUT.PUT_LINE('DATA bucket files remaining: ' || vDataFileCount);
     DBMS_OUTPUT.PUT_LINE('HIST bucket files remaining: ' || vHistFileCount || '+');
-    DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles || '+');
+    DBMS_OUTPUT.PUT_LINE('');
     DBMS_OUTPUT.PUT_LINE('');
     
-    IF vTotalFiles = 0 THEN
+    IF vHistFileCount = 0 THEN
         DBMS_OUTPUT.PUT_LINE('[PASSED] ROLLBACK VERIFICATION PASSED');
         DBMS_OUTPUT.PUT_LINE('  All CSDB export files have been deleted or were not created');
-        DBMS_OUTPUT.PUT_LINE('  Buckets are clean and ready for re-export if needed');
+        DBMS_OUTPUT.PUT_LINE('  HIST bucket is clean and ready for re-export if needed');
     ELSE
         DBMS_OUTPUT.PUT_LINE('[INFO] ROLLBACK VERIFICATION COMPLETED');
-        DBMS_OUTPUT.PUT_LINE('  Found ' || vTotalFiles || '+ file(s) remaining in buckets');
+        DBMS_OUTPUT.PUT_LINE('  Found ' || vHistFileCount || '+ file(s) remaining in HIST bucket');
         DBMS_OUTPUT.PUT_LINE('  NOTE: These may be pre-existing files from before installation.');
         DBMS_OUTPUT.PUT_LINE('  Rollback only deletes files created during this export operation.');
         DBMS_OUTPUT.PUT_LINE('  If needed, manually verify and clean up remaining files.');