Update DATA_EXPORTER package to version 2.7.5, adding pRegisterExport parameter to EXPORT_TABLE_DATA procedure for file registration in A_SOURCE_FILE_RECEIVED.

MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb

@@ -603,6 +603,7 @@ AS
       pBucketArea        IN VARCHAR2,
       pFolderName        IN VARCHAR2,
       pTemplateTableName IN VARCHAR2 default NULL,
+      pRegisterExport    IN BOOLEAN  default FALSE,
       pCredentialName    IN VARCHAR2 default ENV_MANAGER.gvCredentialName
    )
    IS
@@ -622,6 +623,14 @@ AS
       vBucketUri       VARCHAR2(4000);
       vProcessedColumnList VARCHAR2(32767);
       vCurrentCol      VARCHAR2(128);
+      
+      -- Variables for file registration (when pRegisterExport=TRUE)
+      vConfigKey       NUMBER;
+      vSourceKey       VARCHAR2(100);
+      vTableId         VARCHAR2(100);
+      vSlashPos1       NUMBER;
+      vSlashPos2       NUMBER;
+      vSourceFileReceivedKey NUMBER;
 
    BEGIN
       vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(    'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
@@ -630,6 +639,7 @@ AS
                                                                ,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
                                                                ,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
                                                        ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
+                                                           ,'pRegisterExport => '''||CASE WHEN pRegisterExport THEN 'TRUE' ELSE 'FALSE' END||''''
                                                            ,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
                                                             ));
       ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
@@ -710,6 +720,46 @@ AS
       ENV_MANAGER.LOG_PROCESS_EVENT('Template table: ' || NVL(pTemplateTableName, 'NULL - using global default for all dates'), 'INFO', vParameters);
 
       vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
+      
+      -- Lookup A_SOURCE_FILE_CONFIG_KEY based on pFolderName parsing if pRegisterExport is enabled
+      IF pRegisterExport THEN
+         -- Format: {BUCKET_AREA}/{SOURCE_KEY}/{TABLE_ID}
+         -- Example: 'ODS/CSDB/CSDB_DEBT_DAILY' -> SOURCE_KEY='CSDB', TABLE_ID='CSDB_DEBT_DAILY'
+         
+         -- Parse pFolderName to extract SOURCE_KEY and TABLE_ID
+         vSlashPos1 := INSTR(pFolderName, '/', 1, 1);  -- First '/' position
+         vSlashPos2 := INSTR(pFolderName, '/', 1, 2);  -- Second '/' position
+         
+         IF vSlashPos1 > 0 AND vSlashPos2 > 0 THEN
+            -- Extract segment 2 (SOURCE_KEY) and segment 3 (TABLE_ID)
+            vSourceKey := SUBSTR(pFolderName, vSlashPos1 + 1, vSlashPos2 - vSlashPos1 - 1);
+            vTableId := SUBSTR(pFolderName, vSlashPos2 + 1);
+            
+            -- Find configuration based on SOURCE_KEY and TABLE_ID
+            BEGIN
+               SELECT A_SOURCE_FILE_CONFIG_KEY
+               INTO vConfigKey
+               FROM CT_MRDS.A_SOURCE_FILE_CONFIG
+               WHERE A_SOURCE_KEY = vSourceKey
+                 AND TABLE_ID = vTableId
+                 AND SOURCE_FILE_TYPE = 'INPUT'
+                 AND ROWNUM = 1;
+                 
+               ENV_MANAGER.LOG_PROCESS_EVENT('Found config key: ' || vConfigKey || ' for SOURCE=' || vSourceKey || ', TABLE=' || vTableId, 'DEBUG', vParameters);
+            EXCEPTION
+               WHEN NO_DATA_FOUND THEN
+                  vConfigKey := -1;
+                  ENV_MANAGER.LOG_PROCESS_EVENT('No config found for SOURCE=' || vSourceKey || ', TABLE=' || vTableId || ' - using default (-1)', 'INFO', vParameters);
+            END;
+         ELSE
+            -- Cannot parse folder name - use default
+            vConfigKey := -1;
+            ENV_MANAGER.LOG_PROCESS_EVENT('Cannot parse pFolderName: ' || pFolderName || ' - using default (-1)', 'WARNING', vParameters);
+         END IF;
+         
+         ENV_MANAGER.LOG_PROCESS_EVENT('File registration enabled with config key: ' || vConfigKey, 'INFO', vParameters);
+      END IF;
+      
       -- Fetch unique key values from A_LOAD_HISTORY
       vSql := 'SELECT DISTINCT L.A_ETL_LOAD_SET_KEY' ||
                ' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
@@ -759,7 +809,136 @@ AS
               query           => vQuery,
               format          => json_object('type' VALUE 'CSV', 'header' VALUE true)
           );
+          
+          -- Register exported file to A_SOURCE_FILE_RECEIVED if requested
+          IF pRegisterExport THEN
+             DECLARE
+                vChecksum VARCHAR2(128);
+                vCreated TIMESTAMP WITH TIME ZONE;
+                vBytes NUMBER;
+                vActualFileName VARCHAR2(1000);  -- Actual filename with Oracle suffix
+                vSanitizedFileName VARCHAR2(1000);
+                vFileName VARCHAR2(1000);
+                vRetryCount NUMBER := 0;
+                vMaxRetries NUMBER := 1;  -- One retry after initial attempt
+                vRetryDelay NUMBER := 2;  -- 2 seconds delay
+             BEGIN
+                -- Extract filename from URI (after last '/')
+                vFileName := SUBSTR(vUri, INSTR(vUri, '/', -1) + 1);
+                
+                -- Sanitize filename first (PL/SQL function cannot be used directly in SQL)
+                vSanitizedFileName := sanitizeFilename(vFileName);
+                
+                -- Remove .csv extension for LIKE pattern matching (Oracle adds suffixes BEFORE .csv)
+                -- Example: keyvalue.csv becomes keyvalue_1_20260211T102621591769Z.csv
+                vSanitizedFileName := REGEXP_REPLACE(vSanitizedFileName, '\.csv$', '', 1, 0, 'i');
+                
+                -- Try to get file metadata with retry logic
+                <<metadata_retry_loop>>
+                LOOP
+                   BEGIN
+                      SELECT object_name, checksum, created, bytes
+                      INTO vActualFileName, vChecksum, vCreated, vBytes
+                      FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
+                         credential_name => pCredentialName,
+                         location_uri => vBucketUri
+                      ))
+                      WHERE object_name LIKE CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || vSanitizedFileName || '%'
+                      ORDER BY created DESC, bytes DESC
+                      FETCH FIRST 1 ROW ONLY;
+                      
+                      -- Extract filename only from full path (remove bucket folder prefix)
+                      vActualFileName := SUBSTR(vActualFileName, INSTR(vActualFileName, '/', -1) + 1);
+                      
+                      -- Success - exit retry loop
+                      EXIT metadata_retry_loop;
+                      
+                   EXCEPTION
+                      WHEN NO_DATA_FOUND THEN
+                         vRetryCount := vRetryCount + 1;
+                         
+                         IF vRetryCount <= vMaxRetries THEN
+                            -- Log retry attempt
+                            ENV_MANAGER.LOG_PROCESS_EVENT('File not found in bucket (attempt ' || vRetryCount || '/' || (vMaxRetries + 1) || '), retrying after ' || vRetryDelay || ' seconds: ' || vFileName, 'DEBUG', vParameters);
+                            
+                            -- Wait before retry using DBMS_SESSION.SLEEP (alternative to DBMS_LOCK)
+                            DBMS_SESSION.SLEEP(vRetryDelay);
+                         ELSE
+                            -- Max retries exceeded - re-raise exception
+                            RAISE;
+                         END IF;
+                   END;
+                END LOOP metadata_retry_loop;
+                
+                -- Create A_SOURCE_FILE_RECEIVED record for this export with metadata
+                vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
+                INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED (
+                   A_SOURCE_FILE_RECEIVED_KEY,
+                   A_SOURCE_FILE_CONFIG_KEY,
+                   SOURCE_FILE_NAME,
+                   CHECKSUM,
+                   CREATED,
+                   BYTES,
+                   RECEPTION_DATE,
+                   PROCESSING_STATUS,
+                   PARTITION_YEAR,
+                   PARTITION_MONTH,
+                   ARCH_FILE_NAME
+                ) VALUES (
+                   vSourceFileReceivedKey,
+                   NVL(vConfigKey, -1),  -- Use config key if found, otherwise -1
+                   vActualFileName,  -- Use actual filename with Oracle suffix
+                   vChecksum,
+                   vCreated,
+                   vBytes,
+                   SYSDATE,
+                   'INGESTED',
+                   NULL,  -- PARTITION_YEAR not used for single-file exports
+                   NULL,  -- PARTITION_MONTH not used for single-file exports
+                   NULL   -- ARCH_FILE_NAME not used for single-file exports
+                );
+
+                ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || vBytes || ' bytes', 'DEBUG', vParameters);
+             EXCEPTION
+                WHEN NO_DATA_FOUND THEN
+                   -- File not found after retries - log warning and continue without metadata
+                   ENV_MANAGER.LOG_PROCESS_EVENT('WARNING: File not found in bucket after ' || (vMaxRetries + 1) || ' attempts: ' || vFileName, 'WARNING', vParameters);
+                   
+                   -- Sanitize filename for fallback INSERT (function cannot be used in SQL)
+                   vSanitizedFileName := sanitizeFilename(vFileName);
+                   
+                   -- Insert without metadata using theoretical filename
+                   vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
+                   INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED (
+                      A_SOURCE_FILE_RECEIVED_KEY,
+                      A_SOURCE_FILE_CONFIG_KEY,
+                      SOURCE_FILE_NAME,
+                      RECEPTION_DATE,
+                      PROCESSING_STATUS,
+                      PARTITION_YEAR,
+                      PARTITION_MONTH,
+                      ARCH_FILE_NAME
+                   ) VALUES (
+                      vSourceFileReceivedKey,
+                      NVL(vConfigKey, -1),  -- Use config key if found, otherwise -1
+                      vSanitizedFileName,  -- Use pre-calculated sanitized filename
+                      SYSDATE,
+                      'INGESTED',
+                      NULL,  -- PARTITION_YEAR not used for single-file exports
+                      NULL,  -- PARTITION_MONTH not used for single-file exports
+                      NULL   -- ARCH_FILE_NAME not used for single-file exports
+                   );
+
+                   ENV_MANAGER.LOG_PROCESS_EVENT('Registered file without metadata: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vSanitizedFileName, 'DEBUG', vParameters);
+             END;
+          END IF;
       END LOOP;
+      
+      -- Log summary of file registration if enabled
+      IF pRegisterExport THEN
+         ENV_MANAGER.LOG_PROCESS_EVENT('Registered ' || vKeyValues.COUNT || ' exported files to A_SOURCE_FILE_RECEIVED with config key: ' || vConfigKey, 'INFO', vParameters);
+      END IF;
+      
       ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
    EXCEPTION 
       WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN

MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg

@@ -9,12 +9,13 @@ AS
    **/
 
    -- Package Version Information
-   PACKAGE_VERSION     CONSTANT VARCHAR2(10) := '2.7.4';
+   PACKAGE_VERSION     CONSTANT VARCHAR2(10) := '2.7.5';
-   PACKAGE_BUILD_DATE  CONSTANT VARCHAR2(20) := '2026-02-11 12:10:00';
+   PACKAGE_BUILD_DATE  CONSTANT VARCHAR2(20) := '2026-02-11 12:15:00';
    PACKAGE_AUTHOR      CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
 
    -- Version History (last 3-5 changes)
    VERSION_HISTORY     CONSTANT VARCHAR2(4000) := 
+       'v2.7.5 (2026-02-11): Added pRegisterExport parameter to EXPORT_TABLE_DATA procedure. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10) ||
        'v2.7.4 (2026-02-11): ACTUAL FILENAME STORAGE - Store real filename with Oracle suffix in SOURCE_FILE_NAME instead of theoretical filename.' || CHR(10) ||
        'v2.7.3 (2026-02-11): FIX LIKE pattern for DBMS_CLOUD.LIST_OBJECTS - Removed .csv extension from filename before pattern matching.' || CHR(10) ||
        'v2.7.2 (2026-02-11): FIX pRegisterExport in EXPORT_TABLE_DATA_TO_CSV_BY_DATE - Added missing pRegisterExport parameter to EXPORT_SINGLE_PARTITION call.' || CHR(10) ||
@@ -73,10 +74,13 @@ AS
       *           Exports data into CSV file on OCI infrustructure.
       *           pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
       *           Supports template table for column order and per-column date formatting.
+      *           When pRegisterExport=TRUE, successfully exported files are registered in:
+      *           - CT_MRDS.A_SOURCE_FILE_RECEIVED (tracks file location, size, checksum, and metadata)
       * @param    pTemplateTableName - Optional template table (SCHEMA.TABLE or TABLE) for:
       *                                - Column order control (template defines CSV structure)
       *                                - Per-column date formatting via FILE_MANAGER.GET_DATE_FORMAT
       *                                - NULL = use source table columns in natural order
+      * @param    pRegisterExport - When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED table
       * @example  
       *   begin
       *      DATA_EXPORTER.EXPORT_TABLE_DATA(
@@ -85,8 +89,8 @@ AS
       *          pKeyColumnName     => 'A_ETL_LOAD_SET_KEY_FK',
       *          pBucketArea        => 'DATA',
       *          pFolderName        => 'csv_exports',
-      *          pTemplateTableName => 'CT_ET_TEMPLATES.MY_TEMPLATE'  -- Optional
+      *          pTemplateTableName => 'CT_ET_TEMPLATES.MY_TEMPLATE',  -- Optional
-      *      );
+      *          pRegisterExport    => TRUE  -- Optional, default FALSE
       *      );
       *   end;
    **/
@@ -97,6 +101,7 @@ AS
      pBucketArea        IN VARCHAR2,
      pFolderName        IN VARCHAR2,
      pTemplateTableName IN VARCHAR2 default NULL,
+     pRegisterExport    IN BOOLEAN  default FALSE,
      pCredentialName    IN VARCHAR2 default ENV_MANAGER.gvCredentialName
    );