diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb index 7a3c90a..2c780f6 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb @@ -785,15 +785,14 @@ AS -- Register exported file to A_SOURCE_FILE_RECEIVED if requested IF pRegisterExport THEN DECLARE - vChecksum VARCHAR2(128); - vCreated TIMESTAMP WITH TIME ZONE; - vBytes NUMBER; vActualFileName VARCHAR2(1000); -- Actual filename with Oracle suffix vSanitizedFileName VARCHAR2(1000); vFileName VARCHAR2(1000); vRetryCount NUMBER := 0; vMaxRetries NUMBER := 1; -- One retry after initial attempt vRetryDelay NUMBER := 2; -- 2 seconds delay + vFilesFound NUMBER := 0; + vTotalBytes NUMBER := 0; BEGIN -- Extract filename from URI (after last '/') vFileName := SUBSTR(vUri, INSTR(vUri, '/', -1) + 1); @@ -805,24 +804,70 @@ AS -- Example: tablename.csv becomes tablename_1_20260211T102621591769Z.csv vSanitizedFileName := REGEXP_REPLACE(vSanitizedFileName, '\.csv$', '', 1, 0, 'i'); - -- Try to get file metadata with retry logic + -- Try to get ALL exported files with retry logic + -- Oracle DBMS_CLOUD.EXPORT_DATA can create MULTIPLE files due to: + -- 1. maxfilesize parameter (splits files larger than limit) + -- 2. Automatic parallel processing (especially on large production instances) + -- We must register ALL files, not just the first one <> LOOP BEGIN - SELECT object_name, checksum, created, bytes - INTO vActualFileName, vChecksum, vCreated, vBytes - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => pCredentialName, - location_uri => vBucketUri - )) - WHERE object_name LIKE CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || vSanitizedFileName || '%' - ORDER BY created DESC, bytes DESC - FETCH FIRST 1 ROW ONLY; + -- Register ALL files matching the pattern (cursor loop) + FOR rec IN ( + SELECT object_name, checksum, created, bytes + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => pCredentialName, + location_uri => vBucketUri + )) + WHERE object_name LIKE CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || vSanitizedFileName || '%' + ORDER BY created DESC, bytes DESC + ) LOOP + -- Extract filename only from full path (remove bucket folder prefix) + vActualFileName := SUBSTR(rec.object_name, INSTR(rec.object_name, '/', -1) + 1); + + -- Create A_SOURCE_FILE_RECEIVED record for EACH exported file + vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL; + INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED ( + A_SOURCE_FILE_RECEIVED_KEY, + A_SOURCE_FILE_CONFIG_KEY, + SOURCE_FILE_NAME, + CHECKSUM, + CREATED, + BYTES, + RECEPTION_DATE, + PROCESSING_STATUS, + PARTITION_YEAR, + PARTITION_MONTH, + ARCH_PATH, + PROCESS_NAME + ) VALUES ( + vSourceFileReceivedKey, + NVL(vConfigKey, -1), -- Use config key if found, otherwise -1 + vActualFileName, -- Use actual filename with Oracle suffix + rec.checksum, + rec.created, + rec.bytes, + SYSDATE, + 'INGESTED', + NULL, -- PARTITION_YEAR not used for single-file exports + NULL, -- PARTITION_MONTH not used for single-file exports + NULL, -- ARCH_PATH not used for single-file exports + pProcessName -- Process name from parameter + ); + + vFilesFound := vFilesFound + 1; + vTotalBytes := vTotalBytes + rec.bytes; + + ENV_MANAGER.LOG_PROCESS_EVENT('Registered file ' || vFilesFound || ': FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || rec.bytes || ' bytes', 'INFO', vParameters); + END LOOP; - -- Extract filename only from full path (remove bucket folder prefix) - vActualFileName := SUBSTR(vActualFileName, INSTR(vActualFileName, '/', -1) + 1); + -- Check if any files were found + IF vFilesFound = 0 THEN + RAISE NO_DATA_FOUND; + END IF; -- Success - exit retry loop + ENV_MANAGER.LOG_PROCESS_EVENT('Total registered: ' || vFilesFound || ' file(s), Total size: ' || vTotalBytes || ' bytes (' || ROUND(vTotalBytes/1048576, 2) || ' MB)', 'INFO', vParameters); EXIT metadata_retry_loop; EXCEPTION @@ -831,7 +876,7 @@ AS IF vRetryCount <= vMaxRetries THEN -- Log retry attempt - ENV_MANAGER.LOG_PROCESS_EVENT('File not found in bucket (attempt ' || vRetryCount || '/' || (vMaxRetries + 1) || '), retrying after ' || vRetryDelay || ' seconds: ' || vFileName, 'DEBUG', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('File(s) not found in bucket (attempt ' || vRetryCount || '/' || (vMaxRetries + 1) || '), retrying after ' || vRetryDelay || ' seconds: ' || vFileName, 'DEBUG', vParameters); -- Wait before retry using DBMS_SESSION.SLEEP (alternative to DBMS_LOCK) DBMS_SESSION.SLEEP(vRetryDelay); @@ -841,38 +886,6 @@ AS END IF; END; END LOOP metadata_retry_loop; - - -- Create A_SOURCE_FILE_RECEIVED record for this export with metadata - vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL; - INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED ( - A_SOURCE_FILE_RECEIVED_KEY, - A_SOURCE_FILE_CONFIG_KEY, - SOURCE_FILE_NAME, - CHECKSUM, - CREATED, - BYTES, - RECEPTION_DATE, - PROCESSING_STATUS, - PARTITION_YEAR, - PARTITION_MONTH, - ARCH_PATH, - PROCESS_NAME - ) VALUES ( - vSourceFileReceivedKey, - NVL(vConfigKey, -1), -- Use config key if found, otherwise -1 - vActualFileName, -- Use actual filename with Oracle suffix - vChecksum, - vCreated, - vBytes, - SYSDATE, - 'INGESTED', - NULL, -- PARTITION_YEAR not used for single-file exports - NULL, -- PARTITION_MONTH not used for single-file exports - NULL, -- ARCH_PATH not used for single-file exports - pProcessName -- Process name from parameter - ); - - ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || vBytes || ' bytes', 'INFO', vParameters); EXCEPTION WHEN NO_DATA_FOUND THEN -- File not found after retries - log warning and continue without metadata diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg index 6b6a4b8..dc7d4a4 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg @@ -9,17 +9,17 @@ AS **/ -- Package Version Information - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.9.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-13 14:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.10.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-13 16:30:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (last 3-5 changes) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + 'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) || 'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) || 'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10) || 'v2.8.0 (2026-02-12): MAJOR REFACTOR - EXPORT_TABLE_DATA now exports to single CSV file instead of partitioning by key values. Added pFileName parameter.' || CHR(10) || - 'v2.7.5 (2026-02-11): Added pRegisterExport parameter to EXPORT_TABLE_DATA procedure. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10) || - 'v2.7.4 (2026-02-11): ACTUAL FILENAME STORAGE - Store real filename with Oracle suffix in SOURCE_FILE_NAME instead of theoretical filename.' || CHR(10); + 'v2.7.5 (2026-02-11): Added pRegisterExport parameter to EXPORT_TABLE_DATA procedure. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10); cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); vgMsgTmp VARCHAR2(32000);