Update DATA_EXPORTER package to version 2.7.4, enhancing filename storage and metadata retrieval logic for exported files.
This commit is contained in:
Grzegorz Michalski
@@ -1055,7 +1055,6 @@ AS
|
||||
vConfigKey NUMBER := -1;
|
||||
vSlashPos1 NUMBER;
|
||||
vSlashPos2 NUMBER;
|
||||
vFileUri VARCHAR2(4000);
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
@@ -1262,6 +1261,9 @@ AS
|
||||
END;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Note: File registration handled by EXPORT_SINGLE_PARTITION when pRegisterExport=TRUE
|
||||
-- Each partition calls pRegisterExport logic independently during serial/parallel execution
|
||||
|
||||
-- Register exported files to A_SOURCE_FILE_RECEIVED if requested (after successful export)
|
||||
IF pRegisterExport THEN
|
||||
@@ -1307,57 +1309,99 @@ AS
|
||||
vFileName := NVL(vFileBaseName, UPPER(REPLACE(vTableName, vSchemaName || '.', ''))) || '_' || vPartitions(i).year || vPartitions(i).month || '.csv';
|
||||
vFileUri := vBucketUri || CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || sanitizeFilename(vFileName);
|
||||
|
||||
-- Get file metadata from OCI bucket (CHECKSUM, CREATED, BYTES)
|
||||
-- Get file metadata from OCI bucket (CHECKSUM, CREATED, BYTES) with retry logic
|
||||
DECLARE
|
||||
vChecksum VARCHAR2(128);
|
||||
vCreated TIMESTAMP WITH TIME ZONE;
|
||||
vBytes NUMBER;
|
||||
vSanitizedFileName VARCHAR2(1000);
|
||||
BEGIN
|
||||
-- Sanitize filename first (PL/SQL function cannot be used directly in SQL)
|
||||
vSanitizedFileName := sanitizeFilename(vFileName);
|
||||
|
||||
SELECT checksum, created, bytes
|
||||
INTO vChecksum, vCreated, vBytes
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => pCredentialName,
|
||||
location_uri => vBucketUri
|
||||
))
|
||||
WHERE object_name = CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || vSanitizedFileName;
|
||||
|
||||
-- Create A_SOURCE_FILE_RECEIVED record for this export with metadata
|
||||
vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED (
|
||||
A_SOURCE_FILE_RECEIVED_KEY,
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
SOURCE_FILE_NAME,
|
||||
CHECKSUM,
|
||||
CREATED,
|
||||
BYTES,
|
||||
RECEPTION_DATE,
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
vConfigKey, -- Config key from A_SOURCE_FILE_CONFIG lookup
|
||||
vFileUri,
|
||||
vChecksum,
|
||||
vCreated,
|
||||
vBytes,
|
||||
SYSDATE,
|
||||
'ARCHIVED',
|
||||
vPartitions(i).year,
|
||||
vPartitions(i).month,
|
||||
vFileName
|
||||
);
|
||||
vActualFileName VARCHAR2(1000); -- Actual filename with Oracle suffix
|
||||
vSanitizedFileName VARCHAR2(1000);
|
||||
vRetryCount NUMBER := 0;
|
||||
vMaxRetries NUMBER := 1; -- One retry after initial attempt
|
||||
vRetryDelay NUMBER := 2; -- 2 seconds delay
|
||||
BEGIN
|
||||
-- Sanitize filename first (PL/SQL function cannot be used directly in SQL)
|
||||
vSanitizedFileName := sanitizeFilename(vFileName);
|
||||
|
||||
-- Remove .csv extension for LIKE pattern matching (Oracle adds suffixes BEFORE .csv)
|
||||
-- Example: LEGACY_DEBT_202508.csv becomes LEGACY_DEBT_202508_1_20260211T102621591769Z.csv
|
||||
vSanitizedFileName := REGEXP_REPLACE(vSanitizedFileName, '\.csv$', '', 1, 0, 'i');
|
||||
|
||||
-- Try to get file metadata with retry logic
|
||||
<<metadata_retry_loop>>
|
||||
LOOP
|
||||
BEGIN
|
||||
SELECT object_name, checksum, created, bytes
|
||||
INTO vActualFileName, vChecksum, vCreated, vBytes
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => pCredentialName,
|
||||
location_uri => vBucketUri
|
||||
))
|
||||
WHERE object_name LIKE CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END || vSanitizedFileName || '%'
|
||||
ORDER BY created DESC, bytes DESC
|
||||
FETCH FIRST 1 ROW ONLY;
|
||||
|
||||
-- Extract filename only from full path (remove bucket folder prefix)
|
||||
-- vActualFileName contains: 'ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202508_1_20260211T111341375171Z.csv'
|
||||
-- Extract only: 'LEGACY_DEBT_202508_1_20260211T111341375171Z.csv'
|
||||
vActualFileName := SUBSTR(vActualFileName, INSTR(vActualFileName, '/', -1) + 1);
|
||||
|
||||
-- Success - exit retry loop
|
||||
EXIT metadata_retry_loop;
|
||||
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
vRetryCount := vRetryCount + 1;
|
||||
|
||||
IF vRetryCount <= vMaxRetries THEN
|
||||
-- Log retry attempt
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File not found in bucket (attempt ' || vRetryCount || '/' || (vMaxRetries + 1) || '), retrying after ' || vRetryDelay || ' seconds: ' || vFileName, 'DEBUG', vParameters);
|
||||
|
||||
-- Wait before retry using DBMS_SESSION.SLEEP (alternative to DBMS_LOCK)
|
||||
DBMS_SESSION.SLEEP(vRetryDelay);
|
||||
ELSE
|
||||
-- Max retries exceeded - re-raise exception
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
END LOOP metadata_retry_loop;
|
||||
|
||||
-- Create A_SOURCE_FILE_RECEIVED record for this export with metadata
|
||||
vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED (
|
||||
A_SOURCE_FILE_RECEIVED_KEY,
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
SOURCE_FILE_NAME,
|
||||
CHECKSUM,
|
||||
CREATED,
|
||||
BYTES,
|
||||
RECEPTION_DATE,
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
vConfigKey, -- Config key from A_SOURCE_FILE_CONFIG lookup
|
||||
vActualFileName, -- Use actual filename with Oracle suffix
|
||||
vChecksum,
|
||||
vCreated,
|
||||
vBytes,
|
||||
SYSDATE,
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for CSV exports
|
||||
NULL, -- PARTITION_MONTH not used for CSV exports
|
||||
NULL -- ARCH_FILE_NAME not used for CSV exports
|
||||
);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vFileName || ', Size=' || vBytes || ' bytes', 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || vBytes || ' bytes', 'DEBUG', vParameters);
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
-- File not found in bucket - log warning but continue
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('WARNING: File not found in bucket for metadata lookup: ' || vFileName, 'WARNING', vParameters);
|
||||
-- File not found after retries - log warning and continue without metadata
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('WARNING: File not found in bucket after ' || (vMaxRetries + 1) || ' attempts: ' || vFileName, 'WARNING', vParameters);
|
||||
|
||||
-- Sanitize filename for fallback INSERT (function cannot be used in SQL)
|
||||
vSanitizedFileName := sanitizeFilename(vFileName);
|
||||
|
||||
-- Insert without metadata
|
||||
vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
|
||||
@@ -1373,12 +1417,12 @@ AS
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
vConfigKey, -- Config key from A_SOURCE_FILE_CONFIG lookup
|
||||
vFileUri,
|
||||
vSanitizedFileName, -- Fallback: use theoretical filename if actual not found
|
||||
SYSDATE,
|
||||
'ARCHIVED',
|
||||
vPartitions(i).year,
|
||||
vPartitions(i).month,
|
||||
vFileName
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for CSV exports
|
||||
NULL, -- PARTITION_MONTH not used for CSV exports
|
||||
NULL -- ARCH_FILE_NAME not used for CSV exports
|
||||
);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
Reference in New Issue
Block a user