admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update DATA_EXPORTER package to v2.17.0: Fix RFC 4180 compliance and Parquet format corruption

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-12 08:50:08 +01:00
parent 5ba6c30fda
commit 202b535f9f
10 changed files with 346 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/DATA_EXPORTER.pkb
View File

@@ -59,7 +59,8 @@ AS
pTableName IN VARCHAR2,
pSchemaName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pTemplateTableName IN VARCHAR2
pTemplateTableName IN VARCHAR2,
pFormat IN VARCHAR2 DEFAULT 'CSV'
) RETURN VARCHAR2 IS
vResult VARCHAR2(32767);
vColumns VARCHAR2(32767);
@@ -173,10 +174,21 @@ AS
vResult := vResult || CASE WHEN vResult IS NOT NULL THEN ', ' ELSE '' END ||
'TO_CHAR(T.' || vCurrentCol || ', ''' || vDateFormat || ''') AS ' || vCurrentCol;
-- Other columns as-is with T. prefix
-- Other columns: RFC 4180 quote-doubling for CSV character types, as-is for Parquet/others
-- Oracle DBMS_CLOUD.EXPORT_DATA has no native RFC 4180 "" doubling.
-- escape=true -> backslash-escaped (\"\) - incompatible with ORACLE_LOADER.
-- escape=false -> unescaped embedded quotes - also incompatible.
-- Solution (CSV only): pre-double any " in VARCHAR2/CHAR/CLOB before Oracle wraps in quote.
-- Oracle then writes: "BIDDER-'""=:" which ORACLE_LOADER reads as BIDDER-'"=:.
-- Parquet is binary - no quoting needed; REPLACE would corrupt string data.
ELSE
vResult := vResult || CASE WHEN vResult IS NOT NULL THEN ', ' ELSE '' END ||
'T.' || vCurrentCol;
IF pFormat = 'CSV' AND vDataType IN ('VARCHAR2', 'NVARCHAR2', 'CHAR', 'NCHAR', 'CLOB', 'NCLOB') THEN
vResult := vResult || CASE WHEN vResult IS NOT NULL THEN ', ' ELSE '' END ||
'REPLACE(T.' || vCurrentCol || ', CHR(34), CHR(34)||CHR(34)) AS ' || vCurrentCol;
ELSE
vResult := vResult || CASE WHEN vResult IS NOT NULL THEN ', ' ELSE '' END ||
'T.' || vCurrentCol;
END IF;
END IF;
vPos := vNextPos + 1;
@@ -808,7 +820,7 @@ AS
VALIDATE_TABLE_AND_COLUMNS(vSchemaName, vTableName, vKeyColumnName, pColumnList, vParameters);
-- Build query with TO_CHAR for date columns (per-column format support)
vProcessedColumnList := buildQueryWithDateFormats(pColumnList, vTableName, vSchemaName, vKeyColumnName, pTemplateTableName);
vProcessedColumnList := buildQueryWithDateFormats(pColumnList, vTableName, vSchemaName, vKeyColumnName, pTemplateTableName, 'PARQUET');
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (building dynamic list from table metadata)'), 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with TO_CHAR for date columns: ' || vProcessedColumnList, 'DEBUG', vParameters);

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/DATA_EXPORTER.pkg
View File

@@ -9,12 +9,14 @@ AS
**/
-- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.6.3';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2026-01-28 19:30:00';
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.17.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2026-03-11 10:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.17.0 (2026-03-11): PARQUET FIX - Added pFormat parameter to buildQueryWithDateFormats. REPLACE(col,CHR(34)) now applied only when pFormat=CSV. EXPORT_TABLE_DATA_BY_DATE passes PARQUET - string data was being corrupted (single " doubled to ""). Parquet is binary and needs no quote escaping.' || CHR(10) ||
'v2.16.0 (2026-03-11): RFC 4180 FIX - Added REPLACE(col,CHR(34),CHR(34)||CHR(34)) in buildQueryWithDateFormats for VARCHAR2/CHAR/CLOB. Pre-doubled values produce compliant CSV for ORACLE_LOADER OPTIONALLY ENCLOSED BY chr(34).' || CHR(10) ||
'v2.6.3 (2026-01-28): COMPILATION FIX - Resolved ORA-00904 error in EXPORT_PARTITION_PARALLEL. SQLERRM and DBMS_UTILITY.FORMAT_ERROR_BACKTRACE cannot be used directly in SQL UPDATE statements. Now properly assigned to vgMsgTmp variable before UPDATE.' || CHR(10) ||
'v2.6.2 (2026-01-28): CRITICAL FIX - Race condition when multiple exports run simultaneously. Changed DELETE to filter by age (>24h) instead of deleting all COMPLETED chunks. Prevents concurrent sessions from deleting each other chunks. Session-safe cleanup with TASK_NAME filtering. Enables true parallel execution of multiple export jobs.' || CHR(10) ||
'v2.6.1 (2026-01-28): Added DELETE_FAILED_EXPORT_FILE procedure to clean up partial/corrupted files before retry. When partition fails mid-export, partial file is deleted before retry to prevent Oracle from creating _1 suffixed duplicates. Ensures clean retry without orphaned files in OCI bucket.' || CHR(10) ||