Enhance EXPORT_TABLE_DATA procedure by adding pTemplateTableName parameter for flexible column order and date formatting. Update version history for clarity on recent changes.
This commit is contained in:
Grzegorz Michalski
@@ -597,12 +597,13 @@ AS
|
|||||||
----------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
PROCEDURE EXPORT_TABLE_DATA (
|
PROCEDURE EXPORT_TABLE_DATA (
|
||||||
pSchemaName IN VARCHAR2,
|
pSchemaName IN VARCHAR2,
|
||||||
pTableName IN VARCHAR2,
|
pTableName IN VARCHAR2,
|
||||||
pKeyColumnName IN VARCHAR2,
|
pKeyColumnName IN VARCHAR2,
|
||||||
pBucketArea IN VARCHAR2,
|
pBucketArea IN VARCHAR2,
|
||||||
pFolderName IN VARCHAR2,
|
pFolderName IN VARCHAR2,
|
||||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
pTemplateTableName IN VARCHAR2 default NULL,
|
||||||
|
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||||
)
|
)
|
||||||
IS
|
IS
|
||||||
-- Type definition for key values
|
-- Type definition for key values
|
||||||
@@ -621,7 +622,6 @@ AS
|
|||||||
vBucketUri VARCHAR2(4000);
|
vBucketUri VARCHAR2(4000);
|
||||||
vProcessedColumnList VARCHAR2(32767);
|
vProcessedColumnList VARCHAR2(32767);
|
||||||
vCurrentCol VARCHAR2(128);
|
vCurrentCol VARCHAR2(128);
|
||||||
vAllColumnsList VARCHAR2(32767);
|
|
||||||
|
|
||||||
BEGIN
|
BEGIN
|
||||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||||
@@ -629,6 +629,7 @@ AS
|
|||||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||||
|
,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
|
||||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||||
));
|
));
|
||||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||||
@@ -670,18 +671,43 @@ AS
|
|||||||
AND column_name = vKeyColumnName
|
AND column_name = vKeyColumnName
|
||||||
AND owner = vSchemaName;
|
AND owner = vSchemaName;
|
||||||
|
|
||||||
-- Build list of all columns for the table (including key column for aliasing)
|
-- Validate template table if provided
|
||||||
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
|
IF pTemplateTableName IS NOT NULL THEN
|
||||||
INTO vAllColumnsList
|
DECLARE
|
||||||
FROM all_tab_columns
|
vTemplateSchema VARCHAR2(128);
|
||||||
WHERE table_name = vTableName
|
vTemplateTable VARCHAR2(128);
|
||||||
AND owner = vSchemaName;
|
vTemplateCount NUMBER;
|
||||||
|
BEGIN
|
||||||
|
-- Parse template table name (SCHEMA.TABLE or just TABLE)
|
||||||
|
IF INSTR(pTemplateTableName, '.') > 0 THEN
|
||||||
|
vTemplateSchema := UPPER(SUBSTR(pTemplateTableName, 1, INSTR(pTemplateTableName, '.') - 1));
|
||||||
|
vTemplateTable := UPPER(SUBSTR(pTemplateTableName, INSTR(pTemplateTableName, '.') + 1));
|
||||||
|
ELSE
|
||||||
|
vTemplateSchema := vSchemaName;
|
||||||
|
vTemplateTable := UPPER(pTemplateTableName);
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- Check if template table exists
|
||||||
|
SELECT COUNT(*) INTO vTemplateCount
|
||||||
|
FROM all_tables
|
||||||
|
WHERE table_name = vTemplateTable
|
||||||
|
AND owner = vTemplateSchema;
|
||||||
|
|
||||||
|
IF vTemplateCount = 0 THEN
|
||||||
|
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS || ': Template table ' || vTemplateSchema || '.' || vTemplateTable;
|
||||||
|
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
ENV_MANAGER.LOG_PROCESS_EVENT('Template table validated: ' || vTemplateSchema || '.' || vTemplateTable, 'DEBUG', vParameters);
|
||||||
|
END;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- Build query with TO_CHAR for date columns (per-column format support)
|
||||||
|
vProcessedColumnList := buildQueryWithDateFormats(NULL, vTableName, vSchemaName, vKeyColumnName, pTemplateTableName);
|
||||||
|
|
||||||
-- Process column list to add T. prefix and alias key column as A_WORKFLOW_HISTORY_KEY
|
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with TO_CHAR for date columns: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||||
vProcessedColumnList := processColumnList(vAllColumnsList, vTableName, vSchemaName, vKeyColumnName);
|
ENV_MANAGER.LOG_PROCESS_EVENT('Template table: ' || NVL(pTemplateTableName, 'NULL - using global default for all dates'), 'INFO', vParameters);
|
||||||
|
|
||||||
ENV_MANAGER.LOG_PROCESS_EVENT('Dynamic column list built: ' || vAllColumnsList, 'DEBUG', vParameters);
|
|
||||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with T. prefix: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
|
||||||
|
|
||||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||||
-- Fetch unique key values from A_LOAD_HISTORY
|
-- Fetch unique key values from A_LOAD_HISTORY
|
||||||
|
|||||||
@@ -15,21 +15,14 @@ AS
|
|||||||
|
|
||||||
-- Version History (last 3-5 changes)
|
-- Version History (last 3-5 changes)
|
||||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||||
'v2.7.4 (2026-02-11): ACTUAL FILENAME STORAGE - Store real filename with Oracle suffix in SOURCE_FILE_NAME instead of theoretical filename. Changes LIST_OBJECTS query to SELECT object_name and stores actual filename like LEGACY_DEBT_202508_1_20260211T111341375171Z.csv instead of LEGACY_DEBT_202508.csv. Enables accurate file tracking.' || CHR(10) ||
|
'v2.7.4 (2026-02-11): ACTUAL FILENAME STORAGE - Store real filename with Oracle suffix in SOURCE_FILE_NAME instead of theoretical filename.' || CHR(10) ||
|
||||||
'v2.7.3 (2026-02-11): FIX LIKE pattern for DBMS_CLOUD.LIST_OBJECTS - Removed .csv extension from filename before pattern matching. Oracle EXPORT_DATA creates files with suffixes BEFORE .csv so LIKE pattern should be filename% not filename.csv%. Enables proper metadata retrieval (CHECKSUM, CREATED, BYTES).' || CHR(10) ||
|
'v2.7.3 (2026-02-11): FIX LIKE pattern for DBMS_CLOUD.LIST_OBJECTS - Removed .csv extension from filename before pattern matching.' || CHR(10) ||
|
||||||
'v2.7.2 (2026-02-11): FIX pRegisterExport in EXPORT_TABLE_DATA_TO_CSV_BY_DATE - Added missing pRegisterExport parameter to EXPORT_SINGLE_PARTITION call. Previously files were not registered because parameter was not passed through.' || CHR(10) ||
|
'v2.7.2 (2026-02-11): FIX pRegisterExport in EXPORT_TABLE_DATA_TO_CSV_BY_DATE - Added missing pRegisterExport parameter to EXPORT_SINGLE_PARTITION call.' || CHR(10) ||
|
||||||
'v2.7.1 (2026-02-11): AUTO-LOOKUP A_SOURCE_FILE_CONFIG_KEY - Parse pFolderName to automatically find config key from A_SOURCE_FILE_CONFIG. Example: ODS/CSDB/CSDB_DEBT_DAILY extracts SOURCE_KEY=CSDB, TABLE_ID=CSDB_DEBT_DAILY.' || CHR(10) ||
|
'v2.7.1 (2026-02-11): AUTO-LOOKUP A_SOURCE_FILE_CONFIG_KEY - Parse pFolderName to automatically find config key from A_SOURCE_FILE_CONFIG.' || CHR(10) ||
|
||||||
'v2.7.0 (2026-02-10): Added pRegisterExport parameter to EXPORT_TABLE_DATA_TO_CSV_BY_DATE. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED with metadata from DBMS_CLOUD.LIST_OBJECTS. Enables file tracking and integrity verification.' || CHR(10);
|
'v2.7.0 (2026-02-10): Added pRegisterExport parameter to EXPORT_TABLE_DATA_TO_CSV_BY_DATE. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10) ||
|
||||||
'v2.6.3 (2026-01-28): COMPILATION FIX - Resolved ORA-00904 error in EXPORT_PARTITION_PARALLEL. SQLERRM and DBMS_UTILITY.FORMAT_ERROR_BACKTRACE cannot be used directly in SQL UPDATE statements. Now properly assigned to vgMsgTmp variable before UPDATE.' || CHR(10) ||
|
'v2.6.3 (2026-01-28): COMPILATION FIX - Resolved ORA-00904 error in EXPORT_PARTITION_PARALLEL. SQLERRM properly assigned to vgMsgTmp variable.' || CHR(10) ||
|
||||||
'v2.6.2 (2026-01-28): CRITICAL FIX - Race condition when multiple exports run simultaneously. Changed DELETE to filter by age (>24h) instead of deleting all COMPLETED chunks. Prevents concurrent sessions from deleting each other chunks. Session-safe cleanup with TASK_NAME filtering. Enables true parallel execution of multiple export jobs.' || CHR(10) ||
|
'v2.6.2 (2026-01-28): CRITICAL FIX - Race condition when multiple exports run simultaneously. Session-safe cleanup with TASK_NAME filtering.' || CHR(10) ||
|
||||||
'v2.6.0 (2026-01-28): CRITICAL FIX - Added STATUS tracking to A_PARALLEL_EXPORT_CHUNKS table to prevent data duplication on retry. System now restarts ONLY failed partitions instead of re-exporting all data. Added ERROR_MESSAGE and EXPORT_TIMESTAMP columns for better error handling and monitoring. Prevents duplicate file creation when parallel tasks fail (e.g., 22 partitions with 16 threads, 3 failures no longer duplicates 19 successful exports).' || CHR(10) ||
|
'v2.6.0 (2026-01-28): CRITICAL FIX - Added STATUS tracking to A_PARALLEL_EXPORT_CHUNKS table to prevent data duplication on retry.' || CHR(10);
|
||||||
'v2.5.0 (2026-01-26): Added recorddelimiter parameter with CRLF (CHR(13)||CHR(10)) for CSV exports to ensure Windows-compatible line endings. Improves cross-platform compatibility when CSV files are opened in Windows applications (Notepad, Excel).' || CHR(10) ||
|
|
||||||
'v2.4.0 (2026-01-11): Added pTemplateTableName parameter for per-column date format configuration. Implements dynamic query building with TO_CHAR for each date/timestamp column using FILE_MANAGER.GET_DATE_FORMAT. Supports 3-tier hierarchy: column-specific, template DEFAULT, global fallback. Eliminates single dateformat limitation of DBMS_CLOUD.EXPORT_DATA.' || CHR(10) ||
|
|
||||||
'v2.3.0 (2025-12-20): Added parallel partition processing using DBMS_PARALLEL_EXECUTE. New pParallelDegree parameter (1-16, default 1) for EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures. Each year/month partition processed in separate thread for improved performance.' || CHR(10) ||
|
|
||||||
'v2.2.0 (2025-12-19): DRY refactoring - extracted shared helper functions (sanitizeFilename, VALIDATE_TABLE_AND_COLUMNS, GET_PARTITIONS, EXPORT_SINGLE_PARTITION worker procedure). Reduced code duplication by ~400 lines. Prepared architecture for v2.3.0 parallel processing.' || CHR(10) ||
|
|
||||||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
|
|
||||||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
|
|
||||||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10);
|
|
||||||
|
|
||||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||||
vgMsgTmp VARCHAR2(32000);
|
vgMsgTmp VARCHAR2(32000);
|
||||||
@@ -79,24 +72,32 @@ AS
|
|||||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||||
* Exports data into CSV file on OCI infrustructure.
|
* Exports data into CSV file on OCI infrustructure.
|
||||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||||
|
* Supports template table for column order and per-column date formatting.
|
||||||
|
* @param pTemplateTableName - Optional template table (SCHEMA.TABLE or TABLE) for:
|
||||||
|
* - Column order control (template defines CSV structure)
|
||||||
|
* - Per-column date formatting via FILE_MANAGER.GET_DATE_FORMAT
|
||||||
|
* - NULL = use source table columns in natural order
|
||||||
* @example
|
* @example
|
||||||
* begin
|
* begin
|
||||||
* DATA_EXPORTER.EXPORT_TABLE_DATA(
|
* DATA_EXPORTER.EXPORT_TABLE_DATA(
|
||||||
* pSchemaName => 'CT_MRDS',
|
* pSchemaName => 'CT_MRDS',
|
||||||
* pTableName => 'MY_TABLE',
|
* pTableName => 'MY_TABLE',
|
||||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||||
* pBucketArea => 'DATA',
|
* pBucketArea => 'DATA',
|
||||||
* pFolderName => 'csv_exports'
|
* pFolderName => 'csv_exports',
|
||||||
|
* pTemplateTableName => 'CT_ET_TEMPLATES.MY_TEMPLATE' -- Optional
|
||||||
|
* );
|
||||||
* );
|
* );
|
||||||
* end;
|
* end;
|
||||||
**/
|
**/
|
||||||
PROCEDURE EXPORT_TABLE_DATA (
|
PROCEDURE EXPORT_TABLE_DATA (
|
||||||
pSchemaName IN VARCHAR2,
|
pSchemaName IN VARCHAR2,
|
||||||
pTableName IN VARCHAR2,
|
pTableName IN VARCHAR2,
|
||||||
pKeyColumnName IN VARCHAR2,
|
pKeyColumnName IN VARCHAR2,
|
||||||
pBucketArea IN VARCHAR2,
|
pBucketArea IN VARCHAR2,
|
||||||
pFolderName IN VARCHAR2,
|
pFolderName IN VARCHAR2,
|
||||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
pTemplateTableName IN VARCHAR2 default NULL,
|
||||||
|
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user