diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql index a5514cb..bd37230 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql @@ -43,6 +43,7 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS ( FILE_BASE_NAME VARCHAR2(1000), TEMPLATE_TABLE_NAME VARCHAR2(200), MAX_FILE_SIZE NUMBER DEFAULT 104857600 NOT NULL, + JOB_CLASS VARCHAR2(128), STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL, ERROR_MESSAGE VARCHAR2(4000), EXPORT_TIMESTAMP TIMESTAMP, @@ -69,6 +70,7 @@ COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FORMAT_TYPE IS 'Export format COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FILE_BASE_NAME IS 'Base filename for CSV exports (NULL for Parquet)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TEMPLATE_TABLE_NAME IS 'Template table name for per-column date format configuration (e.g., CT_ET_TEMPLATES.TABLE_NAME)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MAX_FILE_SIZE IS 'Maximum file size in bytes for CSV exports only (e.g., 104857600 = 100MB, 1073741824 = 1GB) - default 100MB (104857600). NOTE: Not applicable for PARQUET format (Oracle limitation)'; +COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.JOB_CLASS IS 'Oracle Scheduler job class name for resource management (e.g., ''high'', ''DEFAULT_JOB_CLASS'') - NULL uses default scheduler priority'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.STATUS IS 'Chunk processing status: PENDING (not started), PROCESSING (in progress), COMPLETED (success), FAILED (error) - allows retry of failed partitions only'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.ERROR_MESSAGE IS 'Error message if chunk processing failed (STATUS = FAILED)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.EXPORT_TIMESTAMP IS 'Timestamp when chunk export was completed (STATUS = COMPLETED)'; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb index 762b650..05d97d8 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb @@ -501,6 +501,7 @@ AS vFormat VARCHAR2(20); vFileBaseName VARCHAR2(1000); vMaxFileSize NUMBER; + vJobClass VARCHAR2(128); vParameters VARCHAR2(4000); BEGIN -- Retrieve chunk context from global temporary table @@ -518,7 +519,8 @@ AS CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, - MAX_FILE_SIZE + MAX_FILE_SIZE, + JOB_CLASS INTO vYear, vMonth, @@ -533,7 +535,8 @@ AS vCredentialName, vFormat, vFileBaseName, - vMaxFileSize + vMaxFileSize, + vJobClass FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE CHUNK_ID = pStartId; @@ -953,6 +956,7 @@ AS pMaxDate IN DATE default SYSDATE, pParallelDegree IN NUMBER default 1, pTemplateTableName IN VARCHAR2 default NULL, + pJobClass IN VARCHAR2 default NULL, pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName ) IS @@ -976,6 +980,7 @@ AS ,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||'''' ,'pParallelDegree => '''||nvl(TO_CHAR(pParallelDegree), 'NULL')||'''' ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pJobClass => '''||nvl(pJobClass, 'NULL')||'''' ,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||'''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); @@ -1056,10 +1061,10 @@ AS WHEN NOT MATCHED THEN INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, - CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS) + CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, JOB_CLASS, STATUS) VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName, vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, - pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, 'PENDING') + pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING') WHEN MATCHED THEN UPDATE SET TASK_NAME = vTaskName, STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, @@ -1091,14 +1096,24 @@ AS ); -- Execute task in parallel - ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel task: ' || vTaskName, 'DEBUG', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel task: ' || vTaskName || CASE WHEN pJobClass IS NOT NULL THEN ' with job class: ' || pJobClass ELSE '' END, 'DEBUG', vParameters); - DBMS_PARALLEL_EXECUTE.RUN_TASK( - task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', - language_flag => DBMS_SQL.NATIVE, - parallel_level => pParallelDegree - ); + IF pJobClass IS NOT NULL THEN + DBMS_PARALLEL_EXECUTE.RUN_TASK( + task_name => vTaskName, + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + language_flag => DBMS_SQL.NATIVE, + parallel_level => pParallelDegree, + job_class => pJobClass + ); + ELSE + DBMS_PARALLEL_EXECUTE.RUN_TASK( + task_name => vTaskName, + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + language_flag => DBMS_SQL.NATIVE, + parallel_level => pParallelDegree + ); + END IF; -- Check for errors DECLARE @@ -1203,6 +1218,7 @@ AS pMaxFileSize IN NUMBER default 104857600, pRegisterExport IN BOOLEAN default FALSE, pProcessName IN VARCHAR2 default 'DATA_EXPORTER', + pJobClass IN VARCHAR2 default NULL, pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName ) IS @@ -1240,6 +1256,7 @@ AS ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pMaxFileSize => '''||nvl(TO_CHAR(pMaxFileSize), 'NULL')||'''' ,'pRegisterExport => '''||CASE WHEN pRegisterExport THEN 'TRUE' ELSE 'FALSE' END||'''' + ,'pJobClass => '''||nvl(pJobClass, 'NULL')||'''' ,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||'''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); @@ -1348,10 +1365,10 @@ AS WHEN NOT MATCHED THEN INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, - CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS) + CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, JOB_CLASS, STATUS) VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName, vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, - pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, 'PENDING') + pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING') WHEN MATCHED THEN UPDATE SET TASK_NAME = vTaskName, STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, @@ -1383,14 +1400,24 @@ AS ); -- Execute task in parallel - ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel CSV export task: ' || vTaskName, 'DEBUG', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel CSV export task: ' || vTaskName || CASE WHEN pJobClass IS NOT NULL THEN ' with job class: ' || pJobClass ELSE '' END, 'DEBUG', vParameters); - DBMS_PARALLEL_EXECUTE.RUN_TASK( - task_name => vTaskName, - sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', - language_flag => DBMS_SQL.NATIVE, - parallel_level => pParallelDegree - ); + IF pJobClass IS NOT NULL THEN + DBMS_PARALLEL_EXECUTE.RUN_TASK( + task_name => vTaskName, + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + language_flag => DBMS_SQL.NATIVE, + parallel_level => pParallelDegree, + job_class => pJobClass + ); + ELSE + DBMS_PARALLEL_EXECUTE.RUN_TASK( + task_name => vTaskName, + sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', + language_flag => DBMS_SQL.NATIVE, + parallel_level => pParallelDegree + ); + END IF; -- Check for errors DECLARE diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg index 876e54a..a723db1 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg @@ -9,18 +9,17 @@ AS **/ -- Package Version Information - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.10.1'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-17 16:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.11.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (last 3-5 changes) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + 'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) || 'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) || 'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) || 'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) || - 'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10) || - 'v2.8.0 (2026-02-12): MAJOR REFACTOR - EXPORT_TABLE_DATA now exports to single CSV file instead of partitioning by key values. Added pFileName parameter.' || CHR(10) || - 'v2.7.5 (2026-02-11): Added pRegisterExport parameter to EXPORT_TABLE_DATA procedure. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10); + 'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10); cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); vgMsgTmp VARCHAR2(32000); @@ -148,6 +147,7 @@ AS pMaxDate IN DATE default SYSDATE, pParallelDegree IN NUMBER default 1, pTemplateTableName IN VARCHAR2 default NULL, + pJobClass IN VARCHAR2 default NULL, pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName ); @@ -210,6 +210,7 @@ AS pMaxFileSize IN NUMBER default 104857600, pRegisterExport IN BOOLEAN default FALSE, pProcessName IN VARCHAR2 default 'DATA_EXPORTER', + pJobClass IN VARCHAR2 default NULL, pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName );