admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add JOB_CLASS parameter to DATA_EXPORTER for Oracle Scheduler support

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-18 12:58:06 +01:00
parent 6ade77c3c0
commit b6cf54f103
3 changed files with 55 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/A_PARALLEL_EXPORT_CHUNKS.sql
View File

@@ -43,6 +43,7 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
FILE_BASE_NAME VARCHAR2(1000), FILE_BASE_NAME VARCHAR2(1000),
TEMPLATE_TABLE_NAME VARCHAR2(200), TEMPLATE_TABLE_NAME VARCHAR2(200),
MAX_FILE_SIZE NUMBER DEFAULT 104857600 NOT NULL, MAX_FILE_SIZE NUMBER DEFAULT 104857600 NOT NULL,
JOB_CLASS VARCHAR2(128),
STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL, STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL,
ERROR_MESSAGE VARCHAR2(4000), ERROR_MESSAGE VARCHAR2(4000),
EXPORT_TIMESTAMP TIMESTAMP, EXPORT_TIMESTAMP TIMESTAMP,
@@ -69,6 +70,7 @@ COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FORMAT_TYPE IS 'Export format
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FILE_BASE_NAME IS 'Base filename for CSV exports (NULL for Parquet)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.FILE_BASE_NAME IS 'Base filename for CSV exports (NULL for Parquet)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TEMPLATE_TABLE_NAME IS 'Template table name for per-column date format configuration (e.g., CT_ET_TEMPLATES.TABLE_NAME)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TEMPLATE_TABLE_NAME IS 'Template table name for per-column date format configuration (e.g., CT_ET_TEMPLATES.TABLE_NAME)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MAX_FILE_SIZE IS 'Maximum file size in bytes for CSV exports only (e.g., 104857600 = 100MB, 1073741824 = 1GB) - default 100MB (104857600). NOTE: Not applicable for PARQUET format (Oracle limitation)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MAX_FILE_SIZE IS 'Maximum file size in bytes for CSV exports only (e.g., 104857600 = 100MB, 1073741824 = 1GB) - default 100MB (104857600). NOTE: Not applicable for PARQUET format (Oracle limitation)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.JOB_CLASS IS 'Oracle Scheduler job class name for resource management (e.g., ''high'', ''DEFAULT_JOB_CLASS'') - NULL uses default scheduler priority';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.STATUS IS 'Chunk processing status: PENDING (not started), PROCESSING (in progress), COMPLETED (success), FAILED (error) - allows retry of failed partitions only'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.STATUS IS 'Chunk processing status: PENDING (not started), PROCESSING (in progress), COMPLETED (success), FAILED (error) - allows retry of failed partitions only';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.ERROR_MESSAGE IS 'Error message if chunk processing failed (STATUS = FAILED)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.ERROR_MESSAGE IS 'Error message if chunk processing failed (STATUS = FAILED)';
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.EXPORT_TIMESTAMP IS 'Timestamp when chunk export was completed (STATUS = COMPLETED)'; COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.EXPORT_TIMESTAMP IS 'Timestamp when chunk export was completed (STATUS = COMPLETED)';

43
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkb
View File

@@ -501,6 +501,7 @@ AS
vFormat VARCHAR2(20); vFormat VARCHAR2(20);
vFileBaseName VARCHAR2(1000); vFileBaseName VARCHAR2(1000);
vMaxFileSize NUMBER; vMaxFileSize NUMBER;
vJobClass VARCHAR2(128);
vParameters VARCHAR2(4000); vParameters VARCHAR2(4000);
BEGIN BEGIN
-- Retrieve chunk context from global temporary table -- Retrieve chunk context from global temporary table
@@ -518,7 +519,8 @@ AS
CREDENTIAL_NAME, CREDENTIAL_NAME,
FORMAT_TYPE, FORMAT_TYPE,
FILE_BASE_NAME, FILE_BASE_NAME,
MAX_FILE_SIZE MAX_FILE_SIZE,
JOB_CLASS
INTO INTO
vYear, vYear,
vMonth, vMonth,
@@ -533,7 +535,8 @@ AS
vCredentialName, vCredentialName,
vFormat, vFormat,
vFileBaseName, vFileBaseName,
vMaxFileSize vMaxFileSize,
vJobClass
FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
WHERE CHUNK_ID = pStartId; WHERE CHUNK_ID = pStartId;
@@ -953,6 +956,7 @@ AS
pMaxDate IN DATE default SYSDATE, pMaxDate IN DATE default SYSDATE,
pParallelDegree IN NUMBER default 1, pParallelDegree IN NUMBER default 1,
pTemplateTableName IN VARCHAR2 default NULL, pTemplateTableName IN VARCHAR2 default NULL,
pJobClass IN VARCHAR2 default NULL,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
) )
IS IS
@@ -976,6 +980,7 @@ AS
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||'''' ,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
,'pParallelDegree => '''||nvl(TO_CHAR(pParallelDegree), 'NULL')||'''' ,'pParallelDegree => '''||nvl(TO_CHAR(pParallelDegree), 'NULL')||''''
,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pJobClass => '''||nvl(pJobClass, 'NULL')||''''
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||'''' ,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
@@ -1056,10 +1061,10 @@ AS
WHEN NOT MATCHED THEN WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS) CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, JOB_CLASS, STATUS)
VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName, VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, 'PENDING') pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING')
WHEN MATCHED THEN WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName, UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
@@ -1091,14 +1096,24 @@ AS
); );
-- Execute task in parallel -- Execute task in parallel
ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel task: ' || vTaskName, 'DEBUG', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel task: ' || vTaskName || CASE WHEN pJobClass IS NOT NULL THEN ' with job class: ' || pJobClass ELSE '' END, 'DEBUG', vParameters);
IF pJobClass IS NOT NULL THEN
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree,
job_class => pJobClass
);
ELSE
DBMS_PARALLEL_EXECUTE.RUN_TASK( DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName, task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
language_flag => DBMS_SQL.NATIVE, language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree parallel_level => pParallelDegree
); );
END IF;
-- Check for errors -- Check for errors
DECLARE DECLARE
@@ -1203,6 +1218,7 @@ AS
pMaxFileSize IN NUMBER default 104857600, pMaxFileSize IN NUMBER default 104857600,
pRegisterExport IN BOOLEAN default FALSE, pRegisterExport IN BOOLEAN default FALSE,
pProcessName IN VARCHAR2 default 'DATA_EXPORTER', pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
pJobClass IN VARCHAR2 default NULL,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
) )
IS IS
@@ -1240,6 +1256,7 @@ AS
,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pMaxFileSize => '''||nvl(TO_CHAR(pMaxFileSize), 'NULL')||'''' ,'pMaxFileSize => '''||nvl(TO_CHAR(pMaxFileSize), 'NULL')||''''
,'pRegisterExport => '''||CASE WHEN pRegisterExport THEN 'TRUE' ELSE 'FALSE' END||'''' ,'pRegisterExport => '''||CASE WHEN pRegisterExport THEN 'TRUE' ELSE 'FALSE' END||''''
,'pJobClass => '''||nvl(pJobClass, 'NULL')||''''
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||'''' ,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
@@ -1348,10 +1365,10 @@ AS
WHEN NOT MATCHED THEN WHEN NOT MATCHED THEN
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME, INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE, BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, STATUS) CREDENTIAL_NAME, FORMAT_TYPE, FILE_BASE_NAME, TEMPLATE_TABLE_NAME, MAX_FILE_SIZE, JOB_CLASS, STATUS)
VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName, VALUES (i, vTaskName, vPartitions(i).year, vPartitions(i).month, vSchemaName, vTableName, vKeyColumnName,
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate, vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, 'PENDING') pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING')
WHEN MATCHED THEN WHEN MATCHED THEN
UPDATE SET TASK_NAME = vTaskName, UPDATE SET TASK_NAME = vTaskName,
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END, STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
@@ -1383,14 +1400,24 @@ AS
); );
-- Execute task in parallel -- Execute task in parallel
ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel CSV export task: ' || vTaskName, 'DEBUG', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Executing parallel CSV export task: ' || vTaskName || CASE WHEN pJobClass IS NOT NULL THEN ' with job class: ' || pJobClass ELSE '' END, 'DEBUG', vParameters);
IF pJobClass IS NOT NULL THEN
DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree,
job_class => pJobClass
);
ELSE
DBMS_PARALLEL_EXECUTE.RUN_TASK( DBMS_PARALLEL_EXECUTE.RUN_TASK(
task_name => vTaskName, task_name => vTaskName,
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;', sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
language_flag => DBMS_SQL.NATIVE, language_flag => DBMS_SQL.NATIVE,
parallel_level => pParallelDegree parallel_level => pParallelDegree
); );
END IF;
-- Check for errors -- Check for errors
DECLARE DECLARE

11
MARS_Packages/REL01_ADDITIONS/MARS-835-PREHOOK/new_version/DATA_EXPORTER.pkg
View File

@@ -9,18 +9,17 @@ AS
**/ **/
-- Package Version Information -- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.10.1'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.11.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-17 16:00:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (last 3-5 changes) -- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) ||
'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) || 'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) ||
'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) || 'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) ||
'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) || 'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) ||
'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10) || 'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10);
'v2.8.0 (2026-02-12): MAJOR REFACTOR - EXPORT_TABLE_DATA now exports to single CSV file instead of partitioning by key values. Added pFileName parameter.' || CHR(10) ||
'v2.7.5 (2026-02-11): Added pRegisterExport parameter to EXPORT_TABLE_DATA procedure. When TRUE, registers each exported CSV file in A_SOURCE_FILE_RECEIVED.' || CHR(10);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgMsgTmp VARCHAR2(32000); vgMsgTmp VARCHAR2(32000);
@@ -148,6 +147,7 @@ AS
pMaxDate IN DATE default SYSDATE, pMaxDate IN DATE default SYSDATE,
pParallelDegree IN NUMBER default 1, pParallelDegree IN NUMBER default 1,
pTemplateTableName IN VARCHAR2 default NULL, pTemplateTableName IN VARCHAR2 default NULL,
pJobClass IN VARCHAR2 default NULL,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
); );
@@ -210,6 +210,7 @@ AS
pMaxFileSize IN NUMBER default 104857600, pMaxFileSize IN NUMBER default 104857600,
pRegisterExport IN BOOLEAN default FALSE, pRegisterExport IN BOOLEAN default FALSE,
pProcessName IN VARCHAR2 default 'DATA_EXPORTER', pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
pJobClass IN VARCHAR2 default NULL,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
); );