MARS-1057

This commit is contained in:
Grzegorz Michalski
2026-02-20 15:08:18 +01:00
parent b85172ae84
commit ff034fcd68
4 changed files with 471 additions and 81 deletions

View File

@@ -1968,9 +1968,23 @@ AS
PROCEDURE CREATE_EXTERNAL_TABLES_SET ( PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER, pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
) )
IS IS
-- Type for storing grant information
TYPE tGrantRecord IS RECORD (
grantee VARCHAR2(128),
privilege VARCHAR2(40),
grantable VARCHAR2(3)
);
TYPE tGrantList IS TABLE OF tGrantRecord;
vInboxGrants tGrantList;
vOdsGrants tGrantList;
vArchiveGrants tGrantList;
vSourceKey VARCHAR2(50); vSourceKey VARCHAR2(50);
vSourceFileId VARCHAR2(100); vSourceFileId VARCHAR2(100);
vTableId VARCHAR2(100); vTableId VARCHAR2(100);
@@ -1986,28 +2000,124 @@ AS
vOdsPrefix VARCHAR2(500); vOdsPrefix VARCHAR2(500);
vArchivePrefix VARCHAR2(500); vArchivePrefix VARCHAR2(500);
vTableExists NUMBER;
vParameters VARCHAR2(4000); vParameters VARCHAR2(4000);
vAreaUpper VARCHAR2(20);
-- Nested procedure to save table grants before DROP
PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants for table: ' || pTableName, 'DEBUG');
SELECT grantee, privilege, grantable
BULK COLLECT INTO pGrantList
FROM ALL_TAB_PRIVS
WHERE table_schema = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
AND table_name = pTableName
AND grantee NOT IN ('SYS', 'SYSTEM', 'PUBLIC') -- Exclude system accounts
ORDER BY grantee, privilege;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Saved ' || pGrantList.COUNT || ' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName,
'INFO'
);
EXCEPTION
WHEN NO_DATA_FOUND THEN
pGrantList := tGrantList(); -- Empty list
ENV_MANAGER.LOG_PROCESS_EVENT('No grants found for table: ' || pTableName, 'INFO');
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(
'Warning: Could not save grants for ' || pTableName || ': ' || SQLERRM,
'WARNING'
);
pGrantList := tGrantList(); -- Empty list on error
END SAVE_GRANTS;
-- Nested procedure to restore table grants after CREATE
PROCEDURE RESTORE_GRANTS(pTableName VARCHAR2, pGrantList tGrantList) IS
vGrantSQL VARCHAR2(500);
vGrantCount NUMBER := 0;
vFailCount NUMBER := 0;
BEGIN
IF pGrantList IS NULL OR pGrantList.COUNT = 0 THEN
ENV_MANAGER.LOG_PROCESS_EVENT(
'No grants to restore for table: ' || pTableName,
'INFO'
);
RETURN;
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Restoring ' || pGrantList.COUNT || ' grants for table: ' || pTableName,
'DEBUG'
);
FOR i IN 1..pGrantList.COUNT LOOP
BEGIN
vGrantSQL := 'GRANT ' || pGrantList(i).privilege ||
' ON ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
' TO ' || pGrantList(i).grantee;
IF pGrantList(i).grantable = 'YES' THEN
vGrantSQL := vGrantSQL || ' WITH GRANT OPTION';
END IF;
EXECUTE IMMEDIATE vGrantSQL;
vGrantCount := vGrantCount + 1;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Restored grant: ' || pGrantList(i).privilege ||
' TO ' || pGrantList(i).grantee ||
CASE WHEN pGrantList(i).grantable = 'YES' THEN ' WITH GRANT OPTION' ELSE '' END,
'DEBUG'
);
EXCEPTION
WHEN OTHERS THEN
vFailCount := vFailCount + 1;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Warning: Could not restore grant (' || pGrantList(i).privilege ||
' TO ' || pGrantList(i).grantee || ') on ' || pTableName || ': ' || SQLERRM,
'WARNING'
);
END;
END LOOP;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Restored ' || vGrantCount || ' of ' || pGrantList.COUNT ||
' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
CASE WHEN vFailCount > 0 THEN ' (' || vFailCount || ' failed)' ELSE '' END,
'INFO'
);
END RESTORE_GRANTS;
PROCEDURE DROP_IF_EXISTS(pTableName VARCHAR2) IS PROCEDURE DROP_IF_EXISTS(pTableName VARCHAR2) IS
BEGIN BEGIN
SELECT COUNT(*) INTO vTableExists ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to drop table: ' || pTableName, 'DEBUG');
FROM ALL_TABLES EXECUTE IMMEDIATE 'DROP TABLE ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName;
WHERE OWNER = 'ODS' AND TABLE_NAME = pTableName; ENV_MANAGER.LOG_PROCESS_EVENT('Table dropped successfully: ' || pTableName, 'INFO');
IF vTableExists > 0 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Dropping existing table: ' || pTableName, 'INFO');
EXECUTE IMMEDIATE 'DROP TABLE ODS.' || pTableName;
END IF;
EXCEPTION EXCEPTION
WHEN OTHERS THEN WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING'); IF SQLCODE = -942 THEN -- ORA-00942: table or view does not exist
ENV_MANAGER.LOG_PROCESS_EVENT('Table does not exist, skipping drop: ' || pTableName, 'INFO');
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING');
RAISE; -- Re-raise if not "table not exists" error
END IF;
END DROP_IF_EXISTS; END DROP_IF_EXISTS;
BEGIN BEGIN
-- Validate and normalize pArea parameter
vAreaUpper := UPPER(TRIM(pArea));
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
END IF;
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'), 'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'),
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
'pArea => ''' || vAreaUpper || ''''
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
@@ -2017,7 +2127,7 @@ AS
NVL(ENCODING, 'UTF8') NVL(ENCODING, 'UTF8')
INTO vSourceKey, vSourceFileId, vTableId, vTemplateTableName, INTO vSourceKey, vSourceFileId, vTableId, vTemplateTableName,
vEncoding vEncoding
FROM A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- Set default delimiter (column DELIMITER does not exist in A_SOURCE_FILE_CONFIG) -- Set default delimiter (column DELIMITER does not exist in A_SOURCE_FILE_CONFIG)
@@ -2030,9 +2140,9 @@ AS
END; END;
-- 2. Generate table names -- 2. Generate table names
vInboxTableName := vSourceKey || '_' || vTableId || '_INBOX'; vInboxTableName := vTableId || '_INBOX';
vOdsTableName := vSourceKey || '_' || vTableId || '_ODS'; vOdsTableName := vTableId || '_ODS';
vArchiveTableName := vSourceKey || '_' || vTableId || '_ARCHIVE'; vArchiveTableName := vTableId || '_ARCHIVE';
-- 3. Generate paths (OFFICIAL PATH PATTERNS) -- 3. Generate paths (OFFICIAL PATH PATTERNS)
vInboxPrefix := 'INBOX/' || vSourceKey || '/' || vSourceFileId || '/' || vTableId; vInboxPrefix := 'INBOX/' || vSourceKey || '/' || vSourceFileId || '/' || vTableId;
@@ -2040,52 +2150,103 @@ AS
vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId; vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId;
ENV_MANAGER.LOG_PROCESS_EVENT( ENV_MANAGER.LOG_PROCESS_EVENT(
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId, 'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId ||
' (Area: ' || vAreaUpper || ')',
'INFO' 'INFO'
); );
-- 4. DROP existing tables if pRecreate = TRUE -- 4. DROP existing tables if pRecreate = TRUE
IF pRecreate THEN IF pRecreate THEN
DROP_IF_EXISTS(vInboxTableName); -- Save grants before dropping tables (if pRestoreGrants = TRUE)
DROP_IF_EXISTS(vOdsTableName); IF pRestoreGrants THEN
DROP_IF_EXISTS(vArchiveTableName); ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO');
IF vAreaUpper IN ('INBOX', 'ALL') THEN
SAVE_GRANTS(vInboxTableName, vInboxGrants);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
SAVE_GRANTS(vOdsTableName, vOdsGrants);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
SAVE_GRANTS(vArchiveTableName, vArchiveGrants);
END IF;
END IF;
-- Drop existing tables based on pArea
IF vAreaUpper IN ('INBOX', 'ALL') THEN
DROP_IF_EXISTS(vInboxTableName);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
DROP_IF_EXISTS(vOdsTableName);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
DROP_IF_EXISTS(vArchiveTableName);
END IF;
END IF; END IF;
-- 5. Create INBOX external table -- 5. Create INBOX external table (if requested)
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO'); IF vAreaUpper IN ('INBOX', 'ALL') THEN
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
pTableName => vInboxTableName, CREATE_EXTERNAL_TABLE(
pTemplateTableName => vTemplateTableName, pTableName => vInboxTableName,
pPrefix => vInboxPrefix, pTemplateTableName => vTemplateTableName,
pBucketUri => ENV_MANAGER.gvInboxBucketUri, pPrefix => vInboxPrefix,
pDelimiter => vDelimiter, pBucketUri => ENV_MANAGER.gvInboxBucketUri,
pEncoding => vEncoding pDelimiter => vDelimiter,
); pEncoding => vEncoding
);
END IF;
-- 6. Create ODS external table -- 6. Create ODS external table (if requested)
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO'); IF vAreaUpper IN ('ODS', 'ALL') THEN
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
pTableName => vOdsTableName, CREATE_EXTERNAL_TABLE(
pTemplateTableName => vTemplateTableName, pTableName => vOdsTableName,
pPrefix => vOdsPrefix, pTemplateTableName => vTemplateTableName,
pBucketUri => ENV_MANAGER.gvDataBucketUri, pPrefix => vOdsPrefix,
pDelimiter => vDelimiter, pBucketUri => ENV_MANAGER.gvDataBucketUri,
pEncoding => vEncoding pDelimiter => vDelimiter,
); pEncoding => vEncoding
);
END IF;
-- 7. Create ARCHIVE external table -- 7. Create ARCHIVE external table (if requested)
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO'); IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
pTableName => vArchiveTableName, CREATE_EXTERNAL_TABLE(
pTemplateTableName => vTemplateTableName, pTableName => vArchiveTableName,
pPrefix => vArchivePrefix, pTemplateTableName => vTemplateTableName,
pBucketUri => ENV_MANAGER.gvArchiveBucketUri, pPrefix => vArchivePrefix,
pDelimiter => vDelimiter, pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
pEncoding => vEncoding pDelimiter => vDelimiter,
); pEncoding => vEncoding
);
END IF;
-- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE)
IF pRecreate AND pRestoreGrants THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO');
IF vAreaUpper IN ('INBOX', 'ALL') THEN
RESTORE_GRANTS(vInboxTableName, vInboxGrants);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
RESTORE_GRANTS(vOdsTableName, vOdsGrants);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
RESTORE_GRANTS(vArchiveTableName, vArchiveGrants);
END IF;
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT( ENV_MANAGER.LOG_PROCESS_EVENT(
'End CREATE_EXTERNAL_TABLES_SET - Successfully created all 3 external tables for config: ' || pSourceFileConfigKey, 'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' ||
pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')',
'INFO', 'INFO',
vParameters vParameters
); );
@@ -2103,19 +2264,32 @@ AS
pSourceKey IN VARCHAR2 DEFAULT NULL, pSourceKey IN VARCHAR2 DEFAULT NULL,
pSourceFileId IN VARCHAR2 DEFAULT NULL, pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
) )
IS IS
vCount NUMBER := 0; vCount NUMBER := 0;
vProcessed NUMBER := 0; vProcessed NUMBER := 0;
vFailed NUMBER := 0; vFailed NUMBER := 0;
vParameters VARCHAR2(4000); vParameters VARCHAR2(4000);
vAreaUpper VARCHAR2(20);
BEGIN BEGIN
-- Validate and normalize pArea parameter
vAreaUpper := UPPER(TRIM(pArea));
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
END IF;
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''', 'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''', 'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
'pTableId => ''' || NVL(pTableId, 'NULL') || '''', 'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
'pArea => ''' || vAreaUpper || ''''
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
@@ -2123,7 +2297,7 @@ AS
-- Iterate over configurations matching criteria (only INPUT files) -- Iterate over configurations matching criteria (only INPUT files)
FOR rec IN ( FOR rec IN (
SELECT A_SOURCE_FILE_CONFIG_KEY, A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID SELECT A_SOURCE_FILE_CONFIG_KEY, A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID
FROM A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND (pSourceKey IS NULL OR A_SOURCE_KEY = pSourceKey) AND (pSourceKey IS NULL OR A_SOURCE_KEY = pSourceKey)
AND (pSourceFileId IS NULL OR SOURCE_FILE_ID = pSourceFileId) AND (pSourceFileId IS NULL OR SOURCE_FILE_ID = pSourceFileId)
@@ -2135,14 +2309,16 @@ AS
BEGIN BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT( ENV_MANAGER.LOG_PROCESS_EVENT(
'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' || 'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' ||
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID, rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')',
'INFO' 'INFO'
); );
-- Call procedure to create set of 3 tables -- Call procedure to create set of tables (based on pArea)
CREATE_EXTERNAL_TABLES_SET( CREATE_EXTERNAL_TABLES_SET(
pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY, pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY,
pRecreate => pRecreate pRecreate => pRecreate,
pRestoreGrants => pRestoreGrants,
pArea => vAreaUpper
); );
vProcessed := vProcessed + 1; vProcessed := vProcessed + 1;
@@ -2161,7 +2337,7 @@ AS
ENV_MANAGER.LOG_PROCESS_EVENT( ENV_MANAGER.LOG_PROCESS_EVENT(
'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount || 'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount ||
', Processed: ' || vProcessed || ', Failed: ' || vFailed, ', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')',
'INFO', 'INFO',
vParameters vParameters
); );

View File

@@ -17,12 +17,13 @@ AS
**/ **/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.5.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) ||
'3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) || '3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) ||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) || '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) || '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
@@ -602,48 +603,72 @@ AS
/** /**
* @name CREATE_EXTERNAL_TABLES_SET * @name CREATE_EXTERNAL_TABLES_SET
* @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration * @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table.
* from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following * Automatically generates table names and paths following official path patterns.
* official path patterns. Optionally drops and recreates existing tables. * Optionally drops and recreates existing tables. If pRestoreGrants is TRUE, saves and restores table grants.
* The pArea parameter allows selective table creation.
* @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table * @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table
* @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist * @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist
* @example BEGIN * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
* 'INBOX' - creates only INBOX table
* 'ODS' - creates only ODS table
* 'ARCHIVE' - creates only ARCHIVE table
* 'ALL' - creates all three tables (default)
* @example -- Create only INBOX table
* BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
* pSourceFileConfigKey => 123, * pSourceFileConfigKey => 123,
* pRecreate => FALSE * pArea => 'INBOX'
* ); * );
* END; * END;
* @ex_rslt Creates three external tables in ODS schema: *
* - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX * -- Create all tables with grant preservation
* - C2D_A_UC_DISSEM_METADATA_LOADS_ODS * BEGIN
* - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
* pSourceFileConfigKey => 123,
* pRecreate => TRUE,
* pRestoreGrants => TRUE,
* pArea => 'ALL'
* );
* END;
* @ex_rslt Creates external table(s) in ODS schema based on pArea parameter
**/ **/
PROCEDURE CREATE_EXTERNAL_TABLES_SET ( PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER, pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
); );
/** /**
* @name CREATE_EXTERNAL_TABLES_BATCH * @name CREATE_EXTERNAL_TABLES_BATCH
* @desc Creates external table sets for multiple configurations based on filter criteria. * @desc Creates external table sets for multiple configurations based on filter criteria.
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter
* (INBOX, ODS, ARCHIVE). Continues processing even if individual sets fail. * (INBOX, ODS, ARCHIVE, or ALL). Continues processing even if individual sets fail.
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operations.
* @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources) * @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources)
* @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types) * @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types)
* @param pTableId - Filter by TABLE_ID (NULL = all tables) * @param pTableId - Filter by TABLE_ID (NULL = all tables)
* @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist * @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist
* @example -- Create all external tables for C2D source * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
* @example -- Create only INBOX tables for C2D source
* BEGIN * BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
* pSourceKey => 'C2D', * pSourceKey => 'C2D',
* pRecreate => FALSE * pArea => 'INBOX'
* ); * );
* END; * END;
* *
* -- Recreate all external tables for all sources * -- Create all external tables for all sources with grant preservation
* BEGIN * BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
* pRecreate => TRUE * pRecreate => TRUE,
* pRestoreGrants => TRUE,
* pArea => 'ALL'
* ); * );
* END; * END;
* @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1 * @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1
@@ -652,7 +677,9 @@ AS
pSourceKey IN VARCHAR2 DEFAULT NULL, pSourceKey IN VARCHAR2 DEFAULT NULL,
pSourceFileId IN VARCHAR2 DEFAULT NULL, pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
); );
--------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------

View File

@@ -56,6 +56,120 @@ AS
RAISE; RAISE;
END CREATE_EXTERNAL_TABLE; END CREATE_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
/**
* CREATE_EXTERNAL_TABLES_SET - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET
*/
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
)
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vRecreateStr VARCHAR2(10);
vRestoreGrantsStr VARCHAR2(10);
BEGIN
-- Convert BOOLEAN to VARCHAR2 for logging
vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END;
vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END;
-- Log the start of the procedure
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(
SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => ' || pSourceFileConfigKey,
'pRecreate => ' || vRecreateStr,
'pRestoreGrants => ' || vRestoreGrantsStr,
'pArea => ''' || pArea || ''''
)
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
-- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET procedure
-- This ensures all logic remains centralized in the original package
-- and ODS wrapper simply delegates execution with DEFINER rights
CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
pSourceFileConfigKey => pSourceFileConfigKey,
pRecreate => pRecreate,
pRestoreGrants => pRestoreGrants,
pArea => pArea
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET: ' || SQLERRM,
'ERROR',
vParameters
);
RAISE;
END CREATE_EXTERNAL_TABLES_SET;
----------------------------------------------------------------------------------------------------
/**
* CREATE_EXTERNAL_TABLES_BATCH - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH
*/
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
pSourceKey IN VARCHAR2 DEFAULT NULL,
pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
)
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vRecreateStr VARCHAR2(10);
vRestoreGrantsStr VARCHAR2(10);
BEGIN
-- Convert BOOLEAN to VARCHAR2 for logging
vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END;
vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END;
-- Log the start of the procedure
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(
SYS.ODCIVARCHAR2LIST(
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
'pRecreate => ' || vRecreateStr,
'pRestoreGrants => ' || vRestoreGrantsStr,
'pArea => ''' || pArea || ''''
)
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
-- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH procedure
-- This ensures all logic remains centralized in the original package
-- and ODS wrapper simply delegates execution with DEFINER rights
CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
pSourceKey => pSourceKey,
pSourceFileId => pSourceFileId,
pTableId => pTableId,
pRecreate => pRecreate,
pRestoreGrants => pRestoreGrants,
pArea => pArea
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH: ' || SQLERRM,
'ERROR',
vParameters
);
RAISE;
END CREATE_EXTERNAL_TABLES_BATCH;
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------

View File

@@ -16,12 +16,15 @@ AS
*/ */
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.1.0'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.4.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-24 11:50:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'2.4.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) ||
'2.3.0 (2026-02-18): MARS-1057 - Added pRestoreGrants parameter support for grant preservation during table recreate' || CHR(13)||CHR(10) ||
'2.2.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers for batch external table creation' || CHR(13)||CHR(10) ||
'2.1.0 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) || '2.1.0 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
'1.5.0 (2025-10-12): Enhanced external table creation with official path patterns support' || CHR(13)||CHR(10) || '1.5.0 (2025-10-12): Enhanced external table creation with official path patterns support' || CHR(13)||CHR(10) ||
@@ -60,6 +63,76 @@ AS
pEncoding IN VARCHAR2 DEFAULT NULL pEncoding IN VARCHAR2 DEFAULT NULL
); );
/**
* @name CREATE_EXTERNAL_TABLES_SET
* @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table.
* Automatically generates table names and paths following official path patterns.
* Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET with DEFINER rights.
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operation.
* The pArea parameter allows selective table creation.
* @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table
* @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
* @example -- Create only INBOX table
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET(
* pSourceFileConfigKey => 123,
* pArea => 'INBOX'
* );
*
* -- Create all tables with grant preservation
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET(
* pSourceFileConfigKey => 123,
* pRecreate => TRUE,
* pRestoreGrants => TRUE,
* pArea => 'ALL'
* );
* @ex_rslt Creates external table(s) in ODS schema based on pArea parameter
*/
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
);
/**
* @name CREATE_EXTERNAL_TABLES_BATCH
* @desc Creates external table sets for multiple configurations based on filter criteria.
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter.
* Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH with DEFINER rights.
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operations.
* @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources)
* @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types)
* @param pTableId - Filter by TABLE_ID (NULL = all tables)
* @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
* @example -- Create only INBOX tables for C2D source
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH(
* pSourceKey => 'C2D',
* pArea => 'INBOX'
* );
*
* -- Create all tables with grant preservation
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH(
* pRecreate => TRUE,
* pRestoreGrants => TRUE,
* pArea => 'ALL'
* );
* @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1
*/
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
pSourceKey IN VARCHAR2 DEFAULT NULL,
pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
);
--------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS -- PACKAGE VERSION MANAGEMENT FUNCTIONS
--------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------
@@ -95,8 +168,8 @@ AS
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
* @example SELECT FILE_MANAGER_ODS.GET_BUILD_INFO() FROM DUAL; * @example SELECT FILE_MANAGER_ODS.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: FILE_MANAGER_ODS * @ex_rslt Package: FILE_MANAGER_ODS
* Version: 2.1.0 * Version: 2.2.0
* Build Date: 2025-11-24 11:50:00 * Build Date: 2025-11-27 15:00:00
* Author: Grzegorz Michalski * Author: Grzegorz Michalski
**/ **/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2; FUNCTION GET_BUILD_INFO RETURN VARCHAR2;