diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkb b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkb index 1d99317..f1b7712 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkb +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkb @@ -1968,9 +1968,23 @@ AS PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, - pRecreate IN BOOLEAN DEFAULT FALSE + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS + -- Type for storing grant information + TYPE tGrantRecord IS RECORD ( + grantee VARCHAR2(128), + privilege VARCHAR2(40), + grantable VARCHAR2(3) + ); + TYPE tGrantList IS TABLE OF tGrantRecord; + + vInboxGrants tGrantList; + vOdsGrants tGrantList; + vArchiveGrants tGrantList; + vSourceKey VARCHAR2(50); vSourceFileId VARCHAR2(100); vTableId VARCHAR2(100); @@ -1986,28 +2000,124 @@ AS vOdsPrefix VARCHAR2(500); vArchivePrefix VARCHAR2(500); - vTableExists NUMBER; vParameters VARCHAR2(4000); + vAreaUpper VARCHAR2(20); + + -- Nested procedure to save table grants before DROP + PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS + BEGIN + ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants for table: ' || pTableName, 'DEBUG'); + + SELECT grantee, privilege, grantable + BULK COLLECT INTO pGrantList + FROM ALL_TAB_PRIVS + WHERE table_schema = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') + AND table_name = pTableName + AND grantee NOT IN ('SYS', 'SYSTEM', 'PUBLIC') -- Exclude system accounts + ORDER BY grantee, privilege; + + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Saved ' || pGrantList.COUNT || ' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName, + 'INFO' + ); + EXCEPTION + WHEN NO_DATA_FOUND THEN + pGrantList := tGrantList(); -- Empty list + ENV_MANAGER.LOG_PROCESS_EVENT('No grants found for table: ' || pTableName, 'INFO'); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Warning: Could not save grants for ' || pTableName || ': ' || SQLERRM, + 'WARNING' + ); + pGrantList := tGrantList(); -- Empty list on error + END SAVE_GRANTS; + + -- Nested procedure to restore table grants after CREATE + PROCEDURE RESTORE_GRANTS(pTableName VARCHAR2, pGrantList tGrantList) IS + vGrantSQL VARCHAR2(500); + vGrantCount NUMBER := 0; + vFailCount NUMBER := 0; + BEGIN + IF pGrantList IS NULL OR pGrantList.COUNT = 0 THEN + ENV_MANAGER.LOG_PROCESS_EVENT( + 'No grants to restore for table: ' || pTableName, + 'INFO' + ); + RETURN; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Restoring ' || pGrantList.COUNT || ' grants for table: ' || pTableName, + 'DEBUG' + ); + + FOR i IN 1..pGrantList.COUNT LOOP + BEGIN + vGrantSQL := 'GRANT ' || pGrantList(i).privilege || + ' ON ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName || + ' TO ' || pGrantList(i).grantee; + + IF pGrantList(i).grantable = 'YES' THEN + vGrantSQL := vGrantSQL || ' WITH GRANT OPTION'; + END IF; + + EXECUTE IMMEDIATE vGrantSQL; + vGrantCount := vGrantCount + 1; + + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Restored grant: ' || pGrantList(i).privilege || + ' TO ' || pGrantList(i).grantee || + CASE WHEN pGrantList(i).grantable = 'YES' THEN ' WITH GRANT OPTION' ELSE '' END, + 'DEBUG' + ); + EXCEPTION + WHEN OTHERS THEN + vFailCount := vFailCount + 1; + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Warning: Could not restore grant (' || pGrantList(i).privilege || + ' TO ' || pGrantList(i).grantee || ') on ' || pTableName || ': ' || SQLERRM, + 'WARNING' + ); + END; + END LOOP; + + ENV_MANAGER.LOG_PROCESS_EVENT( + 'Restored ' || vGrantCount || ' of ' || pGrantList.COUNT || + ' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName || + CASE WHEN vFailCount > 0 THEN ' (' || vFailCount || ' failed)' ELSE '' END, + 'INFO' + ); + END RESTORE_GRANTS; PROCEDURE DROP_IF_EXISTS(pTableName VARCHAR2) IS BEGIN - SELECT COUNT(*) INTO vTableExists - FROM ALL_TABLES - WHERE OWNER = 'ODS' AND TABLE_NAME = pTableName; - - IF vTableExists > 0 THEN - ENV_MANAGER.LOG_PROCESS_EVENT('Dropping existing table: ' || pTableName, 'INFO'); - EXECUTE IMMEDIATE 'DROP TABLE ODS.' || pTableName; - END IF; + ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to drop table: ' || pTableName, 'DEBUG'); + EXECUTE IMMEDIATE 'DROP TABLE ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName; + ENV_MANAGER.LOG_PROCESS_EVENT('Table dropped successfully: ' || pTableName, 'INFO'); EXCEPTION WHEN OTHERS THEN - ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING'); + IF SQLCODE = -942 THEN -- ORA-00942: table or view does not exist + ENV_MANAGER.LOG_PROCESS_EVENT('Table does not exist, skipping drop: ' || pTableName, 'INFO'); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING'); + RAISE; -- Re-raise if not "table not exists" error + END IF; END DROP_IF_EXISTS; BEGIN + -- Validate and normalize pArea parameter + vAreaUpper := UPPER(TRIM(pArea)); + + IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN + vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL'; + RAISE_APPLICATION_ERROR(-20010, vgMsgTmp); + END IF; + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'), - 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END + 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END, + 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END, + 'pArea => ''' || vAreaUpper || '''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); @@ -2017,7 +2127,7 @@ AS NVL(ENCODING, 'UTF8') INTO vSourceKey, vSourceFileId, vTableId, vTemplateTableName, vEncoding - FROM A_SOURCE_FILE_CONFIG + FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; -- Set default delimiter (column DELIMITER does not exist in A_SOURCE_FILE_CONFIG) @@ -2030,9 +2140,9 @@ AS END; -- 2. Generate table names - vInboxTableName := vSourceKey || '_' || vTableId || '_INBOX'; - vOdsTableName := vSourceKey || '_' || vTableId || '_ODS'; - vArchiveTableName := vSourceKey || '_' || vTableId || '_ARCHIVE'; + vInboxTableName := vTableId || '_INBOX'; + vOdsTableName := vTableId || '_ODS'; + vArchiveTableName := vTableId || '_ARCHIVE'; -- 3. Generate paths (OFFICIAL PATH PATTERNS) vInboxPrefix := 'INBOX/' || vSourceKey || '/' || vSourceFileId || '/' || vTableId; @@ -2040,52 +2150,103 @@ AS vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId; ENV_MANAGER.LOG_PROCESS_EVENT( - 'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId, + 'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId || + ' (Area: ' || vAreaUpper || ')', 'INFO' ); -- 4. DROP existing tables if pRecreate = TRUE IF pRecreate THEN - DROP_IF_EXISTS(vInboxTableName); - DROP_IF_EXISTS(vOdsTableName); - DROP_IF_EXISTS(vArchiveTableName); + -- Save grants before dropping tables (if pRestoreGrants = TRUE) + IF pRestoreGrants THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO'); + + IF vAreaUpper IN ('INBOX', 'ALL') THEN + SAVE_GRANTS(vInboxTableName, vInboxGrants); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + SAVE_GRANTS(vOdsTableName, vOdsGrants); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + SAVE_GRANTS(vArchiveTableName, vArchiveGrants); + END IF; + END IF; + + -- Drop existing tables based on pArea + IF vAreaUpper IN ('INBOX', 'ALL') THEN + DROP_IF_EXISTS(vInboxTableName); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + DROP_IF_EXISTS(vOdsTableName); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + DROP_IF_EXISTS(vArchiveTableName); + END IF; END IF; - -- 5. Create INBOX external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO'); - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => vInboxTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vInboxPrefix, - pBucketUri => ENV_MANAGER.gvInboxBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 5. Create INBOX external table (if requested) + IF vAreaUpper IN ('INBOX', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vInboxTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vInboxPrefix, + pBucketUri => ENV_MANAGER.gvInboxBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; - -- 6. Create ODS external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO'); - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => vOdsTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vOdsPrefix, - pBucketUri => ENV_MANAGER.gvDataBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 6. Create ODS external table (if requested) + IF vAreaUpper IN ('ODS', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vOdsTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vOdsPrefix, + pBucketUri => ENV_MANAGER.gvDataBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; - -- 7. Create ARCHIVE external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO'); - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => vArchiveTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vArchivePrefix, - pBucketUri => ENV_MANAGER.gvArchiveBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 7. Create ARCHIVE external table (if requested) + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vArchiveTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vArchivePrefix, + pBucketUri => ENV_MANAGER.gvArchiveBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; + + -- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE) + IF pRecreate AND pRestoreGrants THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO'); + + IF vAreaUpper IN ('INBOX', 'ALL') THEN + RESTORE_GRANTS(vInboxTableName, vInboxGrants); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + RESTORE_GRANTS(vOdsTableName, vOdsGrants); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + RESTORE_GRANTS(vArchiveTableName, vArchiveGrants); + END IF; + END IF; ENV_MANAGER.LOG_PROCESS_EVENT( - 'End CREATE_EXTERNAL_TABLES_SET - Successfully created all 3 external tables for config: ' || pSourceFileConfigKey, + 'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' || + pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')', 'INFO', vParameters ); @@ -2103,19 +2264,32 @@ AS pSourceKey IN VARCHAR2 DEFAULT NULL, pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, - pRecreate IN BOOLEAN DEFAULT FALSE + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS vCount NUMBER := 0; vProcessed NUMBER := 0; vFailed NUMBER := 0; vParameters VARCHAR2(4000); + vAreaUpper VARCHAR2(20); BEGIN + -- Validate and normalize pArea parameter + vAreaUpper := UPPER(TRIM(pArea)); + + IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN + vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL'; + RAISE_APPLICATION_ERROR(-20010, vgMsgTmp); + END IF; + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''', 'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''', 'pTableId => ''' || NVL(pTableId, 'NULL') || '''', - 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END + 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END, + 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END, + 'pArea => ''' || vAreaUpper || '''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); @@ -2123,7 +2297,7 @@ AS -- Iterate over configurations matching criteria (only INPUT files) FOR rec IN ( SELECT A_SOURCE_FILE_CONFIG_KEY, A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID - FROM A_SOURCE_FILE_CONFIG + FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' AND (pSourceKey IS NULL OR A_SOURCE_KEY = pSourceKey) AND (pSourceFileId IS NULL OR SOURCE_FILE_ID = pSourceFileId) @@ -2135,14 +2309,16 @@ AS BEGIN ENV_MANAGER.LOG_PROCESS_EVENT( 'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' || - rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID, + rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')', 'INFO' ); - -- Call procedure to create set of 3 tables + -- Call procedure to create set of tables (based on pArea) CREATE_EXTERNAL_TABLES_SET( pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY, - pRecreate => pRecreate + pRecreate => pRecreate, + pRestoreGrants => pRestoreGrants, + pArea => vAreaUpper ); vProcessed := vProcessed + 1; @@ -2161,7 +2337,7 @@ AS ENV_MANAGER.LOG_PROCESS_EVENT( 'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount || - ', Processed: ' || vProcessed || ', Failed: ' || vFailed, + ', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')', 'INFO', vParameters ); diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkg b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkg index 90d69dd..adf8d28 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkg +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkg @@ -17,12 +17,13 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.5.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) || '3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) || '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) || '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) || @@ -602,48 +603,72 @@ AS /** * @name CREATE_EXTERNAL_TABLES_SET - * @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration - * from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following - * official path patterns. Optionally drops and recreates existing tables. + * @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table. + * Automatically generates table names and paths following official path patterns. + * Optionally drops and recreates existing tables. If pRestoreGrants is TRUE, saves and restores table grants. + * The pArea parameter allows selective table creation. * @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table * @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist - * @example BEGIN + * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) + * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * 'INBOX' - creates only INBOX table + * 'ODS' - creates only ODS table + * 'ARCHIVE' - creates only ARCHIVE table + * 'ALL' - creates all three tables (default) + * @example -- Create only INBOX table + * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( * pSourceFileConfigKey => 123, - * pRecreate => FALSE + * pArea => 'INBOX' * ); * END; - * @ex_rslt Creates three external tables in ODS schema: - * - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX - * - C2D_A_UC_DISSEM_METADATA_LOADS_ODS - * - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE + * + * -- Create all tables with grant preservation + * BEGIN + * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( + * pSourceFileConfigKey => 123, + * pRecreate => TRUE, + * pRestoreGrants => TRUE, + * pArea => 'ALL' + * ); + * END; + * @ex_rslt Creates external table(s) in ODS schema based on pArea parameter **/ PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, - pRecreate IN BOOLEAN DEFAULT FALSE + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); /** * @name CREATE_EXTERNAL_TABLES_BATCH * @desc Creates external table sets for multiple configurations based on filter criteria. - * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration - * (INBOX, ODS, ARCHIVE). Continues processing even if individual sets fail. + * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter + * (INBOX, ODS, ARCHIVE, or ALL). Continues processing even if individual sets fail. + * If pRestoreGrants is TRUE, saves and restores table grants during recreate operations. * @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources) * @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types) * @param pTableId - Filter by TABLE_ID (NULL = all tables) * @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist - * @example -- Create all external tables for C2D source + * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) + * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX tables for C2D source * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( * pSourceKey => 'C2D', - * pRecreate => FALSE + * pArea => 'INBOX' * ); * END; * - * -- Recreate all external tables for all sources + * -- Create all external tables for all sources with grant preservation * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - * pRecreate => TRUE + * pRecreate => TRUE, + * pRestoreGrants => TRUE, + * pArea => 'ALL' * ); * END; * @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1 @@ -652,7 +677,9 @@ AS pSourceKey IN VARCHAR2 DEFAULT NULL, pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, - pRecreate IN BOOLEAN DEFAULT FALSE + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); --------------------------------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkb b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkb index 532eec2..b147983 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkb +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkb @@ -56,6 +56,120 @@ AS RAISE; END CREATE_EXTERNAL_TABLE; + ---------------------------------------------------------------------------------------------------- + + /** + * CREATE_EXTERNAL_TABLES_SET - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET + */ + PROCEDURE CREATE_EXTERNAL_TABLES_SET ( + pSourceFileConfigKey IN NUMBER, + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vRecreateStr VARCHAR2(10); + vRestoreGrantsStr VARCHAR2(10); + BEGIN + -- Convert BOOLEAN to VARCHAR2 for logging + vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END; + vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END; + + -- Log the start of the procedure + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS( + SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => ' || pSourceFileConfigKey, + 'pRecreate => ' || vRecreateStr, + 'pRestoreGrants => ' || vRestoreGrantsStr, + 'pArea => ''' || pArea || '''' + ) + ); + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); + + -- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET procedure + -- This ensures all logic remains centralized in the original package + -- and ODS wrapper simply delegates execution with DEFINER rights + CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( + pSourceFileConfigKey => pSourceFileConfigKey, + pRecreate => pRecreate, + pRestoreGrants => pRestoreGrants, + pArea => pArea + ); + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET: ' || SQLERRM, + 'ERROR', + vParameters + ); + RAISE; + END CREATE_EXTERNAL_TABLES_SET; + + ---------------------------------------------------------------------------------------------------- + + /** + * CREATE_EXTERNAL_TABLES_BATCH - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH + */ + PROCEDURE CREATE_EXTERNAL_TABLES_BATCH ( + pSourceKey IN VARCHAR2 DEFAULT NULL, + pSourceFileId IN VARCHAR2 DEFAULT NULL, + pTableId IN VARCHAR2 DEFAULT NULL, + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vRecreateStr VARCHAR2(10); + vRestoreGrantsStr VARCHAR2(10); + BEGIN + -- Convert BOOLEAN to VARCHAR2 for logging + vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END; + vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END; + + -- Log the start of the procedure + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS( + SYS.ODCIVARCHAR2LIST( + 'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''', + 'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''', + 'pTableId => ''' || NVL(pTableId, 'NULL') || '''', + 'pRecreate => ' || vRecreateStr, + 'pRestoreGrants => ' || vRestoreGrantsStr, + 'pArea => ''' || pArea || '''' + ) + ); + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); + + -- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH procedure + -- This ensures all logic remains centralized in the original package + -- and ODS wrapper simply delegates execution with DEFINER rights + CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( + pSourceKey => pSourceKey, + pSourceFileId => pSourceFileId, + pTableId => pTableId, + pRecreate => pRecreate, + pRestoreGrants => pRestoreGrants, + pArea => pArea + ); + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH: ' || SQLERRM, + 'ERROR', + vParameters + ); + RAISE; + END CREATE_EXTERNAL_TABLES_BATCH; + ---------------------------------------------------------------------------------------------------- -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION ---------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkg b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkg index 0010af6..eea529d 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkg +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/ODS/FILE_MANAGER_ODS.pkg @@ -16,12 +16,15 @@ AS */ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.1.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-24 11:50:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.4.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '2.4.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) || + '2.3.0 (2026-02-18): MARS-1057 - Added pRestoreGrants parameter support for grant preservation during table recreate' || CHR(13)||CHR(10) || + '2.2.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers for batch external table creation' || CHR(13)||CHR(10) || '2.1.0 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) || '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || '1.5.0 (2025-10-12): Enhanced external table creation with official path patterns support' || CHR(13)||CHR(10) || @@ -60,6 +63,76 @@ AS pEncoding IN VARCHAR2 DEFAULT NULL ); + /** + * @name CREATE_EXTERNAL_TABLES_SET + * @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table. + * Automatically generates table names and paths following official path patterns. + * Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET with DEFINER rights. + * If pRestoreGrants is TRUE, saves and restores table grants during recreate operation. + * The pArea parameter allows selective table creation. + * @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table + * @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist + * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) + * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX table + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( + * pSourceFileConfigKey => 123, + * pArea => 'INBOX' + * ); + * + * -- Create all tables with grant preservation + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( + * pSourceFileConfigKey => 123, + * pRecreate => TRUE, + * pRestoreGrants => TRUE, + * pArea => 'ALL' + * ); + * @ex_rslt Creates external table(s) in ODS schema based on pArea parameter + */ + PROCEDURE CREATE_EXTERNAL_TABLES_SET ( + pSourceFileConfigKey IN NUMBER, + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' + ); + + /** + * @name CREATE_EXTERNAL_TABLES_BATCH + * @desc Creates external table sets for multiple configurations based on filter criteria. + * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter. + * Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH with DEFINER rights. + * If pRestoreGrants is TRUE, saves and restores table grants during recreate operations. + * @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources) + * @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types) + * @param pTableId - Filter by TABLE_ID (NULL = all tables) + * @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist + * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) + * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX tables for C2D source + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH( + * pSourceKey => 'C2D', + * pArea => 'INBOX' + * ); + * + * -- Create all tables with grant preservation + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH( + * pRecreate => TRUE, + * pRestoreGrants => TRUE, + * pArea => 'ALL' + * ); + * @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1 + */ + PROCEDURE CREATE_EXTERNAL_TABLES_BATCH ( + pSourceKey IN VARCHAR2 DEFAULT NULL, + pSourceFileId IN VARCHAR2 DEFAULT NULL, + pTableId IN VARCHAR2 DEFAULT NULL, + pRecreate IN BOOLEAN DEFAULT FALSE, + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' + ); + --------------------------------------------------------------------------------------------------------------------------- -- PACKAGE VERSION MANAGEMENT FUNCTIONS --------------------------------------------------------------------------------------------------------------------------- @@ -95,8 +168,8 @@ AS * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. * @example SELECT FILE_MANAGER_ODS.GET_BUILD_INFO() FROM DUAL; * @ex_rslt Package: FILE_MANAGER_ODS - * Version: 2.1.0 - * Build Date: 2025-11-24 11:50:00 + * Version: 2.2.0 + * Build Date: 2025-11-27 15:00:00 * Author: Grzegorz Michalski **/ FUNCTION GET_BUILD_INFO RETURN VARCHAR2;