feat(FILE_MANAGER): Add CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation with pArea parameter
This commit is contained in:
Grzegorz Michalski
@@ -2051,6 +2051,394 @@ AS
|
||||
cgBL || 'Check A_SOURCE_FILE_RECEIVED and A_SOURCE_FILE_CONFIG tables for data integrity.';
|
||||
END ANALYZE_VALIDATION_ERRORS;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- EXTERNAL TABLE BATCH OPERATIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
|
||||
pSourceFileConfigKey IN NUMBER,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
-- Type for storing grant information
|
||||
TYPE tGrantRecord IS RECORD (
|
||||
grantee VARCHAR2(128),
|
||||
privilege VARCHAR2(40),
|
||||
grantable VARCHAR2(3)
|
||||
);
|
||||
TYPE tGrantList IS TABLE OF tGrantRecord;
|
||||
|
||||
vInboxGrants tGrantList;
|
||||
vOdsGrants tGrantList;
|
||||
vArchiveGrants tGrantList;
|
||||
|
||||
vSourceKey VARCHAR2(50);
|
||||
vSourceFileId VARCHAR2(100);
|
||||
vTableId VARCHAR2(100);
|
||||
vTemplateTableName VARCHAR2(200);
|
||||
vEncoding VARCHAR2(50);
|
||||
vDelimiter VARCHAR2(10);
|
||||
|
||||
vInboxTableName VARCHAR2(200);
|
||||
vOdsTableName VARCHAR2(200);
|
||||
vArchiveTableName VARCHAR2(200);
|
||||
|
||||
vInboxPrefix VARCHAR2(500);
|
||||
vOdsPrefix VARCHAR2(500);
|
||||
vArchivePrefix VARCHAR2(500);
|
||||
|
||||
vParameters VARCHAR2(4000);
|
||||
vAreaUpper VARCHAR2(20);
|
||||
|
||||
-- Nested procedure to save table grants before DROP
|
||||
PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants for table: ' || pTableName, 'DEBUG');
|
||||
|
||||
SELECT grantee, privilege, grantable
|
||||
BULK COLLECT INTO pGrantList
|
||||
FROM ALL_TAB_PRIVS
|
||||
WHERE table_schema = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
|
||||
AND table_name = pTableName
|
||||
AND grantee NOT IN ('SYS', 'SYSTEM', 'PUBLIC') -- Exclude system accounts
|
||||
ORDER BY grantee, privilege;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Saved ' || pGrantList.COUNT || ' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName,
|
||||
'INFO'
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
pGrantList := tGrantList(); -- Empty list
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('No grants found for table: ' || pTableName, 'INFO');
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Warning: Could not save grants for ' || pTableName || ': ' || SQLERRM,
|
||||
'WARNING'
|
||||
);
|
||||
pGrantList := tGrantList(); -- Empty list on error
|
||||
END SAVE_GRANTS;
|
||||
|
||||
-- Nested procedure to restore table grants after CREATE
|
||||
PROCEDURE RESTORE_GRANTS(pTableName VARCHAR2, pGrantList tGrantList) IS
|
||||
vGrantSQL VARCHAR2(500);
|
||||
vGrantCount NUMBER := 0;
|
||||
vFailCount NUMBER := 0;
|
||||
BEGIN
|
||||
IF pGrantList IS NULL OR pGrantList.COUNT = 0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'No grants to restore for table: ' || pTableName,
|
||||
'INFO'
|
||||
);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restoring ' || pGrantList.COUNT || ' grants for table: ' || pTableName,
|
||||
'DEBUG'
|
||||
);
|
||||
|
||||
FOR i IN 1..pGrantList.COUNT LOOP
|
||||
BEGIN
|
||||
vGrantSQL := 'GRANT ' || pGrantList(i).privilege ||
|
||||
' ON ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
|
||||
' TO ' || pGrantList(i).grantee;
|
||||
|
||||
IF pGrantList(i).grantable = 'YES' THEN
|
||||
vGrantSQL := vGrantSQL || ' WITH GRANT OPTION';
|
||||
END IF;
|
||||
|
||||
EXECUTE IMMEDIATE vGrantSQL;
|
||||
vGrantCount := vGrantCount + 1;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restored grant: ' || pGrantList(i).privilege ||
|
||||
' TO ' || pGrantList(i).grantee ||
|
||||
CASE WHEN pGrantList(i).grantable = 'YES' THEN ' WITH GRANT OPTION' ELSE '' END,
|
||||
'DEBUG'
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vFailCount := vFailCount + 1;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Warning: Could not restore grant (' || pGrantList(i).privilege ||
|
||||
' TO ' || pGrantList(i).grantee || ') on ' || pTableName || ': ' || SQLERRM,
|
||||
'WARNING'
|
||||
);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restored ' || vGrantCount || ' of ' || pGrantList.COUNT ||
|
||||
' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
|
||||
CASE WHEN vFailCount > 0 THEN ' (' || vFailCount || ' failed)' ELSE '' END,
|
||||
'INFO'
|
||||
);
|
||||
END RESTORE_GRANTS;
|
||||
|
||||
PROCEDURE DROP_IF_EXISTS(pTableName VARCHAR2) IS
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to drop table: ' || pTableName, 'DEBUG');
|
||||
EXECUTE IMMEDIATE 'DROP TABLE ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Table dropped successfully: ' || pTableName, 'INFO');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -942 THEN -- ORA-00942: table or view does not exist
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Table does not exist, skipping drop: ' || pTableName, 'INFO');
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING');
|
||||
RAISE; -- Re-raise if not "table not exists" error
|
||||
END IF;
|
||||
END DROP_IF_EXISTS;
|
||||
|
||||
BEGIN
|
||||
-- Validate and normalize pArea parameter
|
||||
vAreaUpper := UPPER(TRIM(pArea));
|
||||
|
||||
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
|
||||
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
|
||||
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
|
||||
END IF;
|
||||
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'),
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pArea => ''' || vAreaUpper || ''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
|
||||
|
||||
-- 1. Retrieve configuration from A_SOURCE_FILE_CONFIG
|
||||
BEGIN
|
||||
SELECT A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID, TEMPLATE_TABLE_NAME,
|
||||
NVL(ENCODING, 'UTF8')
|
||||
INTO vSourceKey, vSourceFileId, vTableId, vTemplateTableName,
|
||||
vEncoding
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
-- Set default delimiter (column DELIMITER does not exist in A_SOURCE_FILE_CONFIG)
|
||||
vDelimiter := ',';
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
vgMsgTmp := 'Source file config not found: ' || pSourceFileConfigKey;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20001, vgMsgTmp);
|
||||
END;
|
||||
|
||||
-- 2. Generate table names
|
||||
vInboxTableName := vTableId || '_INBOX';
|
||||
vOdsTableName := vTableId || '_ODS';
|
||||
vArchiveTableName := vTableId || '_ARCHIVE';
|
||||
|
||||
-- 3. Generate paths (OFFICIAL PATH PATTERNS)
|
||||
vInboxPrefix := 'INBOX/' || vSourceKey || '/' || vSourceFileId || '/' || vTableId;
|
||||
vOdsPrefix := 'ODS/' || vSourceKey || '/' || vTableId;
|
||||
vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId ||
|
||||
' (Area: ' || vAreaUpper || ')',
|
||||
'INFO'
|
||||
);
|
||||
|
||||
-- 4. DROP existing tables if pRecreate = TRUE
|
||||
IF pRecreate THEN
|
||||
-- Save grants before dropping tables (if pRestoreGrants = TRUE)
|
||||
IF pRestoreGrants THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO');
|
||||
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
SAVE_GRANTS(vInboxTableName, vInboxGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
SAVE_GRANTS(vOdsTableName, vOdsGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
SAVE_GRANTS(vArchiveTableName, vArchiveGrants);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Drop existing tables based on pArea
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vInboxTableName);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vOdsTableName);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vArchiveTableName);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- 5. Create INBOX external table (if requested)
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vInboxTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vInboxPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 6. Create ODS external table (if requested)
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vOdsTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vOdsPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvDataBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 7. Create ARCHIVE external table (if requested)
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vArchiveTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vArchivePrefix,
|
||||
pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE)
|
||||
IF pRecreate AND pRestoreGrants THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO');
|
||||
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
RESTORE_GRANTS(vInboxTableName, vInboxGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
RESTORE_GRANTS(vOdsTableName, vOdsGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
RESTORE_GRANTS(vArchiveTableName, vArchiveGrants);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' ||
|
||||
pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO',
|
||||
vParameters
|
||||
);
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vgMsgTmp := 'Error creating external tables for config ' || pSourceFileConfigKey || ': ' || SQLERRM;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20002, vgMsgTmp);
|
||||
END CREATE_EXTERNAL_TABLES_SET;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
|
||||
pSourceKey IN VARCHAR2 DEFAULT NULL,
|
||||
pSourceFileId IN VARCHAR2 DEFAULT NULL,
|
||||
pTableId IN VARCHAR2 DEFAULT NULL,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
vCount NUMBER := 0;
|
||||
vProcessed NUMBER := 0;
|
||||
vFailed NUMBER := 0;
|
||||
vParameters VARCHAR2(4000);
|
||||
vAreaUpper VARCHAR2(20);
|
||||
BEGIN
|
||||
-- Validate and normalize pArea parameter
|
||||
vAreaUpper := UPPER(TRIM(pArea));
|
||||
|
||||
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
|
||||
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
|
||||
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
|
||||
END IF;
|
||||
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
|
||||
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
|
||||
'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pArea => ''' || vAreaUpper || ''''
|
||||
));
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
|
||||
|
||||
-- Iterate over configurations matching criteria (only INPUT files)
|
||||
FOR rec IN (
|
||||
SELECT A_SOURCE_FILE_CONFIG_KEY, A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND (pSourceKey IS NULL OR A_SOURCE_KEY = pSourceKey)
|
||||
AND (pSourceFileId IS NULL OR SOURCE_FILE_ID = pSourceFileId)
|
||||
AND (pTableId IS NULL OR TABLE_ID = pTableId)
|
||||
ORDER BY A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID
|
||||
) LOOP
|
||||
vCount := vCount + 1;
|
||||
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' ||
|
||||
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO'
|
||||
);
|
||||
|
||||
-- Call procedure to create set of tables (based on pArea)
|
||||
CREATE_EXTERNAL_TABLES_SET(
|
||||
pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY,
|
||||
pRecreate => pRecreate,
|
||||
pRestoreGrants => pRestoreGrants,
|
||||
pArea => vAreaUpper
|
||||
);
|
||||
|
||||
vProcessed := vProcessed + 1;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vFailed := vFailed + 1;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Failed to create tables for config ' || rec.A_SOURCE_FILE_CONFIG_KEY ||
|
||||
' (' || rec.A_SOURCE_KEY || '/' || rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || '): ' || SQLERRM,
|
||||
'ERROR'
|
||||
);
|
||||
-- Continue processing despite error
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount ||
|
||||
', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO',
|
||||
vParameters
|
||||
);
|
||||
|
||||
IF vFailed > 0 THEN
|
||||
vgMsgTmp := 'Batch completed with errors. Processed: ' || vProcessed || ', Failed: ' || vFailed;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'WARNING', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20003, vgMsgTmp);
|
||||
END IF;
|
||||
|
||||
END CREATE_EXTERNAL_TABLES_BATCH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user