admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(MARS-1057): Upgrade FILE_MANAGER to v3.5.0 with area filter functionality

Browse Source 
- Updated install script to reflect new version and features.
- Added pArea parameter to CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for selective table creation (INBOX/ODS/ARCHIVE/ALL).
- Enhanced grant preservation logic based on the pArea parameter.
- Updated FILE_MANAGER_ODS package to support new functionality.
- Modified rollback script to remove new features and revert to previous versions.
This commit is contained in:
Grzegorz Michalski
2026-02-18 14:03:11 +01:00
parent 204616252a
commit 2b116c0256
15 changed files with 253 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb
View File

@@ -1969,7 +1969,8 @@ AS
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
)
IS
-- Type for storing grant information
@@ -2001,6 +2002,7 @@ AS
vTableExists NUMBER;
vParameters VARCHAR2(4000);
vAreaUpper VARCHAR2(20);
-- Nested procedure to save table grants before DROP
PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS
@@ -2104,10 +2106,19 @@ AS
END DROP_IF_EXISTS;
BEGIN
-- Validate and normalize pArea parameter
vAreaUpper := UPPER(TRIM(pArea));
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
END IF;
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'),
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
'pArea => ''' || vAreaUpper || ''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
@@ -2140,7 +2151,8 @@ AS
vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId;
ENV_MANAGER.LOG_PROCESS_EVENT(
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId,
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId ||
' (Area: ' || vAreaUpper || ')',
'INFO'
);
@@ -2149,60 +2161,93 @@ AS
-- Save grants before dropping tables (if pRestoreGrants = TRUE)
IF pRestoreGrants THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO');
SAVE_GRANTS(vInboxTableName, vInboxGrants);
SAVE_GRANTS(vOdsTableName, vOdsGrants);
SAVE_GRANTS(vArchiveTableName, vArchiveGrants);
IF vAreaUpper IN ('INBOX', 'ALL') THEN
SAVE_GRANTS(vInboxTableName, vInboxGrants);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
SAVE_GRANTS(vOdsTableName, vOdsGrants);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
SAVE_GRANTS(vArchiveTableName, vArchiveGrants);
END IF;
END IF;
-- Drop existing tables
DROP_IF_EXISTS(vInboxTableName);
DROP_IF_EXISTS(vOdsTableName);
DROP_IF_EXISTS(vArchiveTableName);
-- Drop existing tables based on pArea
IF vAreaUpper IN ('INBOX', 'ALL') THEN
DROP_IF_EXISTS(vInboxTableName);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
DROP_IF_EXISTS(vOdsTableName);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
DROP_IF_EXISTS(vArchiveTableName);
END IF;
END IF;
-- 5. Create INBOX external table
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vInboxTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vInboxPrefix,
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
-- 5. Create INBOX external table (if requested)
IF vAreaUpper IN ('INBOX', 'ALL') THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vInboxTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vInboxPrefix,
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
END IF;
-- 6. Create ODS external table
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vOdsTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vOdsPrefix,
pBucketUri => ENV_MANAGER.gvDataBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
-- 6. Create ODS external table (if requested)
IF vAreaUpper IN ('ODS', 'ALL') THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vOdsTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vOdsPrefix,
pBucketUri => ENV_MANAGER.gvDataBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
END IF;
-- 7. Create ARCHIVE external table
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vArchiveTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vArchivePrefix,
pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
-- 7. Create ARCHIVE external table (if requested)
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
CREATE_EXTERNAL_TABLE(
pTableName => vArchiveTableName,
pTemplateTableName => vTemplateTableName,
pPrefix => vArchivePrefix,
pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
pDelimiter => vDelimiter,
pEncoding => vEncoding
);
END IF;
-- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE)
IF pRecreate AND pRestoreGrants THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO');
RESTORE_GRANTS(vInboxTableName, vInboxGrants);
RESTORE_GRANTS(vOdsTableName, vOdsGrants);
RESTORE_GRANTS(vArchiveTableName, vArchiveGrants);
IF vAreaUpper IN ('INBOX', 'ALL') THEN
RESTORE_GRANTS(vInboxTableName, vInboxGrants);
END IF;
IF vAreaUpper IN ('ODS', 'ALL') THEN
RESTORE_GRANTS(vOdsTableName, vOdsGrants);
END IF;
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
RESTORE_GRANTS(vArchiveTableName, vArchiveGrants);
END IF;
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT(
'End CREATE_EXTERNAL_TABLES_SET - Successfully created all 3 external tables for config: ' || pSourceFileConfigKey,
'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' ||
pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')',
'INFO',
vParameters
);
@@ -2221,20 +2266,31 @@ AS
pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE,
pRestoreGrants IN BOOLEAN DEFAULT TRUE
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
pArea IN VARCHAR2 DEFAULT 'ALL'
)
IS
vCount NUMBER := 0;
vProcessed NUMBER := 0;
vFailed NUMBER := 0;
vParameters VARCHAR2(4000);
vAreaUpper VARCHAR2(20);
BEGIN
-- Validate and normalize pArea parameter
vAreaUpper := UPPER(TRIM(pArea));
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
END IF;
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
'pArea => ''' || vAreaUpper || ''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
@@ -2254,15 +2310,16 @@ AS
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT(
'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' ||
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID,
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')',
'INFO'
);
-- Call procedure to create set of 3 tables
-- Call procedure to create set of tables (based on pArea)
CREATE_EXTERNAL_TABLES_SET(
pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY,
pRecreate => pRecreate,
pRestoreGrants => pRestoreGrants
pRestoreGrants => pRestoreGrants,
pArea => vAreaUpper
);
vProcessed := vProcessed + 1;
@@ -2281,7 +2338,7 @@ AS
ENV_MANAGER.LOG_PROCESS_EVENT(
'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount ||
', Processed: ' || vProcessed || ', Failed: ' || vFailed,
', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')',
'INFO',
vParameters
);