From 2b116c025651844c43f59d7c79e83f836acf99aa Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Wed, 18 Feb 2026 14:03:11 +0100 Subject: [PATCH] feat(MARS-1057): Upgrade FILE_MANAGER to v3.5.0 with area filter functionality - Updated install script to reflect new version and features. - Added pArea parameter to CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for selective table creation (INBOX/ODS/ARCHIVE/ALL). - Enhanced grant preservation logic based on the pArea parameter. - Updated FILE_MANAGER_ODS package to support new functionality. - Modified rollback script to remove new features and revert to previous versions. --- ...1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql | 10 +- ...1057_install_CT_MRDS_FILE_MANAGER_BODY.sql | 10 +- ...1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql | 10 +- ...1057_install_ODS_FILE_MANAGER_ODS_BODY.sql | 10 +- ...057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql | 10 +- ...057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql | 10 +- ...057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql | 4 +- ...057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql | 4 +- MARS_Packages/REL03/MARS-1057/README.md | 376 ------------------ .../REL03/MARS-1057/install_mars1057.sql | 37 +- .../MARS-1057/new_version/FILE_MANAGER.pkb | 157 +++++--- .../MARS-1057/new_version/FILE_MANAGER.pkg | 59 ++- .../new_version/FILE_MANAGER_ODS.pkb | 18 +- .../new_version/FILE_MANAGER_ODS.pkg | 54 +-- .../REL03/MARS-1057/rollback_mars1057.sql | 16 +- 15 files changed, 253 insertions(+), 532 deletions(-) delete mode 100644 MARS_Packages/REL03/MARS-1057/README.md diff --git a/MARS_Packages/REL03/MARS-1057/01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql b/MARS_Packages/REL03/MARS-1057/01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql index 0bdd884..8fdd96d 100644 --- a/MARS_Packages/REL03/MARS-1057/01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql +++ b/MARS_Packages/REL03/MARS-1057/01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql @@ -1,16 +1,16 @@ --============================================================================================================================= --- MARS-1057: Install FILE_MANAGER Package Specification v3.4.0 +-- MARS-1057: Install FILE_MANAGER Package Specification v3.5.0 --============================================================================================================================= --- Purpose: Deploy FILE_MANAGER package specification with new batch external table creation procedures +-- Purpose: Deploy FILE_MANAGER package specification with batch external table creation and area filter functionality -- Author: Grzegorz Michalski --- Date: 2025-11-27 --- Related: MARS-1057 Batch External Table Creation +-- Date: 2026-02-18 +-- Related: MARS-1057 Batch External Table Creation + Area Filter Enhancement --============================================================================================================================= SET SERVEROUTPUT ON PROMPT ======================================================================== -PROMPT Installing FILE_MANAGER Package Specification v3.4.0 +PROMPT Installing FILE_MANAGER Package Specification v3.5.0 PROMPT ======================================================================== @@new_version/FILE_MANAGER.pkg diff --git a/MARS_Packages/REL03/MARS-1057/02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql b/MARS_Packages/REL03/MARS-1057/02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql index 2f4e22f..05eac2b 100644 --- a/MARS_Packages/REL03/MARS-1057/02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql +++ b/MARS_Packages/REL03/MARS-1057/02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql @@ -1,16 +1,16 @@ --============================================================================================================================= --- MARS-1057: Install FILE_MANAGER Package Body v3.4.0 +-- MARS-1057: Install FILE_MANAGER Package Body v3.5.0 --============================================================================================================================= --- Purpose: Deploy FILE_MANAGER package body with implementation of batch external table creation procedures +-- Purpose: Deploy FILE_MANAGER package body with batch external table creation and area filter functionality -- Author: Grzegorz Michalski --- Date: 2025-11-27 --- Related: MARS-1057 Batch External Table Creation +-- Date: 2026-02-18 +-- Related: MARS-1057 Batch External Table Creation + Area Filter Enhancement --============================================================================================================================= SET SERVEROUTPUT ON PROMPT ======================================================================== -PROMPT Installing FILE_MANAGER Package Body v3.4.0 +PROMPT Installing FILE_MANAGER Package Body v3.5.0 PROMPT ======================================================================== @@new_version/FILE_MANAGER.pkb diff --git a/MARS_Packages/REL03/MARS-1057/03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql b/MARS_Packages/REL03/MARS-1057/03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql index 723c316..e527944 100644 --- a/MARS_Packages/REL03/MARS-1057/03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql +++ b/MARS_Packages/REL03/MARS-1057/03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql @@ -1,17 +1,17 @@ -- =================================================================== -- MARS-1057: Install ODS.FILE_MANAGER_ODS Package Specification -- =================================================================== --- Purpose: Deploy FILE_MANAGER_ODS package specification v2.2.0 +-- Purpose: Deploy FILE_MANAGER_ODS package specification v2.4.0 -- Author: Grzegorz Michalski --- Date: 2025-11-27 +-- Date: 2026-02-18 -- Package: ODS.FILE_MANAGER_ODS --- Version: 2.1.0 -> 2.2.0 --- Changes: Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers +-- Version: 2.1.0 -> 2.4.0 +-- Changes: Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers with pArea and pRestoreGrants parameters SET ECHO ON SET DEFINE OFF -PROMPT Installing ODS.FILE_MANAGER_ODS Package Specification v2.2.0... +PROMPT Installing ODS.FILE_MANAGER_ODS Package Specification v2.4.0... @@new_version/FILE_MANAGER_ODS.pkg diff --git a/MARS_Packages/REL03/MARS-1057/04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql b/MARS_Packages/REL03/MARS-1057/04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql index ac6218e..87ed4e4 100644 --- a/MARS_Packages/REL03/MARS-1057/04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql +++ b/MARS_Packages/REL03/MARS-1057/04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql @@ -1,17 +1,17 @@ -- =================================================================== -- MARS-1057: Install ODS.FILE_MANAGER_ODS Package Body -- =================================================================== --- Purpose: Deploy FILE_MANAGER_ODS package body v2.2.0 +-- Purpose: Deploy FILE_MANAGER_ODS package body v2.4.0 -- Author: Grzegorz Michalski --- Date: 2025-11-27 +-- Date: 2026-02-18 -- Package: ODS.FILE_MANAGER_ODS --- Version: 2.1.0 -> 2.2.0 --- Changes: Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrapper implementations +-- Version: 2.1.0 -> 2.4.0 +-- Changes: Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrapper implementations with pArea and pRestoreGrants parameters SET ECHO ON SET DEFINE OFF -PROMPT Installing ODS.FILE_MANAGER_ODS Package Body v2.2.0... +PROMPT Installing ODS.FILE_MANAGER_ODS Package Body v2.4.0... @@new_version/FILE_MANAGER_ODS.pkb diff --git a/MARS_Packages/REL03/MARS-1057/91_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql b/MARS_Packages/REL03/MARS-1057/91_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql index f5f2fb7..24cde43 100644 --- a/MARS_Packages/REL03/MARS-1057/91_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql +++ b/MARS_Packages/REL03/MARS-1057/91_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql @@ -1,19 +1,19 @@ --============================================================================================================================= -- MARS-1057: Rollback FILE_MANAGER Package Specification to v3.3.0 --============================================================================================================================= --- Purpose: Restore FILE_MANAGER package specification to version before MARS-1057 changes +-- Purpose: Restore FILE_MANAGER package specification to version before MARS-1057 changes (from v3.5.0 to v3.3.0) -- Author: Grzegorz Michalski --- Date: 2025-11-27 --- Related: MARS-1057 Batch External Table Creation (ROLLBACK) +-- Date: 2026-02-18 +-- Related: MARS-1057 Batch External Table Creation + Area Filter (ROLLBACK) --============================================================================================================================= SET SERVEROUTPUT ON PROMPT ======================================================================== -PROMPT Rolling back FILE_MANAGER Package Specification to v3.3.0 +PROMPT Rolling back FILE_MANAGER Package Specification from v3.5.0 to v3.3.0 PROMPT ======================================================================== -@@current_version/FILE_MANAGER.pkg +@@rollback_version/FILE_MANAGER.pkg -- Verify compilation status (check specific schema when installing as ADMIN) SELECT object_name, object_type, status diff --git a/MARS_Packages/REL03/MARS-1057/92_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql b/MARS_Packages/REL03/MARS-1057/92_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql index b4291e8..48f363b 100644 --- a/MARS_Packages/REL03/MARS-1057/92_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql +++ b/MARS_Packages/REL03/MARS-1057/92_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql @@ -1,19 +1,19 @@ --============================================================================================================================= -- MARS-1057: Rollback FILE_MANAGER Package Body to v3.3.0 --============================================================================================================================= --- Purpose: Restore FILE_MANAGER package body to version before MARS-1057 changes +-- Purpose: Restore FILE_MANAGER package body to version before MARS-1057 changes (from v3.5.0 to v3.3.0) -- Author: Grzegorz Michalski --- Date: 2025-11-27 --- Related: MARS-1057 Batch External Table Creation (ROLLBACK) +-- Date: 2026-02-18 +-- Related: MARS-1057 Batch External Table Creation + Area Filter (ROLLBACK) --============================================================================================================================= SET SERVEROUTPUT ON PROMPT ======================================================================== -PROMPT Rolling back FILE_MANAGER Package Body to v3.3.0 +PROMPT Rolling back FILE_MANAGER Package Body from v3.5.0 to v3.3.0 PROMPT ======================================================================== -@@current_version/FILE_MANAGER.pkb +@@rollback_version/FILE_MANAGER.pkb -- Verify compilation status (check specific schema when installing as ADMIN) SELECT object_name, object_type, status diff --git a/MARS_Packages/REL03/MARS-1057/93_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql b/MARS_Packages/REL03/MARS-1057/93_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql index 8f8de59..6abcae5 100644 --- a/MARS_Packages/REL03/MARS-1057/93_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql +++ b/MARS_Packages/REL03/MARS-1057/93_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql @@ -3,9 +3,9 @@ -- =================================================================== -- Purpose: Restore FILE_MANAGER_ODS package body to v2.1.0 -- Author: Grzegorz Michalski --- Date: 2025-11-27 +-- Date: 2026-02-18 -- Package: ODS.FILE_MANAGER_ODS --- Version: 2.2.0 -> 2.1.0 (rollback) +-- Version: 2.4.0 -> 2.1.0 (rollback) SET ECHO ON SET DEFINE OFF diff --git a/MARS_Packages/REL03/MARS-1057/94_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql b/MARS_Packages/REL03/MARS-1057/94_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql index de742e2..452762b 100644 --- a/MARS_Packages/REL03/MARS-1057/94_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql +++ b/MARS_Packages/REL03/MARS-1057/94_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql @@ -3,9 +3,9 @@ -- =================================================================== -- Purpose: Restore FILE_MANAGER_ODS package specification to v2.1.0 -- Author: Grzegorz Michalski --- Date: 2025-11-27 +-- Date: 2026-02-18 -- Package: ODS.FILE_MANAGER_ODS --- Version: 2.2.0 -> 2.1.0 (rollback) +-- Version: 2.4.0 -> 2.1.0 (rollback) SET ECHO ON SET DEFINE OFF diff --git a/MARS_Packages/REL03/MARS-1057/README.md b/MARS_Packages/REL03/MARS-1057/README.md deleted file mode 100644 index 7331493..0000000 --- a/MARS_Packages/REL03/MARS-1057/README.md +++ /dev/null @@ -1,376 +0,0 @@ -# MARS-1057: Batch External Table Creation - -## Overview -This MARS package adds batch external table creation capabilities to the FILE_MANAGER and FILE_MANAGER_ODS packages, enabling automatic creation of external table sets (INBOX, ODS, ARCHIVE) based on A_SOURCE_FILE_CONFIG metadata. - -**Jira Issue:** MARS-1057 -**Package Version:** FILE_MANAGER 3.4.0, FILE_MANAGER_ODS 2.2.0 -**Author:** Grzegorz Michalski -**Date:** 2025-11-27 - -## Contents -- `install_mars1057.sql` - Master installation script with SPOOL logging -- `rollback_mars1057.sql` - Master rollback script -- `01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql` - Install FILE_MANAGER package specification -- `02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql` - Install FILE_MANAGER package body -- `03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql` - Install FILE_MANAGER_ODS package specification -- `04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql` - Install FILE_MANAGER_ODS package body -- `91_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_SPEC.sql` - Rollback FILE_MANAGER package specification -- `92_MARS_1057_rollback_CT_MRDS_FILE_MANAGER_BODY.sql` - Rollback FILE_MANAGER package body -- `93_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_BODY.sql` - Rollback FILE_MANAGER_ODS package body -- `94_MARS_1057_rollback_ODS_FILE_MANAGER_ODS_SPEC.sql` - Rollback FILE_MANAGER_ODS package specification -- `track_package_versions.sql` - Universal package version tracking -- `verify_packages_version.sql` - Universal package verification -- `rollback_version/` - FILE_MANAGER v3.3.0, FILE_MANAGER_ODS v2.1.0 (before MARS-1057) -- `new_version/` - FILE_MANAGER v3.4.0, FILE_MANAGER_ODS v2.2.0 (after MARS-1057) -- `.gitignore` - Git exclusions for temporary files - -## Prerequisites -- Oracle Database 23ai -- FILE_MANAGER package v3.3.0 installed -- FILE_MANAGER_ODS package v2.1.0 installed -- ENV_MANAGER package v3.1.0+ with version tracking -- ADMIN user access for deployment -- ODS schema with necessary privileges - -## New Features - -### 1. CREATE_EXTERNAL_TABLES_SET -Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration from A_SOURCE_FILE_CONFIG. - -**Signature:** -```sql -PROCEDURE CREATE_EXTERNAL_TABLES_SET ( - pSourceFileConfigKey IN NUMBER, - pRecreate IN BOOLEAN DEFAULT FALSE -); -``` - -**Example:** -```sql -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123, - pRecreate => FALSE - ); -END; -/ -``` - -**Features:** -- Automatic table naming: `{TABLE_ID}_{INBOX|ODS|ARCHIVE}` -- Official path patterns compliance -- Optional drop and recreate -- Full ENV_MANAGER logging -- Error handling with detailed messages - -### 2. CREATE_EXTERNAL_TABLES_BATCH -Creates external table sets for multiple configurations based on filter criteria. - -**Signature:** -```sql -PROCEDURE CREATE_EXTERNAL_TABLES_BATCH ( - pSourceKey IN VARCHAR2 DEFAULT NULL, - pSourceFileId IN VARCHAR2 DEFAULT NULL, - pTableId IN VARCHAR2 DEFAULT NULL, - pRecreate IN BOOLEAN DEFAULT FALSE -); -``` - -**Examples:** -```sql --- All external tables for C2D source -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pSourceKey => 'C2D' - ); -END; -/ - --- Recreate all external tables -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pRecreate => TRUE - ); -END; -/ - --- Specific table across all sources -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pTableId => 'A_UC_DISSEM_METADATA_LOADS' - ); -END; -/ -``` - -**Features:** -- Filters only INPUT type files -- Continues processing on errors -- Returns summary (Total/Processed/Failed) -- Comprehensive logging - -### 3. FILE_MANAGER_ODS Wrapper Procedures -The ODS.FILE_MANAGER_ODS package now includes DEFINER rights wrappers for the new batch procedures. - -**New Wrappers:** -```sql --- Create single external table set via ODS wrapper -EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123, - pRecreate => FALSE -); - --- Create batch external tables via ODS wrapper -EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH( - pSourceKey => 'C2D', - pRecreate => FALSE -); -``` - -**Benefits:** -- Objects created in ODS schema regardless of execution user -- Same functionality as CT_MRDS.FILE_MANAGER with DEFINER rights -- Full logging and error handling -- Consistent API with original package - -## Installation - -### Option 1: Master Script (Recommended) -```powershell -# IMPORTANT: Execute as ADMIN user -Get-Content "MARS_Packages/REL03/MARS-1057/install_mars1057.sql" | sql "ADMIN/password@service" - -# Log file created: log/INSTALL_MARS_1057__.log -``` - -**Installation Steps:** -1. Install FILE_MANAGER package specification v3.4.0 -2. Install FILE_MANAGER package body v3.4.0 -3. Install FILE_MANAGER_ODS package specification v2.2.0 -4. Install FILE_MANAGER_ODS package body v2.2.0 -5. Track versions in A_PACKAGE_VERSION_TRACKING -6. Verify all tracked packages for untracked changes - -### Option 2: Individual Scripts -```powershell -# IMPORTANT: Execute as ADMIN user -Get-Content "01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql" | sql "ADMIN/password@service" -Get-Content "02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql" | sql "ADMIN/password@service" -Get-Content "03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql" | sql "ADMIN/password@service" -Get-Content "04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql" | sql "ADMIN/password@service" -Get-Content "track_package_versions.sql" | sql "ADMIN/password@service" -Get-Content "verify_packages_version.sql" | sql "ADMIN/password@service" -``` - -## Verification -```sql --- Check FILE_MANAGER version -SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() FROM DUAL; --- Expected: 3.4.0 - --- Check FILE_MANAGER_ODS version -SELECT ODS.FILE_MANAGER_ODS.GET_VERSION() FROM DUAL; --- Expected: 2.2.0 - --- Check for errors (ADMIN user checks specific schema) -SELECT * FROM ALL_ERRORS -WHERE OWNER IN ('CT_MRDS', 'ODS') - AND NAME IN ('FILE_MANAGER', 'FILE_MANAGER_ODS'); --- Expected: No rows - --- Verify new FILE_MANAGER procedures exist -SELECT procedure_name -FROM ALL_PROCEDURES -WHERE OWNER = 'CT_MRDS' - AND object_name = 'FILE_MANAGER' - AND procedure_name IN ('CREATE_EXTERNAL_TABLES_SET', 'CREATE_EXTERNAL_TABLES_BATCH'); --- Expected: 2 rows - --- Verify new FILE_MANAGER_ODS procedures exist -SELECT procedure_name -FROM ALL_PROCEDURES -WHERE OWNER = 'ODS' - AND object_name = 'FILE_MANAGER_ODS' - AND procedure_name IN ('CREATE_EXTERNAL_TABLES_SET', 'CREATE_EXTERNAL_TABLES_BATCH'); --- Expected: 2 rows - --- Check for untracked changes -SELECT CT_MRDS.ENV_MANAGER.CHECK_PACKAGE_CHANGES('CT_MRDS', 'FILE_MANAGER') FROM DUAL; --- Expected: OK: Package CT_MRDS.FILE_MANAGER has not changed. - -SELECT CT_MRDS.ENV_MANAGER.CHECK_PACKAGE_CHANGES('ODS', 'FILE_MANAGER_ODS') FROM DUAL; --- Expected: OK: Package ODS.FILE_MANAGER_ODS has not changed. -``` - -## Rollback -```powershell -# IMPORTANT: Execute as ADMIN user -Get-Content "MARS_Packages/REL03/MARS-1057/rollback_mars1057.sql" | sql "ADMIN/password@service" -``` - -**Rollback restores:** -- FILE_MANAGER package specification v3.3.0 -- FILE_MANAGER package body v3.3.0 -- FILE_MANAGER_ODS package specification v2.1.0 -- FILE_MANAGER_ODS package body v2.1.0 -- Removes CREATE_EXTERNAL_TABLES_SET procedures from both packages -- Removes CREATE_EXTERNAL_TABLES_BATCH procedures from both packages - -## Expected Changes -- **FILE_MANAGER package**: v3.3.0 → v3.4.0 -- **FILE_MANAGER_ODS package**: v2.1.0 → v2.2.0 -- **New procedures in FILE_MANAGER**: CREATE_EXTERNAL_TABLES_SET, CREATE_EXTERNAL_TABLES_BATCH -- **New procedures in FILE_MANAGER_ODS**: CREATE_EXTERNAL_TABLES_SET, CREATE_EXTERNAL_TABLES_BATCH -- **FILE_MANAGER SPEC size**: +3.3 KB (declaration and documentation) -- **FILE_MANAGER BODY size**: +8.6 KB (implementation with logging) -- **FILE_MANAGER_ODS SPEC size**: +2.8 KB (wrapper declarations) -- **FILE_MANAGER_ODS BODY size**: +4.5 KB (wrapper implementations) - -## Testing - -### Test 1: Create Single Set (CT_MRDS.FILE_MANAGER) -```sql -BEGIN - CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123 - ); -END; -/ - --- Verify tables created -SELECT table_name -FROM ALL_TABLES -WHERE OWNER = 'ODS' - AND (table_name LIKE '%_INBOX' - OR table_name LIKE '%_ODS' - OR table_name LIKE '%_ARCHIVE'); -``` - -### Test 2: Create Single Set (ODS.FILE_MANAGER_ODS) -```sql -BEGIN - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123, - pRecreate => TRUE - ); -END; -/ -``` - -### Test 3: Batch Creation (CT_MRDS.FILE_MANAGER) -```sql -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123 - ); -END; -/ - --- Verify tables created -SELECT table_name -FROM ALL_TABLES -WHERE OWNER = 'ODS' - AND table_name LIKE '%_INBOX' - OR table_name LIKE '%_ODS' - OR table_name LIKE '%_ARCHIVE'; -``` - -### Test 2: Batch Creation -```sql -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pSourceKey => 'C2D' - ); -END; -/ - --- Check process log for results -SELECT * -FROM CT_MRDS.A_PROCESS_LOG -WHERE LOG_TIMESTAMP > SYSDATE - INTERVAL '1' HOUR - AND PROCEDURE_NAME LIKE '%CREATE_EXTERNAL_TABLES%' -ORDER BY LOG_TIMESTAMP DESC; -``` - -### Test 3: Recreate Existing Tables -```sql -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( - pSourceFileConfigKey => 123, - pRecreate => TRUE - ); -END; -/ -``` - -## Usage Examples - -### Example 1: Setup All External Tables for New Source -```sql --- 1. Add source configuration -CALL FILE_MANAGER.ADD_SOURCE('LM', 'Liquidity Management'); - --- 2. Add file configurations -CALL FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( - pSourceKey => 'LM', - pSourceFileType => 'INPUT', - pSourceFileId => 'STANDING_FACILITIES', - pSourceFileDesc => 'Standing Facilities Data', - pSourceFileNamePattern => 'SF_*.csv', - pTableId => 'STANDING_FACILITIES', - pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES', - pEncoding => 'UTF8' -); - --- 3. Create all external tables for this source -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pSourceKey => 'LM' - ); -END; -/ -``` - -### Example 2: Recreate All External Tables -```sql --- Useful after bucket URI changes or template table modifications -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pRecreate => TRUE - ); -END; -/ -``` - -### Example 3: Create Tables for Specific File Type -```sql -BEGIN - FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( - pSourceFileId => 'UC_DISSEM' - ); -END; -/ -``` - -## Known Issues -None - -## Dependencies -- **CT_MRDS.ENV_MANAGER** - Logging and error handling -- **ODS.FILE_MANAGER_ODS** - AUTHID DEFINER wrapper for external table creation -- **CT_MRDS.A_SOURCE_FILE_CONFIG** - Source file configuration metadata -- **CT_ET_TEMPLATES schema** - Template table definitions - -## Related -- **MARS-1056** - VARCHAR2 CHAR/BYTE semantics fix -- **MARS-1049** - CSV encoding support -- **Package Deployment Guide** - Standard deployment procedures -- **Tables Setup Guide** - External table configuration guide - -## Notes -- All installations must be executed as ADMIN user -- Use `ALL_*` views instead of `USER_*` views for verification -- Master scripts include SPOOL logging for audit trail -- ACCEPT validation prevents accidental execution -- Follows official path patterns: INBOX (3-level), ODS (2-level), ARCHIVE (2-level) diff --git a/MARS_Packages/REL03/MARS-1057/install_mars1057.sql b/MARS_Packages/REL03/MARS-1057/install_mars1057.sql index 1347817..56a4148 100644 --- a/MARS_Packages/REL03/MARS-1057/install_mars1057.sql +++ b/MARS_Packages/REL03/MARS-1057/install_mars1057.sql @@ -1,10 +1,11 @@ -- =================================================================== -- MARS-1057 INSTALL SCRIPT: Batch External Table Creation -- =================================================================== --- Purpose: Install FILE_MANAGER v3.4.0 with batch external table creation procedures +-- Purpose: Install FILE_MANAGER v3.5.0 with batch external table creation procedures +-- and area filter functionality -- Author: Grzegorz Michalski --- Date: 2025-11-27 --- Version: 3.4.0 +-- Date: 2026-02-18 +-- Version: 3.5.0 -- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) -- Log files are automatically created in log/ subdirectory @@ -29,20 +30,28 @@ SET PAUSE OFF -- ALTER SESSION SET CURRENT_SCHEMA = CT_MRDS; PROMPT ========================================================================= -PROMPT MARS-1057: Batch External Table Creation +PROMPT MARS-1057: Batch External Table Creation + Area Filter PROMPT ========================================================================= PROMPT -PROMPT This script will install FILE_MANAGER package v3.4.0 with new features: +PROMPT This script will install FILE_MANAGER package v3.5.0 with new features: PROMPT - CREATE_EXTERNAL_TABLES_SET procedure PROMPT - CREATE_EXTERNAL_TABLES_BATCH procedure +PROMPT - pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL) +PROMPT - pRestoreGrants parameter for grant preservation during recreate PROMPT -PROMPT And FILE_MANAGER_ODS package v2.2.0 with wrapper procedures. +PROMPT And FILE_MANAGER_ODS package v2.4.0 with wrapper procedures. PROMPT PROMPT Changes: -PROMPT - FILE_MANAGER package specification (3.3.0 -> 3.4.0) -PROMPT - FILE_MANAGER package body (3.3.0 -> 3.4.0) -PROMPT - FILE_MANAGER_ODS package specification (2.1.0 -> 2.2.0) -PROMPT - FILE_MANAGER_ODS package body (2.1.0 -> 2.2.0) +PROMPT - FILE_MANAGER package specification (3.3.0 -> 3.5.0) +PROMPT - FILE_MANAGER package body (3.3.0 -> 3.5.0) +PROMPT - FILE_MANAGER_ODS package specification (2.1.0 -> 2.4.0) +PROMPT - FILE_MANAGER_ODS package body (2.1.0 -> 2.4.0) +PROMPT +PROMPT New functionality: +PROMPT - pArea => 'INBOX' : creates only INBOX table +PROMPT - pArea => 'ODS' : creates only ODS table +PROMPT - pArea => 'ARCHIVE' : creates only ARCHIVE table +PROMPT - pArea => 'ALL' : creates all tables (default) PROMPT PROMPT Expected Duration: 1-2 minutes PROMPT ========================================================================= @@ -60,25 +69,25 @@ WHENEVER SQLERROR CONTINUE PROMPT PROMPT ========================================================================= -PROMPT Step 1: Install FILE_MANAGER Package Specification v3.4.0 +PROMPT Step 1: Install FILE_MANAGER Package Specification v3.5.0 PROMPT ========================================================================= @@01_MARS_1057_install_CT_MRDS_FILE_MANAGER_SPEC.sql PROMPT PROMPT ========================================================================= -PROMPT Step 2: Install FILE_MANAGER Package Body v3.4.0 +PROMPT Step 2: Install FILE_MANAGER Package Body v3.5.0 PROMPT ========================================================================= @@02_MARS_1057_install_CT_MRDS_FILE_MANAGER_BODY.sql PROMPT PROMPT ========================================================================= -PROMPT Step 3: Install FILE_MANAGER_ODS Package Specification v2.2.0 +PROMPT Step 3: Install FILE_MANAGER_ODS Package Specification v2.4.0 PROMPT ========================================================================= @@03_MARS_1057_install_ODS_FILE_MANAGER_ODS_SPEC.sql PROMPT PROMPT ========================================================================= -PROMPT Step 4: Install FILE_MANAGER_ODS Package Body v2.2.0 +PROMPT Step 4: Install FILE_MANAGER_ODS Package Body v2.4.0 PROMPT ========================================================================= @@04_MARS_1057_install_ODS_FILE_MANAGER_ODS_BODY.sql diff --git a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb index f79de19..c4a78e3 100644 --- a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb +++ b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb @@ -1969,7 +1969,8 @@ AS PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS -- Type for storing grant information @@ -2001,6 +2002,7 @@ AS vTableExists NUMBER; vParameters VARCHAR2(4000); + vAreaUpper VARCHAR2(20); -- Nested procedure to save table grants before DROP PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS @@ -2104,10 +2106,19 @@ AS END DROP_IF_EXISTS; BEGIN + -- Validate and normalize pArea parameter + vAreaUpper := UPPER(TRIM(pArea)); + + IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN + vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL'; + RAISE_APPLICATION_ERROR(-20010, vgMsgTmp); + END IF; + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'), 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END, - 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END + 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END, + 'pArea => ''' || vAreaUpper || '''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); @@ -2140,7 +2151,8 @@ AS vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId; ENV_MANAGER.LOG_PROCESS_EVENT( - 'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId, + 'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId || + ' (Area: ' || vAreaUpper || ')', 'INFO' ); @@ -2149,60 +2161,93 @@ AS -- Save grants before dropping tables (if pRestoreGrants = TRUE) IF pRestoreGrants THEN ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO'); - SAVE_GRANTS(vInboxTableName, vInboxGrants); - SAVE_GRANTS(vOdsTableName, vOdsGrants); - SAVE_GRANTS(vArchiveTableName, vArchiveGrants); + + IF vAreaUpper IN ('INBOX', 'ALL') THEN + SAVE_GRANTS(vInboxTableName, vInboxGrants); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + SAVE_GRANTS(vOdsTableName, vOdsGrants); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + SAVE_GRANTS(vArchiveTableName, vArchiveGrants); + END IF; END IF; - -- Drop existing tables - DROP_IF_EXISTS(vInboxTableName); - DROP_IF_EXISTS(vOdsTableName); - DROP_IF_EXISTS(vArchiveTableName); + -- Drop existing tables based on pArea + IF vAreaUpper IN ('INBOX', 'ALL') THEN + DROP_IF_EXISTS(vInboxTableName); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + DROP_IF_EXISTS(vOdsTableName); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + DROP_IF_EXISTS(vArchiveTableName); + END IF; END IF; - -- 5. Create INBOX external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO'); - CREATE_EXTERNAL_TABLE( - pTableName => vInboxTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vInboxPrefix, - pBucketUri => ENV_MANAGER.gvInboxBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 5. Create INBOX external table (if requested) + IF vAreaUpper IN ('INBOX', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vInboxTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vInboxPrefix, + pBucketUri => ENV_MANAGER.gvInboxBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; - -- 6. Create ODS external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO'); - CREATE_EXTERNAL_TABLE( - pTableName => vOdsTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vOdsPrefix, - pBucketUri => ENV_MANAGER.gvDataBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 6. Create ODS external table (if requested) + IF vAreaUpper IN ('ODS', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vOdsTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vOdsPrefix, + pBucketUri => ENV_MANAGER.gvDataBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; - -- 7. Create ARCHIVE external table - ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO'); - CREATE_EXTERNAL_TABLE( - pTableName => vArchiveTableName, - pTemplateTableName => vTemplateTableName, - pPrefix => vArchivePrefix, - pBucketUri => ENV_MANAGER.gvArchiveBucketUri, - pDelimiter => vDelimiter, - pEncoding => vEncoding - ); + -- 7. Create ARCHIVE external table (if requested) + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO'); + CREATE_EXTERNAL_TABLE( + pTableName => vArchiveTableName, + pTemplateTableName => vTemplateTableName, + pPrefix => vArchivePrefix, + pBucketUri => ENV_MANAGER.gvArchiveBucketUri, + pDelimiter => vDelimiter, + pEncoding => vEncoding + ); + END IF; -- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE) IF pRecreate AND pRestoreGrants THEN ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO'); - RESTORE_GRANTS(vInboxTableName, vInboxGrants); - RESTORE_GRANTS(vOdsTableName, vOdsGrants); - RESTORE_GRANTS(vArchiveTableName, vArchiveGrants); + + IF vAreaUpper IN ('INBOX', 'ALL') THEN + RESTORE_GRANTS(vInboxTableName, vInboxGrants); + END IF; + + IF vAreaUpper IN ('ODS', 'ALL') THEN + RESTORE_GRANTS(vOdsTableName, vOdsGrants); + END IF; + + IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN + RESTORE_GRANTS(vArchiveTableName, vArchiveGrants); + END IF; END IF; ENV_MANAGER.LOG_PROCESS_EVENT( - 'End CREATE_EXTERNAL_TABLES_SET - Successfully created all 3 external tables for config: ' || pSourceFileConfigKey, + 'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' || + pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')', 'INFO', vParameters ); @@ -2221,20 +2266,31 @@ AS pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS vCount NUMBER := 0; vProcessed NUMBER := 0; vFailed NUMBER := 0; vParameters VARCHAR2(4000); + vAreaUpper VARCHAR2(20); BEGIN + -- Validate and normalize pArea parameter + vAreaUpper := UPPER(TRIM(pArea)); + + IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN + vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL'; + RAISE_APPLICATION_ERROR(-20010, vgMsgTmp); + END IF; + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''', 'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''', 'pTableId => ''' || NVL(pTableId, 'NULL') || '''', 'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END, - 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END + 'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END, + 'pArea => ''' || vAreaUpper || '''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); @@ -2254,15 +2310,16 @@ AS BEGIN ENV_MANAGER.LOG_PROCESS_EVENT( 'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' || - rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID, + rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')', 'INFO' ); - -- Call procedure to create set of 3 tables + -- Call procedure to create set of tables (based on pArea) CREATE_EXTERNAL_TABLES_SET( pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY, pRecreate => pRecreate, - pRestoreGrants => pRestoreGrants + pRestoreGrants => pRestoreGrants, + pArea => vAreaUpper ); vProcessed := vProcessed + 1; @@ -2281,7 +2338,7 @@ AS ENV_MANAGER.LOG_PROCESS_EVENT( 'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount || - ', Processed: ' || vProcessed || ', Failed: ' || vFailed, + ', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')', 'INFO', vParameters ); diff --git a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkg b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkg index c348808..adf8d28 100644 --- a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkg +++ b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkg @@ -17,12 +17,13 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.5.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) || '3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) || '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) || '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) || @@ -602,38 +603,50 @@ AS /** * @name CREATE_EXTERNAL_TABLES_SET - * @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration - * from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following - * official path patterns. Optionally drops and recreates existing tables. - * If pRestoreGrants is TRUE, saves and restores table grants during recreate operation. + * @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table. + * Automatically generates table names and paths following official path patterns. + * Optionally drops and recreates existing tables. If pRestoreGrants is TRUE, saves and restores table grants. + * The pArea parameter allows selective table creation. * @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table * @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege - * @example BEGIN + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * 'INBOX' - creates only INBOX table + * 'ODS' - creates only ODS table + * 'ARCHIVE' - creates only ARCHIVE table + * 'ALL' - creates all three tables (default) + * @example -- Create only INBOX table + * BEGIN + * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( + * pSourceFileConfigKey => 123, + * pArea => 'INBOX' + * ); + * END; + * + * -- Create all tables with grant preservation + * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( * pSourceFileConfigKey => 123, * pRecreate => TRUE, - * pRestoreGrants => TRUE + * pRestoreGrants => TRUE, + * pArea => 'ALL' * ); * END; - * @ex_rslt Creates three external tables in ODS schema: - * - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX - * - C2D_A_UC_DISSEM_METADATA_LOADS_ODS - * - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE - * Preserves existing grants if pRestoreGrants = TRUE + * @ex_rslt Creates external table(s) in ODS schema based on pArea parameter **/ PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); /** * @name CREATE_EXTERNAL_TABLES_BATCH * @desc Creates external table sets for multiple configurations based on filter criteria. - * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration - * (INBOX, ODS, ARCHIVE). Continues processing even if individual sets fail. + * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter + * (INBOX, ODS, ARCHIVE, or ALL). Continues processing even if individual sets fail. * If pRestoreGrants is TRUE, saves and restores table grants during recreate operations. * @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources) * @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types) @@ -641,20 +654,21 @@ AS * @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege - * @example -- Create all external tables for C2D source with grant preservation + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX tables for C2D source * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( * pSourceKey => 'C2D', - * pRecreate => TRUE, - * pRestoreGrants => TRUE + * pArea => 'INBOX' * ); * END; * - * -- Recreate all external tables without grant preservation + * -- Create all external tables for all sources with grant preservation * BEGIN * FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH( * pRecreate => TRUE, - * pRestoreGrants => FALSE + * pRestoreGrants => TRUE, + * pArea => 'ALL' * ); * END; * @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1 @@ -664,7 +678,8 @@ AS pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); --------------------------------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkb b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkb index 58a83d6..b147983 100644 --- a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkb +++ b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkb @@ -64,7 +64,8 @@ AS PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; @@ -80,7 +81,8 @@ AS SYS.ODCIVARCHAR2LIST( 'pSourceFileConfigKey => ' || pSourceFileConfigKey, 'pRecreate => ' || vRecreateStr, - 'pRestoreGrants => ' || vRestoreGrantsStr + 'pRestoreGrants => ' || vRestoreGrantsStr, + 'pArea => ''' || pArea || '''' ) ); @@ -92,7 +94,8 @@ AS CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET( pSourceFileConfigKey => pSourceFileConfigKey, pRecreate => pRecreate, - pRestoreGrants => pRestoreGrants + pRestoreGrants => pRestoreGrants, + pArea => pArea ); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters); @@ -117,7 +120,8 @@ AS pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ) IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; @@ -135,7 +139,8 @@ AS 'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''', 'pTableId => ''' || NVL(pTableId, 'NULL') || '''', 'pRecreate => ' || vRecreateStr, - 'pRestoreGrants => ' || vRestoreGrantsStr + 'pRestoreGrants => ' || vRestoreGrantsStr, + 'pArea => ''' || pArea || '''' ) ); @@ -149,7 +154,8 @@ AS pSourceFileId => pSourceFileId, pTableId => pTableId, pRecreate => pRecreate, - pRestoreGrants => pRestoreGrants + pRestoreGrants => pRestoreGrants, + pArea => pArea ); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters); diff --git a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkg b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkg index d5d0c72..eea529d 100644 --- a/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkg +++ b/MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER_ODS.pkg @@ -16,12 +16,13 @@ AS */ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.3.0'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 13:30:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.4.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '2.4.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) || '2.3.0 (2026-02-18): MARS-1057 - Added pRestoreGrants parameter support for grant preservation during table recreate' || CHR(13)||CHR(10) || '2.2.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers for batch external table creation' || CHR(13)||CHR(10) || '2.1.0 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) || @@ -64,38 +65,43 @@ AS /** * @name CREATE_EXTERNAL_TABLES_SET - * @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration - * from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following - * official path patterns. Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET - * with DEFINER rights to ensure objects are created in ODS schema. + * @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table. + * Automatically generates table names and paths following official path patterns. + * Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET with DEFINER rights. * If pRestoreGrants is TRUE, saves and restores table grants during recreate operation. + * The pArea parameter allows selective table creation. * @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table * @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege - * @example EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX table + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( + * pSourceFileConfigKey => 123, + * pArea => 'INBOX' + * ); + * + * -- Create all tables with grant preservation + * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET( * pSourceFileConfigKey => 123, * pRecreate => TRUE, - * pRestoreGrants => TRUE + * pRestoreGrants => TRUE, + * pArea => 'ALL' * ); - * @ex_rslt Creates three external tables in ODS schema: - * - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX - * - C2D_A_UC_DISSEM_METADATA_LOADS_ODS - * - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE - * Preserves existing grants if pRestoreGrants = TRUE + * @ex_rslt Creates external table(s) in ODS schema based on pArea parameter */ PROCEDURE CREATE_EXTERNAL_TABLES_SET ( pSourceFileConfigKey IN NUMBER, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); /** * @name CREATE_EXTERNAL_TABLES_BATCH * @desc Creates external table sets for multiple configurations based on filter criteria. - * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration - * (INBOX, ODS, ARCHIVE). Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH - * with DEFINER rights to ensure objects are created in ODS schema. + * Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter. + * Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH with DEFINER rights. * If pRestoreGrants is TRUE, saves and restores table grants during recreate operations. * @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources) * @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types) @@ -103,17 +109,18 @@ AS * @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist * @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE) * Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege - * @example -- Create all external tables for C2D source with grant preservation + * @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default) + * @example -- Create only INBOX tables for C2D source * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH( * pSourceKey => 'C2D', - * pRecreate => TRUE, - * pRestoreGrants => TRUE + * pArea => 'INBOX' * ); * - * -- Recreate all external tables without grant preservation + * -- Create all tables with grant preservation * EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH( * pRecreate => TRUE, - * pRestoreGrants => FALSE + * pRestoreGrants => TRUE, + * pArea => 'ALL' * ); * @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1 */ @@ -122,7 +129,8 @@ AS pSourceFileId IN VARCHAR2 DEFAULT NULL, pTableId IN VARCHAR2 DEFAULT NULL, pRecreate IN BOOLEAN DEFAULT FALSE, - pRestoreGrants IN BOOLEAN DEFAULT TRUE + pRestoreGrants IN BOOLEAN DEFAULT TRUE, + pArea IN VARCHAR2 DEFAULT 'ALL' ); --------------------------------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/REL03/MARS-1057/rollback_mars1057.sql b/MARS_Packages/REL03/MARS-1057/rollback_mars1057.sql index 8571cac..dcfb138 100644 --- a/MARS_Packages/REL03/MARS-1057/rollback_mars1057.sql +++ b/MARS_Packages/REL03/MARS-1057/rollback_mars1057.sql @@ -1,9 +1,9 @@ -- =================================================================== -- MARS-1057 ROLLBACK SCRIPT: Batch External Table Creation -- =================================================================== --- Purpose: Rollback FILE_MANAGER package to v3.3.0 (remove batch external table procedures) +-- Purpose: Rollback FILE_MANAGER package to v3.3.0 (remove batch external table procedures and area filter) -- Author: Grzegorz Michalski --- Date: 2025-11-27 +-- Date: 2026-02-18 -- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) -- Log files are automatically created in log/ subdirectory @@ -25,17 +25,19 @@ SET SERVEROUTPUT ON SIZE UNLIMITED SET PAUSE OFF PROMPT ========================================================================= -PROMPT MARS-1057: Rollback Batch External Table Creation +PROMPT MARS-1057: Rollback Batch External Table Creation + Area Filter PROMPT ========================================================================= PROMPT WARNING: This will reverse all changes from MARS-1057 installation! PROMPT PROMPT Changes to be rolled back: -PROMPT - FILE_MANAGER package specification (3.4.0 -> 3.3.0) -PROMPT - FILE_MANAGER package body (3.4.0 -> 3.3.0) -PROMPT - FILE_MANAGER_ODS package specification (2.2.0 -> 2.1.0) -PROMPT - FILE_MANAGER_ODS package body (2.2.0 -> 2.1.0) +PROMPT - FILE_MANAGER package specification (3.5.0 -> 3.3.0) +PROMPT - FILE_MANAGER package body (3.5.0 -> 3.3.0) +PROMPT - FILE_MANAGER_ODS package specification (2.4.0 -> 2.1.0) +PROMPT - FILE_MANAGER_ODS package body (2.4.0 -> 2.1.0) PROMPT - Remove CREATE_EXTERNAL_TABLES_SET procedures PROMPT - Remove CREATE_EXTERNAL_TABLES_BATCH procedures +PROMPT - Remove pArea parameter (selective table creation) +PROMPT - Remove pRestoreGrants parameter (grant preservation) PROMPT ========================================================================= -- Confirm rollback with user