diff --git a/MARS_Packages/REL02_POST/MARS-1409/.gitignore b/MARS_Packages/REL02_POST/MARS-1409/.gitignore new file mode 100644 index 0000000..025f32f --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/.gitignore @@ -0,0 +1,26 @@ +# MARS-1409 Package - Git Ignore Rules +# Standard exclusions for MARS deployment packages + +# Confluence documentation (generated, not source) +confluence/ + +# Patches directory +patches/ + +# Log files from SPOOL operations +log/ + +# Test directories and files +test/ + +# Mock data scripts (development only) +mock_data/ + +# Temporary files +*.tmp +*.bak +*~ + +# Editor backups +.vscode/ +.idea/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/01_MARS_1409_add_workflow_history_key_column.sql b/MARS_Packages/REL02_POST/MARS-1409/01_MARS_1409_add_workflow_history_key_column.sql new file mode 100644 index 0000000..bcb5e62 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/01_MARS_1409_add_workflow_history_key_column.sql @@ -0,0 +1,22 @@ +-- ============================================================================ +-- MARS-1409 Step 01: Add A_WORKFLOW_HISTORY_KEY column +-- ============================================================================ +-- Purpose: Add A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED table +-- Prerequisites: Table A_SOURCE_FILE_RECEIVED exists, A_WORKFLOW_HISTORY table exists +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Adding A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED... + +-- TODO: Implementation +-- ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD ( +-- A_WORKFLOW_HISTORY_KEY NUMBER, +-- CONSTRAINT FK_SFR_WORKFLOW_HISTORY +-- FOREIGN KEY (A_WORKFLOW_HISTORY_KEY) +-- REFERENCES CT_MRDS.A_WORKFLOW_HISTORY(A_WORKFLOW_HISTORY_KEY) +-- ); + +PROMPT Column addition placeholder - TO BE IMPLEMENTED +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql b/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql new file mode 100644 index 0000000..57ab09a --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql @@ -0,0 +1,16 @@ +-- ============================================================================ +-- MARS-1409 Step 02: Install FILE_MANAGER Package Specification +-- ============================================================================ +-- Purpose: Deploy updated FILE_MANAGER package specification +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Installing FILE_MANAGER package specification... + +-- Source from new_version directory +@@new_version/FILE_MANAGER.pkg + +PROMPT FILE_MANAGER specification installed +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql b/MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql new file mode 100644 index 0000000..33b53b5 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql @@ -0,0 +1,16 @@ +-- ============================================================================ +-- MARS-1409 Step 03: Install FILE_MANAGER Package Body +-- ============================================================================ +-- Purpose: Deploy updated FILE_MANAGER package body +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Installing FILE_MANAGER package body... + +-- Source from new_version directory +@@new_version/FILE_MANAGER.pkb + +PROMPT FILE_MANAGER body installed +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_verify_installation.sql b/MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_verify_installation.sql new file mode 100644 index 0000000..86ad588 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_verify_installation.sql @@ -0,0 +1,76 @@ +-- ============================================================================ +-- MARS-1409 Step 04: Verify Installation +-- ============================================================================ +-- Purpose: Verify successful installation of MARS-1409 changes +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK ON + +PROMPT +PROMPT ============================================================================ +PROMPT Verifying MARS-1409 Installation +PROMPT ============================================================================ + +-- Check if column was added +PROMPT +PROMPT 1. Checking A_WORKFLOW_HISTORY_KEY column existence... +SELECT + column_name, + data_type, + data_length, + nullable +FROM user_tab_columns +WHERE table_name = 'A_SOURCE_FILE_RECEIVED' + AND column_name = 'A_WORKFLOW_HISTORY_KEY'; + +-- Check foreign key constraint +PROMPT +PROMPT 2. Checking foreign key constraint... +SELECT + constraint_name, + constraint_type, + r_constraint_name, + status +FROM user_constraints +WHERE table_name = 'A_SOURCE_FILE_RECEIVED' + AND constraint_type = 'R' + AND constraint_name LIKE '%WORKFLOW_HISTORY%'; + +-- Check package compilation status +PROMPT +PROMPT 3. Checking FILE_MANAGER package compilation... +SELECT + object_name, + object_type, + status, + last_ddl_time +FROM user_objects +WHERE object_name = 'FILE_MANAGER' + AND object_type IN ('PACKAGE', 'PACKAGE BODY') +ORDER BY object_type; + +-- Check for compilation errors +PROMPT +PROMPT 4. Checking for compilation errors... +SELECT + name, + type, + line, + position, + text +FROM user_errors +WHERE name = 'FILE_MANAGER' +ORDER BY type, line, position; + +-- Verify package version +PROMPT +PROMPT 5. Verifying package version... +SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() AS VERSION FROM DUAL; +SELECT CT_MRDS.FILE_MANAGER.GET_BUILD_INFO() AS BUILD_INFO FROM DUAL; + +PROMPT +PROMPT ============================================================================ +PROMPT Verification Complete +PROMPT ============================================================================ diff --git a/MARS_Packages/REL02_POST/MARS-1409/90_MARS_1409_verify_rollback.sql b/MARS_Packages/REL02_POST/MARS-1409/90_MARS_1409_verify_rollback.sql new file mode 100644 index 0000000..a1191d0 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/90_MARS_1409_verify_rollback.sql @@ -0,0 +1,53 @@ +-- ============================================================================ +-- MARS-1409 Rollback 90: Verify Rollback +-- ============================================================================ +-- Purpose: Verify successful rollback of MARS-1409 changes +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT ============================================================================ +PROMPT Verifying MARS-1409 Rollback +PROMPT ============================================================================ + +-- Check if column was removed +PROMPT +PROMPT 1. Verifying A_WORKFLOW_HISTORY_KEY column removal... +DECLARE + v_count NUMBER; +BEGIN + SELECT COUNT(*) + INTO v_count + FROM user_tab_columns + WHERE table_name = 'A_SOURCE_FILE_RECEIVED' + AND column_name = 'A_WORKFLOW_HISTORY_KEY'; + + IF v_count = 0 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: A_WORKFLOW_HISTORY_KEY column removed'); + ELSE + DBMS_OUTPUT.PUT_LINE('WARNING: A_WORKFLOW_HISTORY_KEY column still exists'); + END IF; +END; +/ + +-- Check package versions +PROMPT +PROMPT 2. Verifying FILE_MANAGER package version... +SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() AS VERSION FROM DUAL; + +-- Check compilation status +PROMPT +PROMPT 3. Checking package compilation status... +SELECT + object_name, + object_type, + status +FROM user_objects +WHERE object_name = 'FILE_MANAGER' + AND object_type IN ('PACKAGE', 'PACKAGE BODY'); + +PROMPT +PROMPT ============================================================================ +PROMPT Rollback Verification Complete +PROMPT ============================================================================ diff --git a/MARS_Packages/REL02_POST/MARS-1409/91_MARS_1409_rollback_workflow_history_key_column.sql b/MARS_Packages/REL02_POST/MARS-1409/91_MARS_1409_rollback_workflow_history_key_column.sql new file mode 100644 index 0000000..d6a14b3 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/91_MARS_1409_rollback_workflow_history_key_column.sql @@ -0,0 +1,17 @@ +-- ============================================================================ +-- MARS-1409 Rollback 91: Drop A_WORKFLOW_HISTORY_KEY column +-- ============================================================================ +-- Purpose: Remove A_WORKFLOW_HISTORY_KEY column from A_SOURCE_FILE_RECEIVED +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Dropping A_WORKFLOW_HISTORY_KEY column from A_SOURCE_FILE_RECEIVED... + +-- TODO: Implementation +-- ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED +-- DROP COLUMN A_WORKFLOW_HISTORY_KEY; + +PROMPT Column drop placeholder - TO BE IMPLEMENTED +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/92_MARS_1409_rollback_FILE_MANAGER_SPEC.sql b/MARS_Packages/REL02_POST/MARS-1409/92_MARS_1409_rollback_FILE_MANAGER_SPEC.sql new file mode 100644 index 0000000..b0aa958 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/92_MARS_1409_rollback_FILE_MANAGER_SPEC.sql @@ -0,0 +1,16 @@ +-- ============================================================================ +-- MARS-1409 Rollback 92: Restore FILE_MANAGER Package Specification +-- ============================================================================ +-- Purpose: Restore previous FILE_MANAGER package specification +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Restoring FILE_MANAGER package specification... + +-- Source from rollback_version directory +@@rollback_version/FILE_MANAGER.pkg + +PROMPT FILE_MANAGER specification restored +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/93_MARS_1409_rollback_FILE_MANAGER_BODY.sql b/MARS_Packages/REL02_POST/MARS-1409/93_MARS_1409_rollback_FILE_MANAGER_BODY.sql new file mode 100644 index 0000000..b84acee --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/93_MARS_1409_rollback_FILE_MANAGER_BODY.sql @@ -0,0 +1,16 @@ +-- ============================================================================ +-- MARS-1409 Rollback 93: Restore FILE_MANAGER Package Body +-- ============================================================================ +-- Purpose: Restore previous FILE_MANAGER package body +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT Restoring FILE_MANAGER package body... + +-- Source from rollback_version directory +@@rollback_version/FILE_MANAGER.pkb + +PROMPT FILE_MANAGER body restored +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/README.md b/MARS_Packages/REL02_POST/MARS-1409/README.md new file mode 100644 index 0000000..6dfab90 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/README.md @@ -0,0 +1,46 @@ +# MARS-1409: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED + +## Overview +Package for adding A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED table and updating FILE_MANAGER package to populate this value during file registration. + +## Purpose +Direct tracking of workflow history keys in file registration for improved diagnostics and simplified archival queries. + +## Structure +``` +MARS-1409/ +├── .gitignore +├── install_mars1409.sql # Master installation script +├── rollback_mars1409.sql # Master rollback script +├── verify_packages_version.sql # Version verification +├── track_package_versions.sql # Version tracking +├── 01_MARS_1409_add_workflow_history_key_column.sql +├── 02_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql +├── 03_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql +├── 04_MARS_1409_verify_installation.sql +├── 90_MARS_1409_verify_rollback.sql +├── 91_MARS_1409_rollback_workflow_history_key_column.sql +├── 92_MARS_1409_rollback_FILE_MANAGER_SPEC.sql +├── 93_MARS_1409_rollback_FILE_MANAGER_BODY.sql +├── new_version/ # Updated packages +│ ├── FILE_MANAGER.pkg # TO BE COPIED +│ └── FILE_MANAGER.pkb # TO BE COPIED +├── rollback_version/ # Previous versions +│ ├── FILE_MANAGER.pkg # TO BE COPIED +│ └── FILE_MANAGER.pkb # TO BE COPIED +└── log/ # Installation logs +``` + +## Status +**SCAFFOLD CREATED** - Ready for implementation + +## Next Steps +1. Copy current FILE_MANAGER packages to rollback_version/ +2. Implement column addition (01_*.sql) +3. Update FILE_MANAGER package to populate A_WORKFLOW_HISTORY_KEY +4. Test and validate changes +5. Complete rollback scripts + +## Related Tickets +- Based on MARS-828 package structure +- Supports FILE_ARCHIVER workflow tracking improvements diff --git a/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql b/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql new file mode 100644 index 0000000..3dd8cdc --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql @@ -0,0 +1,91 @@ +-- ============================================================================ +-- MARS-1409 Master Installation Script +-- ============================================================================ +-- Author: Grzegorz Michalski +-- Purpose: Add A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED table +-- Target Schema: CT_MRDS +-- Estimated Time: 1-2 minutes +-- Prerequisites: FILE_MANAGER v3.x, ENV_MANAGER v3.x, ADMIN privileges +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK ON +SET ECHO OFF + +-- Create log directory if it doesn't exist +host mkdir log 2>nul + +-- Generate dynamic SPOOL filename with timestamp +var filename VARCHAR2(100) +BEGIN + :filename := 'log/INSTALL_MARS_1409_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-1409 Installation Starting +PROMPT ============================================================================ +PROMPT Package: CT_MRDS.FILE_MANAGER v3.X.X +PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED +PROMPT Purpose: Direct tracking of workflow history keys in file registration +PROMPT Steps: TBD (DDL, Package updates, Verification) +PROMPT Timestamp: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; +PROMPT ============================================================================ + +-- Confirm installation with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20000, 'Installation aborted by user'); + END IF; +END; +/ + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 1: Add A_WORKFLOW_HISTORY_KEY column to A_SOURCE_FILE_RECEIVED +PROMPT ============================================================================ +@@01_MARS_1409_add_workflow_history_key_column.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 2: Update FILE_MANAGER package specification +PROMPT ============================================================================ +@@02_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 3: Update FILE_MANAGER package body +PROMPT ============================================================================ +@@03_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 4: Verify installation +PROMPT ============================================================================ +@@04_MARS_1409_verify_installation.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 5: Track package versions +PROMPT ============================================================================ +@@track_package_versions.sql + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-1409 Installation Complete +PROMPT ============================================================================ +PROMPT Final Status: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL; +PROMPT +PROMPT Review the log file for detailed results: &_filename +PROMPT ============================================================================ + +spool off diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_RECEIVED.sql b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_RECEIVED.sql new file mode 100644 index 0000000..18739dc --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_RECEIVED.sql @@ -0,0 +1,66 @@ +-- ==================================================================== +-- A_SOURCE_FILE_RECEIVED Table +-- ==================================================================== +-- Purpose: Track received files and their processing status +-- ==================================================================== + +CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ( + A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE, + A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE, + SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL, + CHECKSUM VARCHAR2(128), + CREATED TIMESTAMP(6) WITH TIME ZONE, + BYTES NUMBER, + RECEPTION_DATE DATE NOT NULL, + PROCESSING_STATUS VARCHAR2(200), + EXTERNAL_TABLE_NAME VARCHAR2(200), + PARTITION_YEAR VARCHAR2(4), + PARTITION_MONTH VARCHAR2(2), + ARCH_FILE_NAME VARCHAR2(1000), + CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY), + CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')) +) TABLESPACE "DATA"; + +-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation) +CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1 +ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES); + +-- Column comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS + 'Primary key - unique identifier for received file record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS + 'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS + 'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS + 'MD5 checksum of file content for integrity verification and duplicate detection'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS + 'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS + 'File size in bytes'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS + 'Date when file was registered in the system (extracted from CREATED timestamp)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS + 'Name of temporary external table created for file validation (dropped after validation)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS + 'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS + 'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_FILE_NAME IS + 'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)'; + +GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkb b/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkb new file mode 100644 index 0000000..856d449 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkb @@ -0,0 +1,1171 @@ +create or replace PACKAGE BODY CT_MRDS.ENV_MANAGER +AS + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE INIT_ERRORS IS + BEGIN + Errors(CODE_EMPTY_FILEURI_AND_RECKEY) := Error_Record(CODE_EMPTY_FILEURI_AND_RECKEY, MSG_EMPTY_FILEURI_AND_RECKEY); -- -20001 + Errors(CODE_NO_CONFIG_MATCH_FOR_FILEURI) := Error_Record(CODE_NO_CONFIG_MATCH_FOR_FILEURI, MSG_NO_CONFIG_MATCH_FOR_FILEURI); -- -20002 + Errors(CODE_MULTIPLE_MATCH_FOR_SRCFILE) := Error_Record(CODE_MULTIPLE_MATCH_FOR_SRCFILE, MSG_MULTIPLE_MATCH_FOR_SRCFILE); -- -20003 + Errors(CODE_MISSING_COLUMN_DATE_FORMAT) := Error_Record(CODE_MISSING_COLUMN_DATE_FORMAT, MSG_MISSING_COLUMN_DATE_FORMAT); -- -20004 + Errors(CODE_MULTIPLE_COLUMN_DATE_FORMAT) := Error_Record(CODE_MULTIPLE_COLUMN_DATE_FORMAT, MSG_MULTIPLE_COLUMN_DATE_FORMAT); -- -20005 + Errors(CODE_DIDNT_GET_LOAD_OPERATION_ID) := Error_Record(CODE_DIDNT_GET_LOAD_OPERATION_ID, MSG_DIDNT_GET_LOAD_OPERATION_ID); -- -20006 + Errors(CODE_NO_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_NO_CONFIG_FOR_RECEIVED_FILE, MSG_NO_CONFIG_FOR_RECEIVED_FILE); -- -20007 + Errors(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); -- -20008 + Errors(CODE_FILE_NOT_FOUND_ON_CLOUD) := Error_Record(CODE_FILE_NOT_FOUND_ON_CLOUD, MSG_FILE_NOT_FOUND_ON_CLOUD); -- -20009 + Errors(CODE_FILE_VALIDATION_FAILED) := Error_Record(CODE_FILE_VALIDATION_FAILED, MSG_FILE_VALIDATION_FAILED); -- -20010 + Errors(CODE_EXCESS_COLUMNS_DETECTED) := Error_Record(CODE_EXCESS_COLUMNS_DETECTED, MSG_EXCESS_COLUMNS_DETECTED); -- -20011 + Errors(CODE_NO_CONFIG_MATCH) := Error_Record(CODE_NO_CONFIG_MATCH, MSG_NO_CONFIG_MATCH); -- -20012 + Errors(CODE_UNKNOWN_PREFIX) := Error_Record(CODE_UNKNOWN_PREFIX, MSG_UNKNOWN_PREFIX); -- -20013 + Errors(CODE_TABLE_NOT_EXISTS) := Error_Record(CODE_TABLE_NOT_EXISTS, MSG_TABLE_NOT_EXISTS); -- -20014 + Errors(CODE_COLUMN_NOT_EXISTS) := Error_Record(CODE_COLUMN_NOT_EXISTS, MSG_COLUMN_NOT_EXISTS); -- -20015 + Errors(CODE_UNSUPPORTED_DATA_TYPE) := Error_Record(CODE_UNSUPPORTED_DATA_TYPE, MSG_UNSUPPORTED_DATA_TYPE); -- -20016 + Errors(CODE_MISSING_SOURCE_KEY) := Error_Record(CODE_MISSING_SOURCE_KEY, MSG_MISSING_SOURCE_KEY); -- -20017 + Errors(CODE_NULL_SOURCE_FILE_CONFIG_KEY) := Error_Record(CODE_NULL_SOURCE_FILE_CONFIG_KEY, MSG_NULL_SOURCE_FILE_CONFIG_KEY); -- -20018 + Errors(CODE_DUPLICATED_SOURCE_KEY) := Error_Record(CODE_DUPLICATED_SOURCE_KEY, MSG_DUPLICATED_SOURCE_KEY); -- -20019 + Errors(CODE_MISSING_CONTAINER_CONFIG) := Error_Record(CODE_MISSING_CONTAINER_CONFIG, MSG_MISSING_CONTAINER_CONFIG); -- -20020 + Errors(CODE_MULTIPLE_CONTAINER_ENTRIES) := Error_Record(CODE_MULTIPLE_CONTAINER_ENTRIES, MSG_MULTIPLE_CONTAINER_ENTRIES); -- -20021 + Errors(CODE_WRONG_DESTINATION_PARAM) := Error_Record(CODE_WRONG_DESTINATION_PARAM, MSG_WRONG_DESTINATION_PARAM); -- -20022 + Errors(CODE_FILE_NOT_EXISTS_ON_CLOUD) := Error_Record(CODE_FILE_NOT_EXISTS_ON_CLOUD, MSG_FILE_NOT_EXISTS_ON_CLOUD); -- -20023 + Errors(CODE_FILE_ALREADY_REGISTERED) := Error_Record(CODE_FILE_ALREADY_REGISTERED, MSG_FILE_ALREADY_REGISTERED); -- -20024 + Errors(CODE_WRONG_DATE_TIMESTAMP_FORMAT) := Error_Record(CODE_WRONG_DATE_TIMESTAMP_FORMAT, MSG_WRONG_DATE_TIMESTAMP_FORMAT); -- -20025 + Errors(CODE_ENVIRONMENT_NOT_SET) := Error_Record(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET); -- -20026 + Errors(CODE_CONFIG_VARIABLE_NOT_SET) := Error_Record(CODE_CONFIG_VARIABLE_NOT_SET, MSG_CONFIG_VARIABLE_NOT_SET); -- -20027 + Errors(CODE_NOT_INPUT_SOURCE_FILE_TYPE) := Error_Record(CODE_NOT_INPUT_SOURCE_FILE_TYPE, MSG_NOT_INPUT_SOURCE_FILE_TYPE); -- -20028 + Errors(CODE_EXP_DATA_FOR_ARCH_FAILED) := Error_Record(CODE_EXP_DATA_FOR_ARCH_FAILED, MSG_EXP_DATA_FOR_ARCH_FAILED); -- -20029 + Errors(CODE_RESTORE_FILE_FROM_TRASH) := Error_Record(CODE_RESTORE_FILE_FROM_TRASH, MSG_RESTORE_FILE_FROM_TRASH); -- -20030 + Errors(CODE_CHANGE_STAT_TO_ARCHIVED_FAILED):= Error_Record(CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); -- -20031 + Errors(CODE_MOVE_FILE_TO_TRASH_FAILED) := Error_Record(CODE_MOVE_FILE_TO_TRASH_FAILED, MSG_MOVE_FILE_TO_TRASH_FAILED); -- -20032 + Errors(CODE_DROP_EXPORTED_FILES_FAILED) := Error_Record(CODE_DROP_EXPORTED_FILES_FAILED, MSG_DROP_EXPORTED_FILES_FAILED); -- -20033 + Errors(CODE_INVALID_BUCKET_AREA) := Error_Record(CODE_INVALID_BUCKET_AREA, MSG_INVALID_BUCKET_AREA); -- -20034 + Errors(CODE_INVALID_PARALLEL_DEGREE) := Error_Record(CODE_INVALID_PARALLEL_DEGREE, MSG_INVALID_PARALLEL_DEGREE); -- -20110 + Errors(CODE_PARALLEL_EXECUTION_FAILED) := Error_Record(CODE_PARALLEL_EXECUTION_FAILED, MSG_PARALLEL_EXECUTION_FAILED); -- -20111 + + Errors(CODE_UNKNOWN) := Error_Record(CODE_UNKNOWN, MSG_UNKNOWN); -- -20999 + + END INIT_ERRORS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DEFAULT_ENV + RETURN VARCHAR2 + IS + vDefaultEnv CT_MRDS.a_file_manager_config.config_variable_value%TYPE; + BEGIN + select config_variable_value + into vDefaultEnv + from CT_MRDS.a_file_manager_config + where lower(environment_id)='default' + and lower(config_variable)='environmentid'; + RETURN vDefaultEnv; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + END; + + ---------------------------------------------------------------------------------------------------- + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE INIT_VARIABLES( + pEnv VARCHAR2 + ) IS + BEGIN + for rec in ( + select + ENVIRONMENT_ID + ,REGION + ,NAMESPACE + ,INBOXBUCKETNAME + ,DATABUCKETNAME + ,ARCHIVEBUCKETNAME + ,CREDENTIALNAME + ,LOGGINGENABLED + ,MINLOGLEVEL + ,DEFAULTDATEFORMAT + ,CONSOLELOGGINGENABLED + from ( + select environment_id, config_variable, config_variable_value from CT_MRDS.A_FILE_MANAGER_CONFIG + where environment_id=pEnv + ) + pivot ( + min(config_variable_value) + for config_variable in ( + 'Region' as Region + ,'NameSpace' as NameSpace + ,'InboxBucketName' as InboxBucketName + ,'DataBucketName' as DataBucketName + ,'ArchiveBucketName' as ArchiveBucketName + ,'CredentialName' as CredentialName + ,'LoggingEnabled' as LoggingEnabled + ,'MinLogLevel' as MinLogLevel + ,'DefaultDateFormat' as DefaultDateFormat + ,'ConsoleLoggingEnabled' as ConsoleLoggingEnabled) + ) + ) loop + if (rec.NAMESPACE is NULL + or rec.REGION is NULL + or rec.NAMESPACE is NULL + or rec.INBOXBUCKETNAME is NULL + or rec.DATABUCKETNAME is NULL + or rec.ARCHIVEBUCKETNAME is NULL + or rec.CREDENTIALNAME is NULL + ) THEN + vgMsgTmp := MSG_CONFIG_VARIABLE_NOT_SET + ||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'Region = '||rec.Region + ||cgBL||' '||'NameSpace = '||rec.NameSpace + ||cgBL||' '||'InboxBucketName = '||rec.InboxBucketName + ||cgBL||' '||'DataBucketName = '||rec.DataBucketName + ||cgBL||' '||'ArchiveBucketName = '||rec.ArchiveBucketName + ||cgBL||' '||'CredentialName = '||rec.CredentialName + ; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR'); + RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, vgMsgTmp); + + elsif (rec.LOGGINGENABLED is NULL + or rec.MINLOGLEVEL is NULL + or rec.DEFAULTDATEFORMAT is NULL + ) THEN + vgMsgTmp := 'Missing configuration variables' + ||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'LoggingEnabled = '||rec.LoggingEnabled + ||cgBL||' '||'MinLogLevel = '||rec.MinLogLevel + ||cgBL||' '||'DefaultDateFormat = '||rec.DefaultDateFormat + ; + LOG_PROCESS_EVENT(vgMsgTmp, 'WARNING'); + + else + gvNameSpace := rec.NAMESPACE; + gvRegion := rec.REGION; + gvInboxBucketName := rec.INBOXBUCKETNAME; + gvDataBucketName := rec.DATABUCKETNAME; + gvArchiveBucketName := rec.ARCHIVEBUCKETNAME; + gvCredentialName := rec.CREDENTIALNAME; + gvInboxBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.INBOXBUCKETNAME||'/o/'; + gvDataBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.DATABUCKETNAME||'/o/'; + gvArchiveBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.ARCHIVEBUCKETNAME||'/o/'; + gvLoggingEnabled := rec.LOGGINGENABLED; + gvMinLogLevel := rec.MINLOGLEVEL; + gvDefaultDateFormat := rec.DEFAULTDATEFORMAT; + gvConsoleLoggingEnabled := NVL(rec.CONSOLELOGGINGENABLED, 'ON'); + end if; + end loop; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vgMsgTmp := MSG_CONFIG_VARIABLE_NOT_SET + ||cgBL||' '||'No configuration found for environment_id='||pEnv||' in A_FILE_MANAGER_CONFIG table'; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', 'pEnv='||pEnv); + RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, vgMsgTmp); + WHEN OTHERS THEN + vgMsgTmp := 'Unexpected error while initializing variables for environment: '||pEnv + ||cgBL||' '||'SQLERRM: '||SQLERRM; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', 'pEnv='||pEnv); + RAISE; + END INIT_VARIABLES; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_ERROR_MESSAGE( + pCode PLS_INTEGER + ) RETURN VARCHAR2 + IS + BEGIN + RETURN Errors(pCode).message; + EXCEPTION + WHEN NO_DATA_FOUND THEN + LOG_PROCESS_EVENT('No error message found for pCode='||pCode , 'WARNING', 'pCode='||pCode); + LOG_PROCESS_EVENT('Update ENV_MANAGER package header with new code.' , 'WARNING', 'pCode='||pCode); + RETURN NULL; + WHEN OTHERS THEN + LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pCode='||pCode); + RAISE_APPLICATION_ERROR(CODE_UNKNOWN, MSG_UNKNOWN); + END GET_ERROR_MESSAGE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_ERROR_STACK( + pFormat VARCHAR2 + ,pCode PLS_INTEGER + ,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL + ) RETURN VARCHAR2 + IS + vFullErrorCore VARCHAR2(32000); + vFullErrorMsg VARCHAR2(32000); + BEGIN +-- vgErrorMessage := SQLERRM|| cgBL; +-- vgErrorStack := DBMS_UTILITY.FORMAT_ERROR_STACK; +-- vgErrorBacktrace := DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; + vFullErrorCore :='Error Message:' + ||cgBL|| SQLERRM|| cgBL + ||'-------------------------------------------------------' + ||cgBL||'Error Stack:' + ||cgBL|| DBMS_UTILITY.FORMAT_ERROR_STACK + ||'-------------------------------------------------------' + ||cgBL||'Error Backtrace:' + ||cgBL|| DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; +-- vFullErrorCore := REGEXP_REPLACE (vFullErrorCore, pCode||': ', pCode||': '||GET_ERROR_MESSAGE(pCode) , 1, 1); + IF (pFormat = 'TABLE') THEN + vFullErrorMsg := vFullErrorCore; + ELSE + vFullErrorMsg := cgBL||'------------------------------------------------------+' + ||cgBL||vFullErrorCore + ||'------------------------------------------------------+'; + END IF; +-- IF pSourceFileReceivedKey is not null THEN +-- vFullErrorMsg := vFullErrorMsg ||cgBL||GET_DET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey,1,1,1); +-- END IF; + + RETURN vFullErrorMsg; + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pFormat='||pFormat); + RETURN NULL; + END GET_ERROR_STACK; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FORMAT_PARAMETERS( + pParameterList SYS.ODCIVARCHAR2LIST + ) RETURN VARCHAR2 IS + vResult VARCHAR2(10000); + BEGIN + FOR i IN 1 .. pParameterList.COUNT LOOP +-- dbms_output.put_line('pParameterList(i): '||pParameterList(i)); + if i < pParameterList.COUNT then vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL') ||' ,'|| cgBL; + else vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL'); + end if; + END LOOP; + RETURN vResult; + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_EVENT('Error while formating parameters.' , 'WARNING'); + RETURN NULL; + END FORMAT_PARAMETERS; + + ---------------------------------------------------------------------------------------------------- + + + + PROCEDURE LOG_PROCESS_EVENT ( + pLogMessage VARCHAR2 + ,pLogLevel VARCHAR2 DEFAULT 'ERROR' + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ) IS + PRAGMA AUTONOMOUS_TRANSACTION; + + vLoggingEnabled VARCHAR2(10); + vMinLogLevel VARCHAR2(10); + vCallStack VARCHAR2(10000); + vProcedureName VARCHAR2(100); + vProcedureLevel PLS_INTEGER; + vTotalLines PLS_INTEGER; + vCurrentLine PLS_INTEGER; + + -- Map of priority level + TYPE logLevelMap IS TABLE OF NUMBER INDEX BY VARCHAR2(10); + vLogLevels logLevelMap; + + BEGIN + -- Prority logging level (higher -> more important) + vLogLevels('DEBUG') := 1; + vLogLevels('INFO') := 2; + vLogLevels('WARNING') := 3; + vLogLevels('ERROR') := 4; + + -- Check id logging is TURN-OFF + IF gvLoggingEnabled = 'OFF' THEN + RETURN; + END IF; + -- Check logging level + IF vLogLevels(pLogLevel) < vLogLevels(gvMinLogLevel) THEN + RETURN; + END IF; + + vCallStack := DBMS_UTILITY.FORMAT_CALL_STACK; + vProcedureName := REGEXP_SUBSTR(vCallStack, 'package body\s+\w+\.(\w+\.\w+)', 1, 2, NULL, 1); + vTotalLines := REGEXP_COUNT(vCallStack, CHR(10)) + 1; + vCurrentLine := REGEXP_COUNT(SUBSTR(vCallStack, 1, INSTR(vCallStack, vProcedureName) - 1), CHR(10)) + 1; + vProcedureLevel := (vTotalLines - vCurrentLine + 1) - 3; + vProcedureName := LPAD(vProcedureName, LENGTH(vProcedureName) + 2*vProcedureLevel, ' '); + + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, username, osuser, machine, module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES (guid, gvUsername, gvOsuser, gvMachine, gvModule, pProcessName, vProcedureName, pParameters, pLogLevel, pLogMessage); + + COMMIT; + + -- Also output to console for immediate visibility (if enabled) + IF gvConsoleLoggingEnabled = 'ON' THEN + DBMS_OUTPUT.PUT_LINE('[' || TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') || '] [' || pLogLevel || '] ' || vProcedureName || ': ' || pLogMessage); + END IF; + + END LOG_PROCESS_EVENT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE LOG_PROCESS_ERROR( + pLogMessage IN VARCHAR2, + pParameters IN VARCHAR2 DEFAULT NULL, + pProcessName IN VARCHAR2 DEFAULT 'FILE_MANAGER' + ) IS + PRAGMA AUTONOMOUS_TRANSACTION; + + vCallStack VARCHAR2(32767); + vErrorStack VARCHAR2(32767); + vErrorBacktrace VARCHAR2(32767); + vAdjustedBacktrace VARCHAR2(32767); + vErrorContext VARCHAR2(4000); + vProcName VARCHAR2(100); + vProcedureLevel PLS_INTEGER; + vTotalLines PLS_INTEGER; + vCurrentLine PLS_INTEGER; + vFullErrorMessage CLOB; + vTimestamp VARCHAR2(30); + vSessionInfo VARCHAR2(1000); + + BEGIN + -- Check if logging is disabled + IF gvLoggingEnabled = 'OFF' THEN + RETURN; + END IF; + + -- Capture all available error information + vErrorStack := DBMS_UTILITY.FORMAT_ERROR_STACK; + vErrorBacktrace := DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; + vCallStack := DBMS_UTILITY.FORMAT_CALL_STACK; + vTimestamp := TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'); + + -- Capture session information for better context + vSessionInfo := 'Session ID: ' || SYS_CONTEXT('USERENV', 'SID') || + ', User: ' || SYS_CONTEXT('USERENV', 'SESSION_USER') || + ', Module: ' || SYS_CONTEXT('USERENV', 'MODULE') || + ', Client Info: ' || NVL(SYS_CONTEXT('USERENV', 'CLIENT_INFO'), 'N/A') || + ', Action: ' || NVL(SYS_CONTEXT('USERENV', 'ACTION'), 'N/A'); + + -- Build error context information + vErrorContext := 'Environment: ' || gvEnv || + ', Process: ' || NVL(pProcessName, 'UNKNOWN') || + ', Timestamp: ' || vTimestamp || + ', SQLCODE: ' || SQLCODE || + ', Transaction Active: ' || CASE WHEN DBMS_TRANSACTION.STEP_ID IS NOT NULL THEN 'YES' ELSE 'NO' END; + + -- Extract procedure name and nesting level from call stack + -- Always extract actual procedure name from call stack for precise error location + vProcName := REGEXP_SUBSTR(vCallStack, 'package body\s+\w+\.(\w+\.\w+)', 1, 2, NULL, 1); + + -- If we couldn't extract procedure name from call stack, use provided process name + IF vProcName IS NULL THEN + vProcName := NVL(pProcessName, 'UNKNOWN'); + END IF; + + vTotalLines := REGEXP_COUNT(vCallStack, CHR(10)) + 1; + vCurrentLine := REGEXP_COUNT(SUBSTR(vCallStack, 1, INSTR(vCallStack, vProcName) - 1), CHR(10)) + 1; + vProcedureLevel := (vTotalLines - vCurrentLine + 1) - 3; + vProcName := LPAD(vProcName, LENGTH(vProcName) + 2*vProcedureLevel, ' '); + + -- Enhance line number display to show direct _BODY.sql file line numbers + -- Since packages are now split into separate _SPEC and _BODY files, line numbers map directly + vAdjustedBacktrace := REGEXP_REPLACE(vErrorBacktrace, + 'at "CT_MRDS\.FILE_MANAGER", line ([0-9]+)', + 'at "CT_MRDS.FILE_MANAGER", line \1 (-> FILE_MANAGER_BODY.sql:line \1)', 1, 0, 'i'); + + vAdjustedBacktrace := REGEXP_REPLACE(vAdjustedBacktrace, + 'at "CT_MRDS\.ENV_MANAGER", line ([0-9]+)', + 'at "CT_MRDS.ENV_MANAGER", line \1 (-> ENV_MANAGER_BODY.sql:line \1)', 1, 0, 'i'); + + -- Build comprehensive error message with professional formatting + vFullErrorMessage := 'ERROR REPORT' || cgBL || + '-------------------------------------------------------' || cgBL || + 'ERROR SUMMARY' || cgBL || + ' Message: ' || pLogMessage || cgBL || + ' Context: ' || vErrorContext || cgBL || + '-------------------------------------------------------' || cgBL || + 'SESSION INFORMATION' || cgBL || + ' ' || vSessionInfo || cgBL || + '-------------------------------------------------------' || cgBL || + 'ERROR STACK (Oracle Internal)' || cgBL || + vErrorStack || + '-------------------------------------------------------' || cgBL || + 'BACKTRACE INFORMATION (Oracle Internal)' || cgBL || + vErrorBacktrace || + '-------------------------------------------------------' || cgBL || + 'CALL STACK (Execution Path)' || cgBL || + vCallStack || + '-------------------------------------------------------' || cgBL || + 'QUICK REFERENCE' || cgBL || + ' SQLCODE: ' || SQLCODE || cgBL || + ' SQLERRM: ' || SQLERRM || cgBL || + ' Timestamp: ' || vTimestamp || cgBL || + ' Parameters: ' || NVL(pParameters, 'None provided') || cgBL || + '-------------------------------------------------------'; + + -- Insert comprehensive error record into log table + -- Note: LOG_MESSAGE is VARCHAR2(4000), so we'll truncate if needed but include key info + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, username, osuser, machine, module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES (guid, gvUsername, gvOsuser, gvMachine, gvModule, NVL(pProcessName, 'FILE_MANAGER'), vProcName, pParameters, 'ERROR', + CASE + WHEN LENGTH(vFullErrorMessage) <= 4000 THEN vFullErrorMessage + ELSE SUBSTR(vFullErrorMessage, 1, 3950) || '... [TRUNCATED]' + END); + + COMMIT; + + -- Enhanced console output for immediate visibility (if enabled) + IF gvConsoleLoggingEnabled = 'ON' THEN + DBMS_OUTPUT.PUT_LINE('======================================================='); + DBMS_OUTPUT.PUT_LINE('ERROR DETECTED AT: ' || vTimestamp); + DBMS_OUTPUT.PUT_LINE('PROCEDURE: ' || NVL(vProcName, 'UNKNOWN')); + DBMS_OUTPUT.PUT_LINE('MESSAGE: ' || pLogMessage); + DBMS_OUTPUT.PUT_LINE('SQLCODE: ' || SQLCODE || ' | ENVIRONMENT: ' || gvEnv); + -- Extract and show the most relevant file and line number + IF INSTR(vAdjustedBacktrace, '-> ') > 0 THEN + DBMS_OUTPUT.PUT_LINE('SOURCE FILE LOCATION: ' || REGEXP_SUBSTR(vAdjustedBacktrace, '-> [^)]+')); + END IF; + DBMS_OUTPUT.PUT_LINE('FULL DETAILS: Query A_PROCESS_LOG table for complete diagnostic info'); + DBMS_OUTPUT.PUT_LINE('QUERY (This Error): SELECT * FROM CT_MRDS.A_PROCESS_LOG WHERE GUID = ''' || guid || ''' ORDER BY LOG_TIMESTAMP DESC;'); + DBMS_OUTPUT.PUT_LINE('QUERY (Recent All): SELECT * FROM CT_MRDS.A_PROCESS_LOG WHERE LOG_TIMESTAMP >= SYSDATE - 1/1440 ORDER BY LOG_TIMESTAMP DESC;'); + DBMS_OUTPUT.PUT_LINE('======================================================='); + END IF; + + END LOG_PROCESS_ERROR; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ANALYZE_VALIDATION_ERRORS( + pValidationLogTable VARCHAR2, + pTemplateSchema VARCHAR2, + pTemplateTable VARCHAR2, + pCsvFileUri VARCHAR2 + ) RETURN VARCHAR2 + IS + vAnalysisReport CLOB := ''; + vCsvHeader VARCHAR2(4000); + vExpectedOrder VARCHAR2(4000); + vCsvOrder VARCHAR2(4000); + vErrorDetails VARCHAR2(32000) := ''; + vSolutions VARCHAR2(4000); + vColumnMismatch VARCHAR2(1000); + vErrorCount NUMBER := 0; + vFirstDataError VARCHAR2(1000); + vErrorColumn VARCHAR2(100); + vErrorValue VARCHAR2(500); + vExpectedType VARCHAR2(100); + vTemplateColCount NUMBER := 0; + vCsvColCount NUMBER := 0; + vExcessColumns VARCHAR2(2000); + vCsvFirstLine VARCHAR2(4000); + + -- Cursor for template table columns + CURSOR c_template_columns IS + SELECT COLUMN_NAME, DATA_TYPE, COLUMN_ID + FROM ALL_TAB_COLUMNS + WHERE OWNER = UPPER(REGEXP_SUBSTR(pTemplateSchema || '.' || pTemplateTable, '^([^.]+)')) + AND TABLE_NAME = UPPER(REGEXP_SUBSTR(pTemplateSchema || '.' || pTemplateTable, '\.(.+)$', 1, 1, NULL, 1)) + ORDER BY COLUMN_ID; + + BEGIN + -- Build expected column order from template table and count columns + FOR rec IN c_template_columns LOOP + IF vExpectedOrder IS NOT NULL THEN + vExpectedOrder := vExpectedOrder || ', '; + END IF; + vExpectedOrder := vExpectedOrder || rec.COLUMN_NAME; + vTemplateColCount := vTemplateColCount + 1; + END LOOP; + + -- Parse validation log table for errors and CSV structure + BEGIN + -- Try to extract error information from the validation log table + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ' || pValidationLogTable || + ' WHERE record LIKE ''error processing column%''' + INTO vErrorCount; + + -- Get first error details + IF vErrorCount > 0 THEN + EXECUTE IMMEDIATE 'SELECT record FROM ' || pValidationLogTable || + ' WHERE record LIKE ''error processing column%'' AND ROWNUM = 1' + INTO vFirstDataError; + + -- Parse error to extract column name and error type + vErrorColumn := REGEXP_SUBSTR(vFirstDataError, 'error processing column ([A-Z_]+)', 1, 1, NULL, 1); + + -- Try to get the actual error value from ORA-01722 message + BEGIN + EXECUTE IMMEDIATE 'SELECT record FROM ' || pValidationLogTable || + ' WHERE record LIKE ''ORA-01722%'' AND ROWNUM = 1' + INTO vFirstDataError; + vErrorValue := REGEXP_SUBSTR(vFirstDataError, 'string value containing ''([^'']+)''', 1, 1, NULL, 1); + EXCEPTION + WHEN NO_DATA_FOUND THEN + vErrorValue := 'unknown value'; + WHEN OTHERS THEN + vErrorValue := 'parsing error'; + END; + END IF; + + -- Try to extract CSV structure from validation log field definitions + BEGIN + EXECUTE IMMEDIATE ' + SELECT LISTAGG( + REGEXP_SUBSTR(record, ''^\s+([A-Z_]+)\s+'', 1, 1, NULL, 1), + '', '' + ) WITHIN GROUP (ORDER BY ROWNUM) + FROM ' || pValidationLogTable || ' + WHERE record LIKE '' %CHAR%'' + AND record NOT LIKE ''%Fields in Data Source%'' + AND REGEXP_SUBSTR(record, ''^\s+([A-Z_]+)\s+'') IS NOT NULL' + INTO vCsvOrder; + + -- Count CSV columns from parsed structure + IF vCsvOrder IS NOT NULL THEN + vCsvColCount := REGEXP_COUNT(vCsvOrder, ',') + 1; + END IF; + + EXCEPTION + WHEN OTHERS THEN + vCsvOrder := 'Unable to determine CSV column order from validation log'; + END; + + -- Alternative method: Try to read first line of CSV directly for column count + IF vCsvColCount = 0 THEN + BEGIN + -- This is a fallback - try to get CSV header from external source if possible + -- Note: This would require DBMS_CLOUD.GET_OBJECT or similar approach + -- For now, we'll rely on the validation log parsing + NULL; + EXCEPTION + WHEN OTHERS THEN + NULL; + END; + END IF; + + EXCEPTION + WHEN OTHERS THEN + vErrorDetails := 'Error analyzing validation log: ' || SQLERRM; + END; + + -- Detect column order mismatch and excess columns + IF vCsvOrder IS NOT NULL AND vExpectedOrder IS NOT NULL THEN + IF UPPER(REPLACE(vCsvOrder, ' ', '')) != UPPER(REPLACE(vExpectedOrder, ' ', '')) THEN + vColumnMismatch := 'YES'; + ELSE + vColumnMismatch := 'NO'; + END IF; + END IF; + + -- Check for excess columns + IF vCsvColCount > vTemplateColCount THEN + -- Try to identify which columns are excess + IF vCsvOrder IS NOT NULL THEN + -- Parse CSV columns and compare with template + DECLARE + vCsvCols SYS.ODCIVARCHAR2LIST; + vTemplateCols SYS.ODCIVARCHAR2LIST; + vExcessFound VARCHAR2(1) := 'N'; + i NUMBER; + BEGIN + -- Split CSV columns + SELECT TRIM(REGEXP_SUBSTR(vCsvOrder, '[^,]+', 1, LEVEL)) + BULK COLLECT INTO vCsvCols + FROM DUAL + CONNECT BY REGEXP_SUBSTR(vCsvOrder, '[^,]+', 1, LEVEL) IS NOT NULL; + + -- Split template columns + SELECT TRIM(REGEXP_SUBSTR(vExpectedOrder, '[^,]+', 1, LEVEL)) + BULK COLLECT INTO vTemplateCols + FROM DUAL + CONNECT BY REGEXP_SUBSTR(vExpectedOrder, '[^,]+', 1, LEVEL) IS NOT NULL; + + -- Find excess columns (those in CSV but not in template) + FOR i IN 1..vCsvCols.COUNT LOOP + DECLARE + vFoundInTemplate BOOLEAN := FALSE; + j NUMBER; + BEGIN + -- Check if CSV column exists in template + FOR j IN 1..vTemplateCols.COUNT LOOP + IF UPPER(TRIM(vCsvCols(i))) = UPPER(TRIM(vTemplateCols(j))) THEN + vFoundInTemplate := TRUE; + EXIT; + END IF; + END LOOP; + + -- If not found in template, it's an excess column + IF NOT vFoundInTemplate THEN + IF vExcessFound = 'Y' THEN + vExcessColumns := vExcessColumns || ', '; + END IF; + vExcessColumns := vExcessColumns || vCsvCols(i); + vExcessFound := 'Y'; + END IF; + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + vExcessColumns := 'Unable to determine specific excess columns'; + END; + END IF; + END IF; + + -- Build comprehensive analysis report + vAnalysisReport := 'FILE VALIDATION FAILED - DETAILED ANALYSIS' || cgBL || + '=================================================' || cgBL || cgBL; + + -- Column structure analysis + vAnalysisReport := vAnalysisReport || + 'COLUMN STRUCTURE ANALYSIS:' || cgBL || + '---------------------------------------------------' || cgBL || + 'Template Expected Order: ' || vExpectedOrder || cgBL || + 'Template Column Count: ' || vTemplateColCount || cgBL || + 'CSV Detected Order: ' || NVL(vCsvOrder, 'Unknown') || cgBL || + 'CSV Column Count: ' || vCsvColCount || cgBL || cgBL; + + -- Report column count issues + IF vCsvColCount > vTemplateColCount THEN + vAnalysisReport := vAnalysisReport || + 'EXCESS COLUMNS DETECTED!' || cgBL || + 'CSV file has ' || (vCsvColCount - vTemplateColCount) || ' more columns than template allows.' || cgBL; + IF vExcessColumns IS NOT NULL THEN + vAnalysisReport := vAnalysisReport || + 'Excess columns found: ' || vExcessColumns || cgBL; + END IF; + vAnalysisReport := vAnalysisReport || cgBL; + END IF; + + -- Report column order issues + IF vColumnMismatch = 'YES' THEN + vAnalysisReport := vAnalysisReport || + 'COLUMN ORDER MISMATCH DETECTED!' || cgBL || + 'CSV columns are in different order than template expects.' || cgBL || cgBL; + END IF; + + -- Specific error analysis + IF vErrorCount > 0 THEN + vAnalysisReport := vAnalysisReport || + 'SPECIFIC ERRORS FOUND:' || cgBL || + '---------------------------------------------------' || cgBL; + + IF vErrorColumn IS NOT NULL THEN + -- Get expected data type for error column + FOR rec IN c_template_columns LOOP + IF rec.COLUMN_NAME = vErrorColumn THEN + vExpectedType := rec.DATA_TYPE; + EXIT; + END IF; + END LOOP; + + vAnalysisReport := vAnalysisReport || + '1. Column ' || vErrorColumn || ': Expected ' || vExpectedType || + ', received "' || NVL(vErrorValue, 'unknown value') || '" (TEXT)' || cgBL || + ' → CSV position contains different data type than expected' || cgBL; + END IF; + + vAnalysisReport := vAnalysisReport || + 'Total validation errors found: ' || vErrorCount || cgBL || cgBL; + END IF; + + -- Solutions section + vAnalysisReport := vAnalysisReport || + 'SUGGESTED SOLUTIONS:' || cgBL || + '---------------------------------------------------' || cgBL; + + -- Solutions for excess columns + IF vCsvColCount > vTemplateColCount THEN + vAnalysisReport := vAnalysisReport || + 'FOR EXCESS COLUMNS:' || cgBL || + '• Remove extra columns from CSV file' || cgBL || + '• Keep only these columns in this order: ' || vExpectedOrder || cgBL; + IF vExcessColumns IS NOT NULL THEN + vAnalysisReport := vAnalysisReport || + '• Specifically remove: ' || vExcessColumns || cgBL; + END IF; + vAnalysisReport := vAnalysisReport || cgBL; + END IF; + + -- Solutions for column order + IF vColumnMismatch = 'YES' THEN + vAnalysisReport := vAnalysisReport || + 'FOR COLUMN ORDER:' || cgBL || + '• Reorder CSV columns to match template: ' || vExpectedOrder || cgBL || + '• Or update template table column order to match CSV file' || cgBL || cgBL; + END IF; + + -- General solutions + vAnalysisReport := vAnalysisReport || + 'GENERAL RECOMMENDATIONS:' || cgBL || + '• Ensure CSV has exactly ' || vTemplateColCount || ' columns' || cgBL || + '• Verify column names match template table exactly' || cgBL || + '• Check data types in each column match expectations' || cgBL || cgBL; + + -- Validation log reference + vAnalysisReport := vAnalysisReport || + 'TECHNICAL DETAILS:' || cgBL || + '---------------------------------------------------' || cgBL || + 'Validation Log Table: ' || pValidationLogTable || cgBL || + 'Template Table: ' || pTemplateSchema || '.' || pTemplateTable || cgBL || + 'CSV File: ' || pCsvFileUri || cgBL || + 'Query validation details: SELECT * FROM ' || pValidationLogTable || ';' || cgBL; + + RETURN vAnalysisReport; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error generating validation analysis: ' || SQLERRM || cgBL || + 'Validation Log Table: ' || pValidationLogTable || cgBL || + 'Check table manually: SELECT * FROM ' || pValidationLogTable || ';'; + END ANALYZE_VALIDATION_ERRORS; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN GET_PACKAGE_VERSION_INFO( + pPackageName => 'ENV_MANAGER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN FORMAT_VERSION_HISTORY( + pPackageName => 'ENV_MANAGER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_PACKAGE_VERSION_INFO( + pPackageName VARCHAR2, + pVersion VARCHAR2, + pBuildDate VARCHAR2, + pAuthor VARCHAR2 + ) RETURN VARCHAR2 + IS + BEGIN + RETURN 'Package: ' || pPackageName || cgBL || + 'Version: ' || pVersion || cgBL || + 'Build Date: ' || pBuildDate || cgBL || + 'Author: ' || pAuthor; + END GET_PACKAGE_VERSION_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FORMAT_VERSION_HISTORY( + pPackageName VARCHAR2, + pVersionHistory VARCHAR2 + ) RETURN VARCHAR2 + IS + BEGIN + RETURN pPackageName || ' Version History:' || cgBL || pVersionHistory; + END FORMAT_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE HASH + CHANGE DETECTION FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION CALCULATE_PACKAGE_HASH( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageType VARCHAR2 + ) RETURN VARCHAR2 + IS + vSourceCode CLOB; + vHash VARCHAR2(64); + vRawHash RAW(32); + BEGIN + -- Build complete source code from ALL_SOURCE using XMLAGG (no 4000 char limit) + -- CRITICAL: Cannot use LISTAGG due to VARCHAR2 limit + SELECT XMLAGG(XMLELEMENT(E, TEXT) ORDER BY LINE).GETCLOBVAL() + INTO vSourceCode + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = UPPER(pPackageType); + + -- If empty, return NULL + IF vSourceCode IS NULL OR DBMS_LOB.GETLENGTH(vSourceCode) = 0 THEN + RETURN NULL; + END IF; + + -- Calculate SHA256 hash directly from CLOB + -- DBMS_CRYPTO.HASH has overload for CLOB in Oracle 19c+ + vRawHash := DBMS_CRYPTO.HASH( + src => vSourceCode, + typ => DBMS_CRYPTO.HASH_SH256 + ); + + -- Convert to hex string + vHash := LOWER(RAWTOHEX(vRawHash)); + + RETURN vHash; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + WHEN OTHERS THEN + LOG_PROCESS_ERROR('Error calculating package hash: ' || SQLERRM, + 'pPackageOwner=' || pPackageOwner || ', pPackageName=' || pPackageName); + RETURN NULL; + END CALCULATE_PACKAGE_HASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE TRACK_PACKAGE_VERSION( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageVersion VARCHAR2, + pPackageBuildDate VARCHAR2, + pPackageAuthor VARCHAR2 + ) + IS + vHashSpec VARCHAR2(64); + vHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLineCountSpec NUMBER; + vLineCountBody NUMBER; + vChangeDetected CHAR(1) := 'N'; + vChangeMessage VARCHAR2(4000); + vParameters VARCHAR2(4000); + BEGIN + vParameters := FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pPackageOwner => ''' || pPackageOwner || '''', + 'pPackageName => ''' || pPackageName || '''', + 'pPackageVersion => ''' || pPackageVersion || '''' + )); + + LOG_PROCESS_EVENT('Start TRACK_PACKAGE_VERSION', 'INFO', vParameters); + + -- Calculate current hashes + vHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get line counts + BEGIN + SELECT COUNT(*) + INTO vLineCountSpec + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = 'PACKAGE'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vLineCountSpec := 0; + END; + + BEGIN + SELECT COUNT(*) + INTO vLineCountBody + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = 'PACKAGE BODY'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vLineCountBody := 0; + END; + + -- Get last tracked version and hashes + BEGIN + SELECT PACKAGE_VERSION, SOURCE_CODE_HASH_SPEC, SOURCE_CODE_HASH_BODY + INTO vLastVersion, vLastHashSpec, vLastHashBody + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + + -- Check if hash changed but version didn't + IF (vHashSpec != vLastHashSpec OR NVL(vHashBody,'X') != NVL(vLastHashBody,'X')) + AND pPackageVersion = vLastVersion THEN + + vChangeDetected := 'Y'; + vChangeMessage := 'WARNING: Source code changed without version update!' || cgBL || + 'Last Version: ' || vLastVersion || cgBL || + 'Current Version: ' || pPackageVersion || cgBL; + + IF vHashSpec != vLastHashSpec THEN + vChangeMessage := vChangeMessage || + 'SPEC Changed - Hash: ' || SUBSTR(vHashSpec, 1, 16) || '... (was: ' || + SUBSTR(vLastHashSpec, 1, 16) || '...)' || cgBL; + END IF; + + IF NVL(vHashBody,'X') != NVL(vLastHashBody,'X') THEN + vChangeMessage := vChangeMessage || + 'BODY Changed - Hash: ' || SUBSTR(vHashBody, 1, 16) || '... (was: ' || + SUBSTR(NVL(vLastHashBody,'NULL'), 1, 16) || '...)' || cgBL; + END IF; + + vChangeMessage := vChangeMessage || + 'RECOMMENDATION: Update PACKAGE_VERSION constant and PACKAGE_BUILD_DATE'; + + LOG_PROCESS_EVENT(vChangeMessage, 'WARNING', vParameters); + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + -- First time tracking this package + vChangeDetected := 'N'; + vChangeMessage := 'First tracking record for this package'; + LOG_PROCESS_EVENT(vChangeMessage, 'INFO', vParameters); + END; + + -- Insert tracking record + INSERT INTO CT_MRDS.A_PACKAGE_VERSION_TRACKING ( + PACKAGE_OWNER, + PACKAGE_NAME, + PACKAGE_TYPE, + PACKAGE_VERSION, + PACKAGE_BUILD_DATE, + PACKAGE_AUTHOR, + SOURCE_CODE_HASH_SPEC, + SOURCE_CODE_HASH_BODY, + LINE_COUNT_SPEC, + LINE_COUNT_BODY, + DETECTED_CHANGE_WITHOUT_VERSION, + CHANGE_DETECTION_MESSAGE + ) VALUES ( + UPPER(pPackageOwner), + UPPER(pPackageName), + 'BOTH', + pPackageVersion, + pPackageBuildDate, + pPackageAuthor, + vHashSpec, + vHashBody, + vLineCountSpec, + vLineCountBody, + vChangeDetected, + vChangeMessage + ); + + COMMIT; + + LOG_PROCESS_EVENT('End TRACK_PACKAGE_VERSION - Record inserted', 'INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_ERROR('Error in TRACK_PACKAGE_VERSION: ' || SQLERRM, vParameters); + RAISE; + END TRACK_PACKAGE_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION CHECK_PACKAGE_CHANGES( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2 + IS + vCurrentHashSpec VARCHAR2(64); + vCurrentHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLastTrackingDate TIMESTAMP; + vChangeReport VARCHAR2(4000); + vSpecChanged BOOLEAN := FALSE; + vBodyChanged BOOLEAN := FALSE; + BEGIN + -- Get current hashes + vCurrentHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vCurrentHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get last tracked hashes + BEGIN + SELECT PACKAGE_VERSION, SOURCE_CODE_HASH_SPEC, SOURCE_CODE_HASH_BODY, TRACKING_DATE + INTO vLastVersion, vLastHashSpec, vLastHashBody, vLastTrackingDate + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN 'Package ' || pPackageOwner || '.' || pPackageName || ' has never been tracked.' || cgBL || + 'Run TRACK_PACKAGE_VERSION to establish baseline.'; + END; + + -- Check for changes + IF vCurrentHashSpec != vLastHashSpec THEN + vSpecChanged := TRUE; + END IF; + + IF NVL(vCurrentHashBody, 'X') != NVL(vLastHashBody, 'X') THEN + vBodyChanged := TRUE; + END IF; + + -- Build report + IF vSpecChanged OR vBodyChanged THEN + vChangeReport := 'WARNING: Package ' || pPackageOwner || '.' || pPackageName || ' has changed!' || cgBL || + '========================================' || cgBL || + 'Last Tracked Version: ' || vLastVersion || cgBL || + 'Last Tracked Date: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + cgBL; + + IF vSpecChanged THEN + vChangeReport := vChangeReport || + 'SPECIFICATION Changed:' || cgBL || + ' Current Hash: ' || SUBSTR(vCurrentHashSpec, 1, 16) || '...' || cgBL || + ' Last Hash: ' || SUBSTR(vLastHashSpec, 1, 16) || '...' || cgBL || + cgBL; + END IF; + + IF vBodyChanged THEN + vChangeReport := vChangeReport || + 'BODY Changed:' || cgBL || + ' Current Hash: ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 16) || '...' || cgBL || + ' Last Hash: ' || SUBSTR(NVL(vLastHashBody, 'NULL'), 1, 16) || '...' || cgBL || + cgBL; + END IF; + + vChangeReport := vChangeReport || + 'RECOMMENDATION:' || cgBL || + '1. Update PACKAGE_VERSION constant' || cgBL || + '2. Update PACKAGE_BUILD_DATE constant' || cgBL || + '3. Add entry to VERSION_HISTORY' || cgBL || + '4. Call TRACK_PACKAGE_VERSION to update tracking'; + ELSE + vChangeReport := 'OK: Package ' || pPackageOwner || '.' || pPackageName || ' has not changed.' || cgBL || + 'Last Tracked: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + 'Version: ' || vLastVersion; + END IF; + + RETURN vChangeReport; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error checking package changes: ' || SQLERRM; + END CHECK_PACKAGE_CHANGES; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_PACKAGE_HASH_INFO( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2 + IS + vCurrentHashSpec VARCHAR2(64); + vCurrentHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLastTrackingDate TIMESTAMP; + vLastChangeDetected CHAR(1); + vInfo VARCHAR2(4000); + BEGIN + -- Get current hashes + vCurrentHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vCurrentHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get last tracking info + BEGIN + SELECT PACKAGE_VERSION, + SOURCE_CODE_HASH_SPEC, + SOURCE_CODE_HASH_BODY, + TRACKING_DATE, + DETECTED_CHANGE_WITHOUT_VERSION + INTO vLastVersion, vLastHashSpec, vLastHashBody, vLastTrackingDate, vLastChangeDetected + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN 'Package: ' || pPackageOwner || '.' || pPackageName || cgBL || + 'Status: Never tracked' || cgBL || + 'Current Hash (SPEC): ' || SUBSTR(vCurrentHashSpec, 1, 16) || '...' || cgBL || + 'Current Hash (BODY): ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 16) || '...'; + END; + + -- Build info report + vInfo := 'Package: ' || pPackageOwner || '.' || pPackageName || cgBL || + 'Current Version: ' || vLastVersion || cgBL || + 'Last Tracked: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + cgBL || + 'Current Hash (SPEC): ' || SUBSTR(vCurrentHashSpec, 1, 32) || '...' || cgBL || + 'Last Hash (SPEC): ' || SUBSTR(vLastHashSpec, 1, 32) || '...' || cgBL; + + IF vCurrentHashBody IS NOT NULL OR vLastHashBody IS NOT NULL THEN + vInfo := vInfo || + 'Current Hash (BODY): ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 32) || '...' || cgBL || + 'Last Hash (BODY): ' || SUBSTR(NVL(vLastHashBody, 'NULL'), 1, 32) || '...' || cgBL; + END IF; + + vInfo := vInfo || cgBL; + + -- Status + IF vCurrentHashSpec = vLastHashSpec AND NVL(vCurrentHashBody, 'X') = NVL(vLastHashBody, 'X') THEN + vInfo := vInfo || 'Status: OK - No changes detected'; + ELSE + vInfo := vInfo || 'Status: CHANGED - Source code modified since last tracking'; + END IF; + + IF vLastChangeDetected = 'Y' THEN + vInfo := vInfo || cgBL || 'Last Tracking Warning: Change detected without version update'; + END IF; + + RETURN vInfo; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error getting package hash info: ' || SQLERRM; + END GET_PACKAGE_HASH_INFO; + + ---------------------------------------------------------------------------------------------------- + +BEGIN + INIT_ERRORS; + guid := sys_guid(); + gvUsername := SYS_CONTEXT('USERENV', 'SESSION_USER'); + gvOsuser := SYS_CONTEXT('USERENV', 'OS_USER'); + gvMachine := SYS_CONTEXT('USERENV', 'HOST'); + gvModule := SYS_CONTEXT('USERENV', 'MODULE'); + + -- Get info about EnvironmentID. Without it package cannot proceed further. + -- Information about environment is needed to get proper configuration values + -- It can be set up in two different ways : + -- 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => 'dev'); + -- 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values ('default','environment_id','dev'); + -- Session level setup (1.) takes precedence over configuration level one (2.) + + gvEnv := nvl(SYS_CONTEXT ('USERENV', 'CLIENT_IDENTIFIER'), GET_DEFAULT_ENV()); + if gvEnv is null then + dbms_output.put_line(MSG_ENVIRONMENT_NOT_SET); + LOG_PROCESS_EVENT(MSG_ENVIRONMENT_NOT_SET, 'ERROR'); + RAISE_APPLICATION_ERROR(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET); + else + dbms_output.put_line('EnvironmentID set to: '||gvEnv); + end if; + + INIT_VARIABLES(pEnv => gvEnv); +END ENV_MANAGER; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkg b/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkg new file mode 100644 index 0000000..fded944 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/ENV_MANAGER.pkg @@ -0,0 +1,625 @@ +create or replace PACKAGE CT_MRDS.ENV_MANAGER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select ENV_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-12-20 10:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.2.0 (2025-12-20): Added error codes for parallel execution support (CODE_INVALID_PARALLEL_DEGREE -20110, CODE_PARALLEL_EXECUTION_FAILED -20111)' || CHR(13)||CHR(10) || + '3.1.0 (2025-10-22): Added package hash tracking and automatic change detection system (SHA256 hashing)' || CHR(13)||CHR(10) || + '3.0.0 (2025-10-22): Added package versioning system with centralized version management functions' || CHR(13)||CHR(10) || + '2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function for comprehensive CSV validation analysis' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-01): Added LOG_PROCESS_ERROR procedure with enhanced error diagnostics and stack traces' || CHR(13)||CHR(10) || + '1.5.0 (2025-09-20): Added console logging support with gvConsoleLoggingEnabled configuration' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-01): Initial release with error management and configuration system'; + + TYPE Error_Record IS RECORD ( + code PLS_INTEGER, + message VARCHAR2(4000) + ); + + TYPE tErrorList IS TABLE OF Error_Record INDEX BY PLS_INTEGER; + + Errors tErrorList; + + + guid VARCHAR2(32); + gvEnv VARCHAR2(200); + gvUsername VARCHAR2(128); + gvOsuser VARCHAR2(128); + gvMachine VARCHAR2(64); + gvModule VARCHAR2(64); + + gvNameSpace VARCHAR2(200); + gvRegion VARCHAR2(200); + gvDataBucketName VARCHAR2(200); + gvInboxBucketName VARCHAR2(200); + gvArchiveBucketName VARCHAR2(200); + gvDataBucketUri VARCHAR2(200); + gvInboxBucketUri VARCHAR2(200); + gvArchiveBucketUri VARCHAR2(200); + gvCredentialName VARCHAR2(200); + + -- Overwritten by variable "LoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF' + + -- Overwritten by variable "MinLogLevel" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + -- Possible values: DEBUG ,INFO ,WARNING ,ERROR + gvMinLogLevel VARCHAR2(10) := 'DEBUG'; + + -- Overwritten by variable "DefaultDateFormat" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvDefaultDateFormat VARCHAR2(200) := 'DD/MM/YYYY HH24:MI:SS'; + + -- Overwritten by variable "ConsoleLoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvConsoleLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF' + + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + + vgSourceFileConfigKey PLS_INTEGER; + + vgMsgTmp VARCHAR2(32000); + --Exceptions + ERR_EMPTY_FILEURI_AND_RECKEY EXCEPTION; + CODE_EMPTY_FILEURI_AND_RECKEY CONSTANT PLS_INTEGER := -20001; + MSG_EMPTY_FILEURI_AND_RECKEY VARCHAR2(4000) := 'Either pFileUri or pSourceFileReceivedKey must be not null'; + PRAGMA EXCEPTION_INIT( ERR_EMPTY_FILEURI_AND_RECKEY + ,CODE_EMPTY_FILEURI_AND_RECKEY); + + + ERR_NO_CONFIG_MATCH_FOR_FILEURI EXCEPTION; + CODE_NO_CONFIG_MATCH_FOR_FILEURI CONSTANT PLS_INTEGER := -20002; + MSG_NO_CONFIG_MATCH_FOR_FILEURI VARCHAR2(4000) := 'No match for source file in A_SOURCE_FILE_CONFIG table' + ||cgBL||' The file provided in parameter: pFileUri does not have ' + ||cgBL||' coresponding configuration in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH_FOR_FILEURI + ,CODE_NO_CONFIG_MATCH_FOR_FILEURI); + + ERR_MULTIPLE_MATCH_FOR_SRCFILE EXCEPTION; + CODE_MULTIPLE_MATCH_FOR_SRCFILE CONSTANT PLS_INTEGER := -20003; + MSG_MULTIPLE_MATCH_FOR_SRCFILE VARCHAR2(4000) := 'Multiple match for source file in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_MATCH_FOR_SRCFILE + ,CODE_MULTIPLE_MATCH_FOR_SRCFILE); + + ERR_MISSING_COLUMN_DATE_FORMAT EXCEPTION; + CODE_MISSING_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20004; + MSG_MISSING_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Missing entry in config table: A_COLUMN_DATE_FORMAT primary key(TEMPLATE_TABLE_NAME, COLUMN_NAME)' + ||cgBL||' Remember: each column which data_type IN (''DATE'', ''TIMESTAMP'')' + ||cgBL||' should have DateFormat specified in A_COLUMN_DATE_FORMAT table ' + ||cgBL||' for example: ''YYYY-MM-DD'''; + PRAGMA EXCEPTION_INIT( ERR_MISSING_COLUMN_DATE_FORMAT + ,CODE_MISSING_COLUMN_DATE_FORMAT); + + ERR_MULTIPLE_COLUMN_DATE_FORMAT EXCEPTION; + CODE_MULTIPLE_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20005; + MSG_MULTIPLE_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Multiple records for date format in A_COLUMN_DATE_FORMAT table' + ||cgBL||' There should be only one format specified for each DAT/TIMESTAMP column'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_COLUMN_DATE_FORMAT + ,CODE_MULTIPLE_COLUMN_DATE_FORMAT); + + + ERR_DIDNT_GET_LOAD_OPERATION_ID EXCEPTION; + CODE_DIDNT_GET_LOAD_OPERATION_ID CONSTANT PLS_INTEGER := -20006; + MSG_DIDNT_GET_LOAD_OPERATION_ID VARCHAR2(4000) := 'Didnt get load operation id from external table validation'; + PRAGMA EXCEPTION_INIT( ERR_DIDNT_GET_LOAD_OPERATION_ID + ,CODE_DIDNT_GET_LOAD_OPERATION_ID); + + ERR_NO_CONFIG_FOR_RECEIVED_FILE EXCEPTION; + CODE_NO_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20007; + MSG_NO_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'No match for received source file in A_SOURCE_FILE_CONFIG ' + ||cgBL||' or missing data in A_SOURCE_FILE_RECEIVED table for provided pSourceFileReceivedKey parameter'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_FOR_RECEIVED_FILE + ,CODE_NO_CONFIG_FOR_RECEIVED_FILE); + + ERR_MULTI_CONFIG_FOR_RECEIVED_FILE EXCEPTION; + CODE_MULTI_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20008; + MSG_MULTI_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'Multiple matchs for received source file in A_SOURCE_FILE_CONFIG'; + PRAGMA EXCEPTION_INIT( ERR_MULTI_CONFIG_FOR_RECEIVED_FILE + ,CODE_MULTI_CONFIG_FOR_RECEIVED_FILE); + + ERR_FILE_NOT_FOUND_ON_CLOUD EXCEPTION; + CODE_FILE_NOT_FOUND_ON_CLOUD CONSTANT PLS_INTEGER := -20009; + MSG_FILE_NOT_FOUND_ON_CLOUD VARCHAR2(4000) := 'File not found on the cloud'; + PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_FOUND_ON_CLOUD + ,CODE_FILE_NOT_FOUND_ON_CLOUD); + + ERR_FILE_VALIDATION_FAILED EXCEPTION; + CODE_FILE_VALIDATION_FAILED CONSTANT PLS_INTEGER := -20010; + MSG_FILE_VALIDATION_FAILED VARCHAR2(4000) := 'File validation failed'; + PRAGMA EXCEPTION_INIT( ERR_FILE_VALIDATION_FAILED + ,CODE_FILE_VALIDATION_FAILED); + + ERR_EXCESS_COLUMNS_DETECTED EXCEPTION; + CODE_EXCESS_COLUMNS_DETECTED CONSTANT PLS_INTEGER := -20011; + MSG_EXCESS_COLUMNS_DETECTED VARCHAR2(4000) := 'CSV file contains more columns than template allows'; + PRAGMA EXCEPTION_INIT( ERR_EXCESS_COLUMNS_DETECTED + ,CODE_EXCESS_COLUMNS_DETECTED); + + ERR_NO_CONFIG_MATCH EXCEPTION; + CODE_NO_CONFIG_MATCH CONSTANT PLS_INTEGER := -20012; + MSG_NO_CONFIG_MATCH VARCHAR2(4000) := 'No match for specified parameters in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH + ,CODE_NO_CONFIG_MATCH); + + ERR_UNKNOWN_PREFIX EXCEPTION; + CODE_UNKNOWN_PREFIX CONSTANT PLS_INTEGER := -20013; + MSG_UNKNOWN_PREFIX VARCHAR2(4000) := 'Unknown prefix'; + PRAGMA EXCEPTION_INIT( ERR_UNKNOWN_PREFIX + ,CODE_UNKNOWN_PREFIX); + + ERR_TABLE_NOT_EXISTS EXCEPTION; + CODE_TABLE_NOT_EXISTS CONSTANT PLS_INTEGER := -20014; + MSG_TABLE_NOT_EXISTS VARCHAR2(4000) := 'Table does not exist'; + PRAGMA EXCEPTION_INIT( ERR_TABLE_NOT_EXISTS + ,CODE_TABLE_NOT_EXISTS); + + ERR_COLUMN_NOT_EXISTS EXCEPTION; + CODE_COLUMN_NOT_EXISTS CONSTANT PLS_INTEGER := -20015; + MSG_COLUMN_NOT_EXISTS VARCHAR2(4000) := 'Column does not exist in table'; + PRAGMA EXCEPTION_INIT( ERR_COLUMN_NOT_EXISTS + ,CODE_COLUMN_NOT_EXISTS); + + ERR_UNSUPPORTED_DATA_TYPE EXCEPTION; + CODE_UNSUPPORTED_DATA_TYPE CONSTANT PLS_INTEGER := -20016; + MSG_UNSUPPORTED_DATA_TYPE VARCHAR2(4000) := 'Unsupported data type'; + PRAGMA EXCEPTION_INIT( ERR_UNSUPPORTED_DATA_TYPE + ,CODE_UNSUPPORTED_DATA_TYPE); + + ERR_MISSING_SOURCE_KEY EXCEPTION; + CODE_MISSING_SOURCE_KEY CONSTANT PLS_INTEGER := -20017; + MSG_MISSING_SOURCE_KEY VARCHAR2(4000) := 'The Source was not found in parent table A_SOURCE'; + PRAGMA EXCEPTION_INIT( ERR_MISSING_SOURCE_KEY + ,CODE_MISSING_SOURCE_KEY); + + ERR_NULL_SOURCE_FILE_CONFIG_KEY EXCEPTION; + CODE_NULL_SOURCE_FILE_CONFIG_KEY CONSTANT PLS_INTEGER := -20018; + MSG_NULL_SOURCE_FILE_CONFIG_KEY VARCHAR2(4000) := 'No entry in A_SOURCE_FILE_CONFIG table for specified A_SOURCE_FILE_CONFIG_KEY'; + PRAGMA EXCEPTION_INIT( ERR_NULL_SOURCE_FILE_CONFIG_KEY + ,CODE_NULL_SOURCE_FILE_CONFIG_KEY); + + ERR_DUPLICATED_SOURCE_KEY EXCEPTION; + CODE_DUPLICATED_SOURCE_KEY CONSTANT PLS_INTEGER := -20019; + MSG_DUPLICATED_SOURCE_KEY VARCHAR2(4000) := 'The Source already exists in the A_SOURCE table'; + PRAGMA EXCEPTION_INIT( ERR_DUPLICATED_SOURCE_KEY + ,CODE_DUPLICATED_SOURCE_KEY); + + ERR_MISSING_CONTAINER_CONFIG EXCEPTION; + CODE_MISSING_CONTAINER_CONFIG CONSTANT PLS_INTEGER := -20020; + MSG_MISSING_CONTAINER_CONFIG VARCHAR2(4000) := 'No match in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID'; + PRAGMA EXCEPTION_INIT( ERR_MISSING_CONTAINER_CONFIG + ,CODE_MISSING_CONTAINER_CONFIG); + + ERR_MULTIPLE_CONTAINER_ENTRIES EXCEPTION; + CODE_MULTIPLE_CONTAINER_ENTRIES CONSTANT PLS_INTEGER := -20021; + MSG_MULTIPLE_CONTAINER_ENTRIES VARCHAR2(4000) := 'Multiple matches in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_CONTAINER_ENTRIES + ,CODE_MULTIPLE_CONTAINER_ENTRIES); + + ERR_WRONG_DESTINATION_PARAM EXCEPTION; + CODE_WRONG_DESTINATION_PARAM CONSTANT PLS_INTEGER := -20022; + MSG_WRONG_DESTINATION_PARAM VARCHAR2(4000) := 'Wrong destination parameter provided.'; + PRAGMA EXCEPTION_INIT( ERR_WRONG_DESTINATION_PARAM + ,CODE_WRONG_DESTINATION_PARAM); + + ERR_FILE_NOT_EXISTS_ON_CLOUD EXCEPTION; + CODE_FILE_NOT_EXISTS_ON_CLOUD CONSTANT PLS_INTEGER := -20023; + MSG_FILE_NOT_EXISTS_ON_CLOUD VARCHAR2(4000) := 'File not exists on cloud.'; + PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_EXISTS_ON_CLOUD + ,CODE_FILE_NOT_EXISTS_ON_CLOUD); + + ERR_FILE_ALREADY_REGISTERED EXCEPTION; + CODE_FILE_ALREADY_REGISTERED CONSTANT PLS_INTEGER := -20024; + MSG_FILE_ALREADY_REGISTERED VARCHAR2(4000) := 'File already registered in A_SOURCE_FILE_RECEIVED table.'; + PRAGMA EXCEPTION_INIT( ERR_FILE_ALREADY_REGISTERED + ,CODE_FILE_ALREADY_REGISTERED); + + ERR_WRONG_DATE_TIMESTAMP_FORMAT EXCEPTION; + CODE_WRONG_DATE_TIMESTAMP_FORMAT CONSTANT PLS_INTEGER := -20025; + MSG_WRONG_DATE_TIMESTAMP_FORMAT VARCHAR2(4000) := 'Provided DATE or TIMESTAMP format has errors (possible duplicated codes, ex: ''DD'').'; + PRAGMA EXCEPTION_INIT( ERR_WRONG_DATE_TIMESTAMP_FORMAT + ,CODE_WRONG_DATE_TIMESTAMP_FORMAT); + + ERR_ENVIRONMENT_NOT_SET EXCEPTION; + CODE_ENVIRONMENT_NOT_SET CONSTANT PLS_INTEGER := -20026; + MSG_ENVIRONMENT_NOT_SET VARCHAR2(4000) := 'EnvironmentID not set' + ||cgBL||' Information about environment is needed to get proper configuration values.' + ||cgBL||' It can be set up in two different ways:' + ||cgBL||' 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => ''dev'')' + ||cgBL||' 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values (''default'',''environment_id'',''dev'')' + ||cgBL||' Session level setup (1.) takes precedence over configuration level one (2.)' + ; + PRAGMA EXCEPTION_INIT( ERR_ENVIRONMENT_NOT_SET + ,CODE_ENVIRONMENT_NOT_SET); + + + ERR_CONFIG_VARIABLE_NOT_SET EXCEPTION; + CODE_CONFIG_VARIABLE_NOT_SET CONSTANT PLS_INTEGER := -20027; + MSG_CONFIG_VARIABLE_NOT_SET VARCHAR2(4000) := 'Missing configuration value in A_FILE_MANAGER_CONFIG'; + PRAGMA EXCEPTION_INIT( ERR_CONFIG_VARIABLE_NOT_SET + ,CODE_CONFIG_VARIABLE_NOT_SET); + + ERR_NOT_INPUT_SOURCE_FILE_TYPE EXCEPTION; + CODE_NOT_INPUT_SOURCE_FILE_TYPE CONSTANT PLS_INTEGER := -20028; + MSG_NOT_INPUT_SOURCE_FILE_TYPE VARCHAR2(4000) := 'Archival can be executed only for A_SOURCE_FILE_CONFIG_KEY where SOURCE_FILE_TYPE=''INPUT'''; + PRAGMA EXCEPTION_INIT( ERR_NOT_INPUT_SOURCE_FILE_TYPE + ,CODE_NOT_INPUT_SOURCE_FILE_TYPE); + + ERR_EXP_DATA_FOR_ARCH_FAILED EXCEPTION; + CODE_EXP_DATA_FOR_ARCH_FAILED CONSTANT PLS_INTEGER := -20029; + MSG_EXP_DATA_FOR_ARCH_FAILED VARCHAR2(4000) := 'Export data for archival failed.'; + PRAGMA EXCEPTION_INIT( ERR_EXP_DATA_FOR_ARCH_FAILED + ,CODE_EXP_DATA_FOR_ARCH_FAILED); + + ERR_RESTORE_FILE_FROM_TRASH EXCEPTION; + CODE_RESTORE_FILE_FROM_TRASH CONSTANT PLS_INTEGER := -20030; + MSG_RESTORE_FILE_FROM_TRASH VARCHAR2(4000) := 'Unexpected issues occured while archival process. Restoration of exported files failed.'; + PRAGMA EXCEPTION_INIT( ERR_RESTORE_FILE_FROM_TRASH + ,CODE_RESTORE_FILE_FROM_TRASH); + + ERR_CHANGE_STAT_TO_ARCHIVED_FAILED EXCEPTION; + CODE_CHANGE_STAT_TO_ARCHIVED_FAILED CONSTANT PLS_INTEGER := -20031; + MSG_CHANGE_STAT_TO_ARCHIVED_FAILED VARCHAR2(4000) := 'Failed to change file status to: ARCHIVED in A_SOURCE_FILE_RECEIVED table.'; + PRAGMA EXCEPTION_INIT( ERR_CHANGE_STAT_TO_ARCHIVED_FAILED + ,CODE_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ERR_MOVE_FILE_TO_TRASH_FAILED EXCEPTION; + CODE_MOVE_FILE_TO_TRASH_FAILED CONSTANT PLS_INTEGER := -20032; + MSG_MOVE_FILE_TO_TRASH_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.'; + PRAGMA EXCEPTION_INIT( ERR_MOVE_FILE_TO_TRASH_FAILED + ,CODE_MOVE_FILE_TO_TRASH_FAILED); + + ERR_DROP_EXPORTED_FILES_FAILED EXCEPTION; + CODE_DROP_EXPORTED_FILES_FAILED CONSTANT PLS_INTEGER := -20033; + MSG_DROP_EXPORTED_FILES_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.'; + PRAGMA EXCEPTION_INIT( ERR_DROP_EXPORTED_FILES_FAILED + ,CODE_DROP_EXPORTED_FILES_FAILED); + + ERR_INVALID_BUCKET_AREA EXCEPTION; + CODE_INVALID_BUCKET_AREA CONSTANT PLS_INTEGER := -20034; + MSG_INVALID_BUCKET_AREA VARCHAR2(4000) := 'Invalid bucket area specified. Valid values: INBOX, ODS, DATA, ARCHIVE'; + PRAGMA EXCEPTION_INIT( ERR_INVALID_BUCKET_AREA + ,CODE_INVALID_BUCKET_AREA); + + ERR_INVALID_PARALLEL_DEGREE EXCEPTION; + CODE_INVALID_PARALLEL_DEGREE CONSTANT PLS_INTEGER := -20110; + MSG_INVALID_PARALLEL_DEGREE VARCHAR2(4000) := 'Invalid parallel degree parameter. Must be between 1 and 16'; + PRAGMA EXCEPTION_INIT( ERR_INVALID_PARALLEL_DEGREE + ,CODE_INVALID_PARALLEL_DEGREE); + + ERR_PARALLEL_EXECUTION_FAILED EXCEPTION; + CODE_PARALLEL_EXECUTION_FAILED CONSTANT PLS_INTEGER := -20111; + MSG_PARALLEL_EXECUTION_FAILED VARCHAR2(4000) := 'Parallel execution failed'; + PRAGMA EXCEPTION_INIT( ERR_PARALLEL_EXECUTION_FAILED + ,CODE_PARALLEL_EXECUTION_FAILED); + + ERR_UNKNOWN EXCEPTION; + CODE_UNKNOWN CONSTANT PLS_INTEGER := -20999; + MSG_UNKNOWN VARCHAR2(4000) := 'Unknown Error Occured'; + PRAGMA EXCEPTION_INIT( ERR_UNKNOWN + ,CODE_UNKNOWN); + + --------------------------------------------------------------------------------------------------------------------------- + --------------------------------------------------------------------------------------------------------------------------- + + + + + /** + * @name LOG_PROCESS_EVENT + * @desc Insert a new log record into A_PROCESS_LOG table. + * Also outputs to console if gvConsoleLoggingEnabled = 'ON'. + * Respects logging level configuration (gvMinLogLevel). + * @example ENV_MANAGER.LOG_PROCESS_EVENT('Process completed successfully', 'INFO', 'pParam1=value1'); + * @ex_rslt Record inserted into A_PROCESS_LOG table and optionally displayed in console output + **/ + PROCEDURE LOG_PROCESS_EVENT ( + pLogMessage VARCHAR2 + ,pLogLevel VARCHAR2 DEFAULT 'ERROR' + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ); + + /** + * @name LOG_PROCESS_ERROR + * @desc Insert a detailed error record into A_PROCESS_LOG table with full stack trace, backtrace, and call stack. + * This procedure captures comprehensive error information for debugging purposes while + * allowing clean user-facing error messages to be raised separately. + * @param pLogMessage - Base error message description + * @param pParameters - Procedure parameters for context + * @param pProcessName - Name of the calling process/package + * @ex_rslt Record inserted into A_PROCESS_LOG table with complete error stack information + */ + PROCEDURE LOG_PROCESS_ERROR ( + pLogMessage VARCHAR2 + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ); + + /** + * @name INIT_ERRORS + * @desc Loads data into Errors array. + * Errors array is a list of Record(Error_Code, Error_Message) index by Error_Code. + * Called automatically during package initialization. + * @example Called automatically when package is first referenced + * @ex_rslt Errors array populated with all error codes and messages + **/ + PROCEDURE INIT_ERRORS; + + + + /** + * @name GET_DEFAULT_ENV + * @desc It returns string with name of default environment. + * Return string is A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID value. + * @example select ENV_MANAGER.GET_DEFAULT_ENV() from dual; + * @ex_rslt dev + **/ + FUNCTION GET_DEFAULT_ENV + RETURN VARCHAR2; + + + + /** + * @name INIT_VARIABLES + * @desc For specified pEnv parameter (A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID) + * Assign values to following global package variables: + * - gvNameSpace + * - gvRegion + * - gvCredentialName + * - gvInboxBucketName + * - gvDataBucketName + * - gvArchiveBucketName + * - gvInboxBucketUri + * - gvDataBucketUri + * - gvArchiveBucketUri + * - gvLoggingEnabled + * - gvMinLogLevel + * - gvDefaultDateFormat + * - gvConsoleLoggingEnabled + **/ + PROCEDURE INIT_VARIABLES( + pEnv VARCHAR2 + ); + + + + /** + * @name GET_ERROR_MESSAGE + * @desc It returns string with error message for specified pCode (Error_Code). + * Error message is take from Errors Array loaded by INIT_ERRORS procedure + * @example select ENV_MANAGER.GET_ERROR_MESSAGE(pCode => -20009) from dual; + * @ex_rslt File not found on the cloud + **/ + FUNCTION GET_ERROR_MESSAGE( + pCode PLS_INTEGER + ) RETURN VARCHAR2; + + + + /** + * @name GET_ERROR_STACK + * @desc It returns string with all possible error stack info. + * Error message is take from Errors Array loaded by INIT_ERRORS procedure + * @example + * select ENV_MANAGER.GET_ERROR_STACK( + * pFormat => 'OUTPUT' + * ,pCode => -20009 + * ,pSourceFileReceivedKey => NULL) + * from dual + * @ex_rslt + * ------------------------------------------------------+ + * Error Message: + * ORA-0000: normal, successful completion + * ------------------------------------------------------- + * Error Stack: + * ------------------------------------------------------- + * Error Backtrace: + * ------------------------------------------------------+ + **/ + FUNCTION GET_ERROR_STACK( + pFormat VARCHAR2 + ,pCode PLS_INTEGER + ,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL + ) RETURN VARCHAR2; + + /** + * @name FORMAT_PARAMETERS + * @desc Formats parameter list for logging purposes. + * Converts SYS.ODCIVARCHAR2LIST to formatted string with proper NULL handling. + * @example select ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('param1=value1', 'param2=NULL')) from dual; + * @ex_rslt param1=value1 , + * param2=NULL + **/ + FUNCTION FORMAT_PARAMETERS( + pParameterList SYS.ODCIVARCHAR2LIST + ) RETURN VARCHAR2; + + /** + * @name ANALYZE_VALIDATION_ERRORS + * @desc Analyzes CSV validation errors and generates detailed diagnostic report. + * Compares CSV structure with template table and provides specific error analysis. + * Includes suggested solutions for common validation issues. + * @param pValidationLogTable - Name of validation log table (e.g., VALIDATE$242_LOG) + * @param pTemplateSchema - Schema of template table (e.g., CT_ET_TEMPLATES) + * @param pTemplateTable - Name of template table (e.g., MOCK_PROC_TABLE) + * @param pCsvFileUri - URI of CSV file being validated + * @example SELECT ENV_MANAGER.ANALYZE_VALIDATION_ERRORS('VALIDATE$242_LOG', 'CT_ET_TEMPLATES', 'MOCK_PROC_TABLE', 'https://...') FROM DUAL; + * @ex_rslt Detailed validation analysis report with column mismatches and solutions + **/ + FUNCTION ANALYZE_VALIDATION_ERRORS( + pValidationLogTable VARCHAR2, + pTemplateSchema VARCHAR2, + pTemplateTable VARCHAR2, + pCsvFileUri VARCHAR2 + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the ENV_MANAGER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT ENV_MANAGER.GET_VERSION() FROM DUAL; + * @ex_rslt 3.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Formatted for display in logs or monitoring systems. + * @example SELECT ENV_MANAGER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: ENV_MANAGER + * Version: 3.0.0 + * Build Date: 2025-10-22 16:00:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Shows evolution of package features over time. + * @example SELECT ENV_MANAGER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt ENV_MANAGER Version History: + * 3.0.0 (2025-10-22): Added package versioning system... + * 2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + + /** + * @name GET_PACKAGE_VERSION_INFO + * @desc Universal function to get formatted version information for any package. + * This centralized function is used by all packages in the system. + * @param pPackageName - Name of the package + * @param pVersion - Version string (MAJOR.MINOR.PATCH format) + * @param pBuildDate - Build date timestamp + * @param pAuthor - Package author name + * @example SELECT ENV_MANAGER.GET_PACKAGE_VERSION_INFO('FILE_MANAGER', '2.1.0', '2025-10-22 15:00:00', 'Grzegorz Michalski') FROM DUAL; + * @ex_rslt Package: FILE_MANAGER + * Version: 2.1.0 + * Build Date: 2025-10-22 15:00:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_PACKAGE_VERSION_INFO( + pPackageName VARCHAR2, + pVersion VARCHAR2, + pBuildDate VARCHAR2, + pAuthor VARCHAR2 + ) RETURN VARCHAR2; + + /** + * @name FORMAT_VERSION_HISTORY + * @desc Universal function to format version history for any package. + * Adds package name header and proper formatting. + * @param pPackageName - Name of the package + * @param pVersionHistory - Complete version history text + * @example SELECT ENV_MANAGER.FORMAT_VERSION_HISTORY('FILE_MANAGER', '2.1.0 (2025-10-22): Export procedures...') FROM DUAL; + * @ex_rslt FILE_MANAGER Version History: + * 2.1.0 (2025-10-22): Export procedures... + **/ + FUNCTION FORMAT_VERSION_HISTORY( + pPackageName VARCHAR2, + pVersionHistory VARCHAR2 + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE HASH + CHANGE DETECTION FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name CALCULATE_PACKAGE_HASH + * @desc Calculates SHA256 hash of package source code from ALL_SOURCE. + * Returns hash for both SPEC and BODY (if exists). + * Used for automatic change detection. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @param pPackageType - Type of package code ('PACKAGE' for SPEC, 'PACKAGE BODY' for BODY) + * @example SELECT ENV_MANAGER.CALCULATE_PACKAGE_HASH('CT_MRDS', 'FILE_MANAGER', 'PACKAGE') FROM DUAL; + * @ex_rslt A7B3C5D9E8F1234567890ABCDEF... (64-character SHA256 hash) + **/ + FUNCTION CALCULATE_PACKAGE_HASH( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageType VARCHAR2 -- 'PACKAGE' or 'PACKAGE BODY' + ) RETURN VARCHAR2; + + /** + * @name TRACK_PACKAGE_VERSION + * @desc Records package version and source code hash in A_PACKAGE_VERSION_TRACKING table. + * Automatically detects if source code changed without version update. + * Should be called after every package deployment. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @param pPackageVersion - Current version from PACKAGE_VERSION constant + * @param pPackageBuildDate - Build date from PACKAGE_BUILD_DATE constant + * @param pPackageAuthor - Author from PACKAGE_AUTHOR constant + * @example EXEC ENV_MANAGER.TRACK_PACKAGE_VERSION('CT_MRDS', 'FILE_MANAGER', '3.2.0', '2025-10-22 16:30:00', 'Grzegorz Michalski'); + * @ex_rslt Record inserted into A_PACKAGE_VERSION_TRACKING with change detection status + **/ + PROCEDURE TRACK_PACKAGE_VERSION( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageVersion VARCHAR2, + pPackageBuildDate VARCHAR2, + pPackageAuthor VARCHAR2 + ); + + /** + * @name CHECK_PACKAGE_CHANGES + * @desc Checks if package source code has changed since last tracking. + * Compares current hash with last recorded hash in A_PACKAGE_VERSION_TRACKING. + * Returns detailed change detection report. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @example SELECT ENV_MANAGER.CHECK_PACKAGE_CHANGES('CT_MRDS', 'FILE_MANAGER') FROM DUAL; + * @ex_rslt WARNING: Package changed without version update! + * Last Version: 3.2.0 + * Current Hash (SPEC): A7B3C5D9... + * Last Hash (SPEC): B8C4D6E0... + * RECOMMENDATION: Update PACKAGE_VERSION and PACKAGE_BUILD_DATE + **/ + FUNCTION CHECK_PACKAGE_CHANGES( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2; + + /** + * @name GET_PACKAGE_HASH_INFO + * @desc Returns formatted information about package hash and tracking history. + * Includes current hash, last tracked hash, and change detection status. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @example SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('CT_MRDS', 'FILE_MANAGER') FROM DUAL; + * @ex_rslt Package: CT_MRDS.FILE_MANAGER + * Current Version: 3.2.0 + * Current Hash (SPEC): A7B3C5D9... + * Last Tracked: 2025-10-22 16:30:00 + * Status: OK - No changes detected + **/ + FUNCTION GET_PACKAGE_HASH_INFO( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2; + +END ENV_MANAGER; +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..1c9f880 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb @@ -0,0 +1,1285 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE + ---------------------------------------------------------------------------------------------------- + /** + * @name GET_ARCHIVAL_WHERE_CLAUSE + * @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration. + * Supports three strategies: THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID. + * @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY + * @return VARCHAR2 - WHERE clause for filtering archival candidates + **/ + FUNCTION GET_ARCHIVAL_WHERE_CLAUSE( + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) RETURN VARCHAR2 + IS + vWhereClause VARCHAR2(4000); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + CASE pSourceFileConfig.ARCHIVAL_STRATEGY + -- Legacy threshold-based strategy (backward compatible) + WHEN 'THRESHOLD_BASED' THEN + vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS; + + -- Archive data older than X months (0 = current month only) + WHEN 'MINIMUM_AGE_MONTHS' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); + END IF; + vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + -- Hybrid: Current month exclusion AND minimum age requirement + WHEN 'HYBRID' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy'); + END IF; + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' || + 'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + ELSE + RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY); + END CASE; + + RETURN vWhereClause; + END GET_ARCHIVAL_WHERE_CLAUSE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + vKeepInTrash BOOLEAN; -- Derived from config + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + -- Check if archiving is enabled for this configuration + IF vSourceFileConfig.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archiving disabled for this configuration (IS_ARCHIVE_ENABLED=N). Skipping.', 'WARNING', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN; + END IF; + + -- Get TRASH policy from configuration + vKeepInTrash := (vSourceFileConfig.IS_KEEP_IN_TRASH = 'Y'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEEP_IN_TRASH=' || vSourceFileConfig.IS_KEEP_IN_TRASH, 'INFO', vParameters); + + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + -- Strategy-based trigger logic (MARS-828) + IF vSourceFileConfig.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN + -- MINIMUM_AGE_MONTHS: Archive based on age only, ignore thresholds + vArchivalTriggeredBy := 'AGE_BASED'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival strategy: MINIMUM_AGE_MONTHS (threshold-independent)','INFO'); + ELSE + -- THRESHOLD_BASED and HYBRID: Check thresholds + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + END IF; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + + -- Use strategy-based WHERE clause (MARS-828) + -- Using GROUP BY instead of DISTINCT to avoid ORA-22950 (object type ordering issue) + vQuery := ' + select CT_MRDS.t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' + and h.WORKFLOW_SUCCESSFUL = ''Y'' + group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')' + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||''' + and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||''' + and h.WORKFLOW_SUCCESSFUL = ''Y'' + ' + ; + vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month,'INFO', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + -- Note: DBMS_CLOUD.EXPORT_DATA may create multiple parquet files (parallel execution) + -- Instead of tracking individual files, we store the archive directory prefix + -- ARCH_PATH contain the directory URI where all parquet files are located + vFilename := vUri; -- Store directory prefix instead of individual filename + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status to ARCHIVED_AND_TRASHED (file will be moved to TRASH folder) + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -- Status reflects file is archived and kept in TRASH + ,ARCH_PATH = vFilename -- Now contains directory prefix, not individual file + ,PARTITION_YEAR = ym_loop.year -- Record which partition year the data was archived to + ,PARTITION_MONTH = ym_loop.month -- Record which partition month the data was archived to + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + + -- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) + -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) + IF vProcessControlStatus = 'OK' THEN + IF NOT vKeepInTrash THEN + -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + -- Update status to ARCHIVED_AND_PURGED + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + END LOOP; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEEP_IN_TRASH=N).','INFO'); + ELSE + -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO'); + END IF; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED') + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_PATH = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + vWhereClause VARCHAR2(4000); + vOdsBucketUri VARCHAR2(1000); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + + -- Get WHERE clause based on archival strategy (MARS-828) + vWhereClause := GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vWhereClause','DEBUG',vWhereClause); + + -- Get ODS bucket URI before building query + vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + -- Use strategy-based WHERE clause for statistics (MARS-828) + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES + ,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''', + location_uri => '''||vOdsBucketUri||''' + ) + ) r + on t.filename = r.object_name' + ; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into CT_MRDS.A_TABLE_STAT_HIST values vStats; + delete from CT_MRDS.A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into CT_MRDS.A_TABLE_STAT values vStats; + COMMIT; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- TRASH FOLDER MANAGEMENT PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashPath VARCHAR2(1000); + vOdsPath VARCHAR2(1000); + vFilesRestored PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vRestoreLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath); + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Restore all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + FOR file_rec IN ( + SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pRestoreAll THEN + vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations + FOR file_rec IN ( + SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME, + c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashLocationUri VARCHAR2(1000); + vFilesDeleted PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vPurgeLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri + ); + vFilesDeleted := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Delete all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pPurgeAll THEN + vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations + FOR config_rec IN ( + SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_CONFIG c + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PURGE_TRASH_FOLDER( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pPurgeAll => pPurgeAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + RESTORE_FILE_FROM_TRASH( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pRestoreAll => pRestoreAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- BATCH ARCHIVAL PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesArchived PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pArchiveAll THEN + vProcessingLevel := 'LEVEL 3: Archive All'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Archive All Enabled Tables', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pArchiveAll=TRUE'); + END IF; + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + IS_ARCHIVE_ENABLED, + IS_KEEP_IN_TRASH, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pArchiveAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pArchiveAll = TRUE) + ) + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF config_rec.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (IS_ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEEP_IN_TRASH=' || config_rec.IS_KEEP_IN_TRASH || ']', + 'INFO' + ); + + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesArchived := vTablesArchived + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully archived table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to archive table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch archival summary: Archived=' || vTablesArchived || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed, + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pArchiveAll => pArchiveAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + -- BATCH STATISTICS GATHERING PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesProcessed PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + vEnabledFilter VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pGatherAll THEN + vProcessingLevel := 'LEVEL 3: Gather All Statistics'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Gather All Enabled Statistics', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pGatherAll=TRUE'); + END IF; + + -- Set enabled filter info + vEnabledFilter := CASE WHEN pOnlyEnabled THEN 'IS_ARCHIVE_ENABLED=Y only' ELSE 'All tables' END; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Filter mode: ' || vEnabledFilter, 'INFO'); + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + IS_ARCHIVE_ENABLED, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pGatherAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pGatherAll = TRUE) + ) + -- Apply IS_ARCHIVE_ENABLED filter if pOnlyEnabled = TRUE + AND (pOnlyEnabled = FALSE OR IS_ARCHIVE_ENABLED = 'Y') + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF pOnlyEnabled AND config_rec.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (IS_ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Gathering statistics for table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_ARCHIVE_ENABLED=' || config_rec.IS_ARCHIVE_ENABLED || ']', + 'INFO' + ); + + GATHER_TABLE_STAT(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesProcessed := vTablesProcessed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully gathered statistics for table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to gather statistics for table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch statistics gathering summary: Processed=' || vTablesProcessed || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed || ' [Filter: ' || vEnabledFilter || ']', + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pGatherAll => pGatherAll, + pOnlyEnabled => pOnlyEnabled + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_GATHER_TABLE_STAT_ALL; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..8b14196 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg @@ -0,0 +1,279 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) || + '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || + '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || + '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || + '3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) || + '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name GET_TABLE_STAT + * @desc Private function to retrieve table statistics for archival processing. + * Returns A_TABLE_STAT record with table metadata and row counts. + * @param pSourceFileConfigKey - Configuration key for source file + * @return CT_MRDS.A_TABLE_STAT%ROWTYPE - Table statistics record + * @private Internal function for archival operations + **/ + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name FN_ARCHIVE_TABLE_DATA + * @desc Function wrapper for ARCHIVE_TABLE_DATA procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name FN_GATHER_TABLE_STAT + * @desc Function wrapper for GATHER_TABLE_STAT procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT procedure and captures execution result. + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + /** + * @name GATHER_TABLE_STAT_ALL + * @desc Multi-level batch statistics gathering procedure with three granularity levels. + * Processes configurations based on IS_ARCHIVE_ENABLED setting (when pOnlyEnabled=TRUE). + * Gathers statistics for external tables and inserts data into A_TABLE_STAT and A_TABLE_STAT_HIST. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (e.g., 'LM', 'C2D') (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y' + * @example -- Level 1: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE); + * @example -- All tables regardless of IS_ARCHIVE_ENABLED: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE, pOnlyEnabled => FALSE); + **/ + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ); + + /** + * @name FN_GATHER_TABLE_STAT_ALL + * @desc Function wrapper for GATHER_TABLE_STAT_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y' + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER; + + /** + * @name ARCHIVE_ALL + * @desc Multi-level batch archival procedure with three granularity levels. + * Only processes configurations where IS_ARCHIVE_ENABLED='Y'. + * TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example -- Level 1: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.ARCHIVE_ALL(pArchiveAll => TRUE); + **/ + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name FN_ARCHIVE_ALL + * @desc Function wrapper for ARCHIVE_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Restores files from TRASH folder back to ODS at three different granularity levels. + * Moves files from TRASH subfolder back to ODS subfolder in DATA bucket. + * Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345); + * @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341); + * @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE); + **/ + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Function overload for RESTORE_FILE_FROM_TRASH procedure. + * Returns SQLCODE for Python library integration. + * Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name PURGE_TRASH_FOLDER + * @desc Deletes files from TRASH folder at three different granularity levels. + * Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345); + * @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341); + * @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE); + **/ + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name PURGE_TRASH_FOLDER + * @desc Function overload for PURGE_TRASH_FOLDER procedure. + * Returns SQLCODE for Python library integration. + * Calls the main PURGE_TRASH_FOLDER procedure and captures execution result. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb new file mode 100644 index 0000000..3daac26 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb @@ -0,0 +1,2009 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_MANAGER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: NORMALIZE_DATE_FORMAT + ---------------------------------------------------------------------------------------------------- + /** + * Purpose: Normalize Oracle date format strings for use in external tables + * + * Problem: ISO 8601 formats like 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' fail because + * literal character 'T' must be enclosed in double quotes for Oracle + * external table DATE column definitions. + * + * Solution: Detect unquoted 'T' separator and wrap it in double quotes + * + * Parameters: + * pDateFormat - Original date format from A_COLUMN_DATE_FORMAT table + * + * Returns: Normalized format with quoted 'T' if applicable + * + * Examples: + * Input: 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' + * Output: 'YYYY-MM-DD"T"HH24:MI:SS.FF3TZH:TZM' + * + * Input: 'DD/MM/YYYY HH24:MI:SS' (no T) + * Output: 'DD/MM/YYYY HH24:MI:SS' (unchanged) + * + * Input: 'YYYY-MM-DD"T"HH24:MI:SS' (already quoted) + * Output: 'YYYY-MM-DD"T"HH24:MI:SS' (unchanged) + * + * Author: Grzegorz Michalski + * Date: 2025-11-27 + * Version: 1.0.0 (MARS-1046) + */ + FUNCTION NORMALIZE_DATE_FORMAT(pDateFormat VARCHAR2) RETURN VARCHAR2 IS + vNormalizedFormat VARCHAR2(500); + BEGIN + -- Return NULL if input is NULL + IF pDateFormat IS NULL THEN + RETURN NULL; + END IF; + + vNormalizedFormat := pDateFormat; + + -- Check if 'T' separator exists and is NOT already quoted + -- Pattern: [YMD]T[HM] (date component + T + time component) + IF INSTR(vNormalizedFormat, '"T"') = 0 AND + REGEXP_LIKE(vNormalizedFormat, '[YMD]T[HM]') THEN + + -- Wrap 'T' in double quotes using regex replace + -- Pattern matches: (date format char) + T + (time format char) + -- Replacement: \1 + "T" + \2 + vNormalizedFormat := REGEXP_REPLACE(vNormalizedFormat, '([YMD])T([HM])', '\1"T"\2'); + END IF; + + RETURN vNormalizedFormat; + + EXCEPTION + WHEN OTHERS THEN + -- If normalization fails, return original format (safety fallback) + RETURN pDateFormat; + END NORMALIZE_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL + , pSourceFileReceivedKey IN NUMBER DEFAULT NULL + , pSourceFileConfigKey IN NUMBER DEFAULT NULL) + RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pFileUri => '''||nvl(pFileUri,'NULL')||'''' + ,'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') + ,'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + + BEGIN + IF pFileUri IS NOT NULL THEN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE REGEXP_LIKE(pFileUri, A_SOURCE_KEY||'/'||SOURCE_FILE_ID||'/'||TABLE_ID||'/'||SOURCE_FILE_NAME_PATTERN); + ELSIF pSourceFileReceivedKey IS NOT NULL THEN + SELECT T.* + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG T, CT_MRDS.A_SOURCE_FILE_RECEIVED R + WHERE T.A_SOURCE_FILE_CONFIG_KEY = R.A_SOURCE_FILE_CONFIG_KEY + AND R.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + ELSIF pSourceFileConfigKey IS NOT NULL THEN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + END IF; + -- Set global package variable vgSourceFileConfigKey - used in error messages + vgSourceFileConfigKey := vSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY; + EXCEPTION + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); + + WHEN OTHERS THEN + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileConfig; + + END GET_SOURCE_FILE_CONFIG; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey IN NUMBER DEFAULT NULL) + -- + -- Get source file received info + -- + RETURN tSourceFileReceived + IS + vSourceFileReceived tSourceFileReceived; + vBucket VARCHAR2(400); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + BEGIN + SELECT R.A_SOURCE_FILE_RECEIVED_KEY, R.A_SOURCE_FILE_CONFIG_KEY, + 'INBOX'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_INBOX, + 'ODS'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_ODS, + 'QUARANTINE'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_QUARANTINE, + 'ARCHIVE'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/' as SOURCE_FILE_PREFIX_ARCHIVE, + R.SOURCE_FILE_NAME, + R.RECEPTION_DATE, R.PROCESSING_STATUS, R.EXTERNAL_TABLE_NAME + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED R, CT_MRDS.A_SOURCE_FILE_CONFIG T + WHERE R.A_SOURCE_FILE_CONFIG_KEY = T.A_SOURCE_FILE_CONFIG_KEY + AND A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + RETURN vSourceFileReceived; + + END GET_SOURCE_FILE_RECEIVED_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER + -- + -- Register a newly received source file A_SOURCE_FILE_RECEIVED + -- This overload automatically determines source file type from the file name + -- + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceivedKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO',vParameters); + + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedName); + vSourceFileReceivedKey := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName, vSourceFileConfig); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN vSourceFileReceivedKey; + EXCEPTION + + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + + WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); + + WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END REGISTER_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION REGISTER_SOURCE_FILE_RECEIVED( + pSourceFileReceivedName IN VARCHAR2 + ,pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE) + RETURN PLS_INTEGER + -- + -- Register a newly received source file A_SOURCE_FILE_RECEIVED + -- + IS + vExternalTableName VARCHAR2(200); + vDirName VARCHAR2(1000); + vFileName VARCHAR2(1000); + vChecksum A_SOURCE_FILE_RECEIVED.CHECKSUM%TYPE; + vCreated A_SOURCE_FILE_RECEIVED.CREATED%TYPE; + vBytes A_SOURCE_FILE_RECEIVED.BYTES%TYPE; + vSourceFileReceivedKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vRow CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''' + ,'pSourceFileConfig => '||'tSourceFileConfig record type')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + vDirName := REGEXP_SUBSTR(pSourceFileReceivedName, '(.*/)(.*)', 1, 1, NULL, 1); + -- Remove prefix from file name + vFileName := REGEXP_SUBSTR(pSourceFileReceivedName,'[^/]*$'); + + ENV_MANAGER.LOG_PROCESS_EVENT('gvCredentialName','DEBUG',ENV_MANAGER.gvCredentialName); + ENV_MANAGER.LOG_PROCESS_EVENT('gvInboxBucketUri','DEBUG',ENV_MANAGER.gvInboxBucketUri); + ENV_MANAGER.LOG_PROCESS_EVENT('vDirName','DEBUG',vDirName); + + SELECT + checksum, created, bytes + INTO + vChecksum, vCreated, vBytes + FROM DBMS_CLOUD.LIST_OBJECTS(ENV_MANAGER.gvCredentialName, + ENV_MANAGER.gvInboxBucketUri || vDirName + ) + WHERE object_name = vFileName + ; + vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL; + vExternalTableName := REPLACE( + REGEXP_SUBSTR(pSourceFileConfig.TEMPLATE_TABLE_NAME||'_'||vSourceFileReceivedKey, + '\..*'), + '.',''); + + INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED + (A_SOURCE_FILE_RECEIVED_KEY, A_SOURCE_FILE_CONFIG_KEY, + SOURCE_FILE_NAME, RECEPTION_DATE, + PROCESSING_STATUS, EXTERNAL_TABLE_NAME, + CHECKSUM, CREATED, BYTES) + VALUES (vSourceFileReceivedKey, pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY, + vFileName, SYSDATE, + 'RECEIVED', vExternalTableName, + vChecksum, vCreated, vBytes); + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileReceivedKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD + ||cgBL||' '||'File: '||ENV_MANAGER.gvInboxBucketUri || vDirName || vFileName; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, vgMsgTmp); + + WHEN DUP_VAL_ON_INDEX THEN + select * into vRow + from CT_MRDS.A_SOURCE_FILE_RECEIVED + where CHECKSUM = vChecksum + and CREATED = vCreated + and BYTES = vBytes + ; + vgMsgTmp := ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED + ||cgBL||' '||'Details about existing File: ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'A_SOURCE_FILE_RECEIVED_KEY = '||vRow.A_SOURCE_FILE_RECEIVED_KEY + ||cgBL||' '||'A_SOURCE_FILE_CONFIG_KEY = '||vRow.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||' '||'SOURCE_FILE_NAME = '||vRow.SOURCE_FILE_NAME + ||cgBL||' '||'CHECKSUM = '||vRow.CHECKSUM + ||cgBL||' '||'CREATED = '||vRow.CREATED + ||cgBL||' '||'BYTES = '||vRow.BYTES + ||cgBL||' '||'RECEPTION_DATE = '||vRow.RECEPTION_DATE + ||cgBL||' '||'PROCESSING_STATUS = '||vRow.PROCESSING_STATUS + ||cgBL||' '||'EXTERNAL_TABLE_NAME = '||vRow.EXTERNAL_TABLE_NAME + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'There cannot be two files with the same values for (CHECKSUM, CREATED, BYTES)' + ; + + +-- vChecksum, vCreated, vBytes + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, vgMsgTmp); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END REGISTER_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey IN PLS_INTEGER, pStatus IN VARCHAR2) + -- + -- Change status of file in the A_SOURCE_FILE_RECEIVED table + -- + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') + ,'pStatus => '''||nvl(pStatus, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS=pStatus + WHERE A_SOURCE_FILE_RECEIVED_KEY=pSourceFileReceivedKey; + COMMIT; + ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to '||pStatus,'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END SET_SOURCE_FILE_RECEIVED_STATUS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate IN VARCHAR2) + RETURN CLOB + -- + -- Create list of columns for DBMS_CLOUD.CREATE_EXTERNAL_TABLE from existing template table + -- + IS + vColumnList CLOB; + vTableName VARCHAR2(200); + vSchemaName VARCHAR2(200); + BEGIN + vSchemaName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'.*\.'),'.',''); + vTableName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'\..*'),'.',''); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SQLTERMINATOR', True); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'COLLATION_CLAUSE', 'NEVER'); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'REF_CONSTRAINTS', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'STORAGE', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'TABLESPACE', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SEGMENT_ATTRIBUTES', False); + vColumnList := RTRIM( + LTRIM( + REGEXP_SUBSTR(DBMS_METADATA.GET_DDL('TABLE', vTableName, vSchemaName),'\(.*\)',1,1,'mn'), + '('), + ')'); + RETURN vColumnList; + END GET_EXTERNAL_TABLE_COLUMNS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE CREATE_EXTERNAL_TABLE ( + pTableName IN VARCHAR2, + pTemplateTableName IN VARCHAR2, + pPrefix IN VARCHAR2, + pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri, + pFileName IN VARCHAR2 DEFAULT NULL, + pDelimiter IN VARCHAR2 DEFAULT ',', + pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NEW PARAMETER FOR FILE ENCODING + ) + -- + -- Create external table for a single source file to validate the file structure + -- + IS + vTableName VARCHAR2(200); + vColumnList CLOB; + vFieldList CLOB; + vFormat VARCHAR2(200); + + vPrefix VARCHAR2(200); + vFileName VARCHAR2(1000); + vFileExtension VARCHAR2(200); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTableName => '''||nvl(pTableName, 'NULL')||'''' + ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pPrefix => '''||nvl(pPrefix, 'NULL')||'''' + ,'pBucketUri => '''||nvl(pBucketUri, 'NULL')||'''' + ,'pFileName => '''||nvl(pFileName, 'NULL')||'''' + ,'pDelimiter => '''||nvl(pDelimiter, 'NULL')||'''' + ,'pEncoding => '''||nvl(pEncoding, 'NULL')||'''' -- MARS-1049: NOWY + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + -- Strip off leading and trailing slashes from prefix + vPrefix := TRIM(BOTH '/' FROM pPrefix); + + -- Generate column and field list from template table + GENERATE_EXTERNAL_TABLE_PARAMS (pTemplateTableName, vColumnList, vFieldList); + + --vFormat evaluation based on pBucketUri first, then pPrefix + -- Archive bucket should use parquet + implicit partitioning regardless of prefix + IF INSTR(pBucketUri, ENV_MANAGER.gvArchiveBucketName)>0 THEN + vFormat := '{"type": "parquet" + ,"implicit_partition_type": "hive" + ,"implicit_partition_columns":["PARTITION_YEAR","PARTITION_MONTH"]}'; + vColumnList := vColumnList||cgBL||' , "PARTITION_YEAR" varchar2(4)'||cgBL||', "PARTITION_MONTH" varchar2(2)'; + vFieldList := NULL; + vFileExtension := '.parquet'; + + -- For INBOX, ODS, and other ARCHIVE prefixes (not in archive bucket) use CSV + ELSIF SUBSTR(pPrefix,1,5) = 'INBOX' OR SUBSTR(pPrefix,1,3) = 'ODS' + OR SUBSTR(pPrefix,1,7) = 'ARCHIVE' + THEN + -- MARS-1049: Create format with encoding if specified + IF pDelimiter = '|' THEN + IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN + vFormat := json_object( + 'delimiter' VALUE '|', + 'skipheaders' VALUE '1', + 'characterset' VALUE pEncoding + ); + ELSE + vFormat := json_object('delimiter' VALUE '|', 'skipheaders' VALUE '1'); + END IF; + ELSE + IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN + vFormat := json_object( + 'type' VALUE 'CSV', + 'skipheaders' VALUE '1', + 'ignoremissingcolumns' VALUE 'true', + 'characterset' VALUE pEncoding + ); + ELSE + vFormat := json_object('type' VALUE 'CSV', 'skipheaders' VALUE '1', 'ignoremissingcolumns' value 'true'); + END IF; + END IF; + + vFileExtension := '.csv'; + + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); + END IF; + + -- No filename give: Match all csv files + IF pFileName IS NOT NULL THEN + vFileName := pFileName; + ELSE + vFileName := pBucketUri||vPrefix||'/*'||vFileExtension; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('pTableName', 'DEBUG', pTableName); + ENV_MANAGER.LOG_PROCESS_EVENT('ENV_MANAGER.vpCredentialName', 'DEBUG', ENV_MANAGER.gvCredentialName); + ENV_MANAGER.LOG_PROCESS_EVENT('vFileName', 'DEBUG', vFileName); + ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', vColumnList); + ENV_MANAGER.LOG_PROCESS_EVENT('vFieldList', 'DEBUG', vFieldList); + ENV_MANAGER.LOG_PROCESS_EVENT('vFormat', 'DEBUG', vFormat); + + -- Pre-validation: Check CSV column count for CSV files only + IF SUBSTR(pPrefix,1,5) = 'INBOX' AND pFileName IS NOT NULL THEN + DECLARE + vCsvFirstLine VARCHAR2(4000); + vCsvColCount NUMBER := 0; + vTemplateColCount NUMBER := 0; + vExcessColumns VARCHAR2(2000); + + -- Get template column count + CURSOR c_template_count IS + SELECT COUNT(*) as col_count + FROM ALL_TAB_COLUMNS + WHERE OWNER = UPPER(REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.','')) + AND TABLE_NAME = UPPER(REGEXP_REPLACE(pTemplateTableName,'^.*\.','')); + + BEGIN + -- Get template column count + FOR rec IN c_template_count LOOP + vTemplateColCount := rec.col_count; + END LOOP; + + -- Read first line of CSV to count columns + BEGIN + SELECT UTL_RAW.CAST_TO_VARCHAR2( + DBMS_LOB.SUBSTR( + DBMS_CLOUD.GET_OBJECT( + credential_name => ENV_MANAGER.gvCredentialName, + object_uri => pFileName + ), + 4000, 1 + ) + ) INTO vCsvFirstLine FROM DUAL; + + -- Count commas in header line + 1 for total columns + vCsvColCount := REGEXP_COUNT(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), ',') + 1; + + ENV_MANAGER.LOG_PROCESS_EVENT('CSV Column Count: ' || vCsvColCount || ', Template Column Count: ' || vTemplateColCount, 'INFO', vParameters); + + -- Check for excess columns + IF vCsvColCount > vTemplateColCount THEN + vgMsgTmp := ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED + ||cgBL||'EXCESS COLUMNS DETECTED!' + ||cgBL||'CSV file has ' || vCsvColCount || ' columns but template expects only ' || vTemplateColCount + ||cgBL||'Excess columns: ' || (vCsvColCount - vTemplateColCount) + ||cgBL||'CSV header: ' || SUBSTR(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), 1, 200) + ||cgBL||'POSSIBLE SOLUTIONS:' + ||cgBL||' 1. Remove excess columns from CSV file before processing' + ||cgBL||' 2. Add excess columns to template table: ' || pTemplateTableName; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, vgMsgTmp); + END IF; + + EXCEPTION + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + RAISE; -- Re-raise the excess columns error + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + RAISE; -- Re-raise the validation error + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not perform pre-validation column count check: ' || SQLERRM, 'WARN', vParameters); + -- Continue with normal processing if pre-validation fails + END; + END; + END IF; + + DBMS_CLOUD.CREATE_EXTERNAL_TABLE( + TABLE_NAME => pTableName, + CREDENTIAL_NAME => ENV_MANAGER.gvCredentialName, + FILE_URI_LIST => vFileName, + COLUMN_LIST => vColumnList, + FIELD_LIST => vFieldList, + FORMAT => vFormat + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + RAISE; -- Re-raise the excess columns error with specific code -20011 + WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN + vgMsgTmp := ENV_MANAGER.MSG_UNKNOWN_PREFIX || ': ' || pPrefix; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, vgMsgTmp); + WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END CREATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) + -- + -- Create external table for a single source file to validate the file structure + -- + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived tSourceFileReceived; + vTableName VARCHAR2(200); + vFileName VARCHAR2(1000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + vTableName := vSourceFileConfig.TEMPLATE_TABLE_NAME; + + vFileName := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX||vSourceFileReceived.SOURCE_FILE_NAME; + + CREATE_EXTERNAL_TABLE( + pTableName => vSourceFileReceived.EXTERNAL_TABLE_NAME, + pTemplateTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME, + pPrefix => vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX, + pBucketUri => ENV_MANAGER.gvInboxBucketUri, + pFileName => vFileName, + pDelimiter => ',', + pEncoding => vSourceFileConfig.ENCODING -- MARS-1049: NOWY PARAMETR + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + END CREATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey IN NUMBER) + -- + -- Check the structure of the received file using DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + -- + IS + vSourceFileReceived tSourceFileReceived; + vOperationId NUMBER := -1; + vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; + vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; + vErrors NUMBER := 0; + vNumRows NUMBER := 0; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + ENV_MANAGER.LOG_PROCESS_EVENT('vSourceFileReceived.EXTERNAL_TABLE_NAME: '||vSourceFileReceived.EXTERNAL_TABLE_NAME,'DEBUG', vParameters); + BEGIN + DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(vSourceFileReceived.EXTERNAL_TABLE_NAME, vOperationId); + EXCEPTION + WHEN OTHERS THEN + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_VALIDATION_FAILED, vParameters, 'FILE_MANAGER'); + + -- Call detailed validation error analysis and log the results + DECLARE + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vValidationLogTable VARCHAR2(200); + vTemplateSchema VARCHAR2(200); + vTemplateTable VARCHAR2(200); + vCsvFileUri VARCHAR2(2000); + vAnalysisResult VARCHAR2(32000); + vFailedOperationId NUMBER; + BEGIN + -- Get source file configuration + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); + + -- Extract template schema and table from template table name + vTemplateSchema := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'.*\.'),'.',''); + vTemplateTable := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'\..*'),'.',''); + + -- Construct CSV file URI + vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX || vSourceFileReceived.SOURCE_FILE_NAME; + + -- Find the failed validation operation ID + SELECT MAX(ID) INTO vFailedOperationId + FROM USER_LOAD_OPERATIONS + WHERE TABLE_NAME = vSourceFileReceived.EXTERNAL_TABLE_NAME + AND TYPE = 'VALIDATE' + AND STATUS != 'COMPLETED'; + + -- Get validation log table name + IF vFailedOperationId IS NOT NULL THEN + SELECT LOGFILE_TABLE INTO vValidationLogTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vFailedOperationId; + + -- Call detailed error analysis + vAnalysisResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( + pValidationLogTable => vValidationLogTable, + pTemplateSchema => vTemplateSchema, + pTemplateTable => vTemplateTable, + pCsvFileUri => vCsvFileUri + ); + + -- Log detailed analysis results + ENV_MANAGER.LOG_PROCESS_EVENT('DETAILED VALIDATION ERROR ANALYSIS:', 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(vAnalysisResult, 'ERROR', vParameters); + END IF; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error during validation analysis: ' || SQLERRM, 'ERROR', vParameters); + END; + + MOVE_FILE(pSourceFileReceivedKey => pSourceFileReceivedKey, pDestination => 'QUARANTINE'); + -- Ensure the status change is committed before raising exception + COMMIT; + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); + IF vOperationId = -1 + THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + END IF; + + SELECT BADFILE_TABLE, ROWS_LOADED, STATUS + INTO vBadfileTable, vNumRows, vStatus + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + +-- DBMS_OUTPUT.PUT_LINE(vStatus); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATED'); + ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to VALIDATED','DEBUG', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED; + ENV_MANAGER.LOG_PROCESS_ERROR(vgMsgTmp, vParameters, 'FILE_MANAGER'); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); + + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD||cgBL||SQLERRM; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, vgMsgTmp); + + ELSIF SQLCODE = -20003 THEN + execute immediate 'select LISTAGG(record, '''||cgBL||''') from (select * from '||REGEXP_SUBSTR(SQLERRM, '"([^"]+)"."([^"]+)"')||' order by rownum desc) where rownum <=2' + into vgMsgTmp; + vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||vgMsgTmp; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); +-- ELSIF SQLCODE = -20000 THEN + -- TO_DO Add additional info about current config + -- ENV_MANAGER.MSG_FILE_VALIDATION_FAILED := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||FILE_MANAGER.OUTPUT_SOURCE_FILE_CONFIG_INFO( ..config key value.. ); +-- dbms_output.put_line(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT, 'ERROR', vParameters); +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); +-- RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); + + ELSE + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END IF; + END VALIDATE_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2 + -- + -- wrapper for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + -- + IS + vOperationId NUMBER := -1; + vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; + vLogfileTable USER_LOAD_OPERATIONS.LOGFILE_TABLE%TYPE; + vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vDetails clob; + TYPE TCURSOR is REF CURSOR; + vCursor TCURSOR; + vQuery VARCHAR2(1000); + vRecord VARCHAR2(10000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTableName => '''||nvl(pTableName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(pTableName, vOperationId); + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); + IF vOperationId = -1 + THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + END IF; + + SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE + INTO vStatus, vLogfileTable, vBadfileTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + + RETURN vStatus; + EXCEPTION + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + + WHEN OTHERS THEN + SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE + INTO vStatus, vLogfileTable, vBadfileTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + vQuery := 'select record from ( + select + nvl(l.record,''----------------------------------------------------'') as record + ,rownum as lp + ,max(case when nvl(instr(l.record, ''error'' ),0) > 0 then rownum else 0 end) over (partition by 1) as ENV_MANAGER.ERR_row + from '||vLogfileTable||' l + ) + where lp >= ENV_MANAGER.ERR_row + order by rownum'; + + vDetails := vStatus||cgBL||'----------------------------------------------------'||cgBL; + + OPEN vCursor for vQuery; + loop + fetch vCursor into vRecord; + EXIT WHEN vCursor%NOTFOUND; + vDetails := vDetails ||vRecord ||cgBL; +-- for i in loop +-- vDetails := vDetails ||i.record ||cgBL; + end loop; + CLOSE vCursor; + vDetails := vDetails||'More details can be found in below tables:'||cgBL|| + ' SELECT * FROM USER_LOAD_OPERATIONS WHERE ID = '||vOperationId||';'||cgBL|| + ' SELECT * FROM '||vLogfileTable||';'||cgBL|| + ' SELECT * FROM '||vBadfileTable||';' + ; + + RETURN vDetails; + + END VALIDATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2 + -- + -- Simple + -- + IS + vCount PLS_INTEGER; + BEGIN + execute immediate 'select count(1) from '||pTableName into vCount; + IF vCount >= 0 + THEN + RETURN 'PASSED'; + END IF; + + RETURN 'FAILED'; + EXCEPTION + WHEN OTHERS THEN + RETURN 'FAILED'; + END S_VALIDATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DROP_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) + -- + -- Drop external table created to validate the file structure + -- + IS + vSourceFileReceived tSourceFileReceived; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + EXECUTE IMMEDIATE 'DROP TABLE '||vSourceFileReceived.EXTERNAL_TABLE_NAME; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END DROP_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE COPY_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) + -- + -- Possible pDestination values are: 'ODS' or 'ARCHIVE' + -- + IS + vSourceFileReceivedInfo tSourceFileReceived; + vSourceObject VARCHAR2(2000); + vTargetObject VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; +-- vStatus VARCHAR2(20); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + + IF pDestination = 'ODS' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + END IF; + + DBMS_CLOUD.COPY_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => vSourceObject, + target_object_uri => vTargetObject, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File copied to '||pDestination||' target location','DEBUG', vTargetObject); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END COPY_FILE; + + ---------------------------------------------------------------------------------------------------- + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE MOVE_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) + -- + -- Possible pDestination values are: 'ODS' or 'ARCHIVE' + -- + IS + vSourceFileReceivedInfo tSourceFileReceived; + vSourceObject VARCHAR2(2000); + vTargetObject VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStatus VARCHAR2(20); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + + vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + + IF pDestination = 'ODS' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vStatus := 'READY_FOR_INGESTION'; + ELSIF pDestination = 'QUARANTINE' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_QUARANTINE||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vStatus := 'VALIDATION_FAILED'; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + END IF; + + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => vSourceObject, + target_object_uri => vTargetObject, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File moved to '||pDestination||' target location','DEBUG', vTargetObject); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => vStatus); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN + ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM := ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM + ||cgBL||' '||'Possible parameters are: ''ODS'' or ''ARCHIVE''' + ||cgBL||' '||'Provided destination parameter: '''||pDestination||''''; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END MOVE_FILE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DELETE_FOLDER_CONTENTS(pBucketArea IN VARCHAR2, pFolderPrefix IN VARCHAR2) + -- + -- Delete all files from specified folder in cloud storage + -- pBucketArea: 'INBOX', 'DATA', 'ARCHIVE' + -- pFolderPrefix: folder path within bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/') + -- + IS + vBucketUri VARCHAR2(2000); + vFolderUri VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vFilesDeleted PLS_INTEGER := 0; + vObjectName VARCHAR2(4000); + vFullObjectUri VARCHAR2(4000); + + -- Cursor to list all objects in the folder + CURSOR c_objects IS + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => ENV_MANAGER.gvCredentialName, + location_uri => vBucketUri + )) + WHERE object_name IS NOT NULL + AND object_name LIKE pFolderPrefix || '%'; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pBucketArea => '''||nvl(pBucketArea, 'NULL')||'''', + 'pFolderPrefix => '''||nvl(pFolderPrefix, 'NULL')||'''' + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Get bucket URI based on bucket area + vBucketUri := GET_BUCKET_URI(pBucketArea); + + ENV_MANAGER.LOG_PROCESS_EVENT('Listing objects in bucket with prefix: ' || pFolderPrefix, 'DEBUG', vBucketUri); + + -- List and delete all objects in the folder + FOR obj_rec IN c_objects LOOP + vObjectName := obj_rec.object_name; + vFullObjectUri := vBucketUri || vObjectName; + + BEGIN + ENV_MANAGER.LOG_PROCESS_EVENT('Deleting object', 'DEBUG', vFullObjectUri); + + DBMS_CLOUD.DELETE_OBJECT( + credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vFullObjectUri + ); + + vFilesDeleted := vFilesDeleted + 1; + ENV_MANAGER.LOG_PROCESS_EVENT('Object deleted successfully', 'DEBUG', vObjectName); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error deleting object: ' || vObjectName || ' - ' || SQLERRM, 'ERROR', vParameters); + -- Continue with next file instead of stopping the whole process + END; + END LOOP; + + ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted: ' || vFilesDeleted, 'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error in DELETE_FOLDER_CONTENTS: ' || SQLERRM, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END DELETE_FOLDER_CONTENTS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + -- + -- Ubmrella procedure that calls + -- - REGISTER_SOURCE_FILE_RECEIVED + -- - CREATE_EXTERNAL_TABLE + -- - VALIDATE_SOURCE_FILE_RECEIVED + -- - DROP_EXTERNAL_TABLE + -- - MOVE_FILE + IS + vSourceFileId NUMBER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + vSourceFileId := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName); + CREATE_EXTERNAL_TABLE(vSourceFileId); + VALIDATE_SOURCE_FILE_RECEIVED(vSourceFileId); + DROP_EXTERNAL_TABLE(vSourceFileId); +-- COPY_FILE(vSourceFileId, 'ODS'); + MOVE_FILE(vSourceFileId, 'ODS'); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => vSourceFileId, pStatus => 'READY_FOR_INGESTION'); + + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + -- -20001 + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + -- -20002 + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + -- -20003 + WHEN ENV_MANAGER.ERR_MULTIPLE_MATCH_FOR_SRCFILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); + -- -20004 + WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); + -- -20005 + WHEN ENV_MANAGER.ERR_MULTIPLE_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); + -- -20006 + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + -- -20007 + WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + -- -20008 + WHEN ENV_MANAGER.ERR_MULTI_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); + -- -20009 + WHEN ENV_MANAGER.ERR_FILE_NOT_FOUND_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD); + -- -20010 + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_VALIDATION_FAILED), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); + -- -20011 + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NO_CONFIG_MATCH), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH, ENV_MANAGER.MSG_NO_CONFIG_MATCH); + -- -20012 + WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN_PREFIX), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); + -- -20013 + WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_TABLE_NOT_EXISTS), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS); + -- -20014 + WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_COLUMN_NOT_EXISTS), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS); + -- -20015 + WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE); + -- -20016 + WHEN ENV_MANAGER.ERR_MISSING_SOURCE_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_SOURCE_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); + -- -20017 + WHEN ENV_MANAGER.ERR_NULL_SOURCE_FILE_CONFIG_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY, ENV_MANAGER.MSG_NULL_SOURCE_FILE_CONFIG_KEY); + -- -20018 + WHEN ENV_MANAGER.ERR_DUPLICATED_SOURCE_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); + -- -20019 + WHEN ENV_MANAGER.ERR_MISSING_CONTAINER_CONFIG THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG, ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG); + -- -20020 + WHEN ENV_MANAGER.ERR_MULTIPLE_CONTAINER_ENTRIES THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); + -- -20021 + WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + -- -20022 + WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); + -- -20023 + WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); + -- -20024 + WHEN ENV_MANAGER.ERR_WRONG_DATE_TIMESTAMP_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); + + -- -20011 + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED); + + -- -20999 + WHEN ENV_MANAGER.ERR_UNKNOWN THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END PROCESS_SOURCE_FILE; + + +---------------------------------------------------------------------------------------------------- + + FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PROCESS_SOURCE_FILE(pSourceFileReceivedName => pSourceFileReceivedName); + ---- + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PROCESS_SOURCE_FILE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DATE_FORMAT( + pTemplateTableName IN VARCHAR2, + pColumnName IN VARCHAR2 + ) + RETURN VARCHAR2 + IS + vDateFormat A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vColumnName VARCHAR2(200); + vGetDefault BOOLEAN := FALSE; + BEGIN + vColumnName := trim(pColumnName); + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnName => '''||nvl(vColumnName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + BEGIN + SELECT DATE_FORMAT + INTO vDateFormat + FROM CT_MRDS.A_COLUMN_DATE_FORMAT F + WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName + AND F.COLUMN_NAME = vColumnName; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vGetDefault := TRUE; + WHEN TOO_MANY_ROWS THEN + -- Below error should not happened because: + -- Unique constraint added on table A_COLUMN_DATE_FORMAT on columns: (TEMPLATE_TABLE_NAME, COLUMN_NAME) + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); + END; + IF vGetDefault THEN + BEGIN + SELECT DATE_FORMAT + INTO vDateFormat + FROM CT_MRDS.A_COLUMN_DATE_FORMAT F + WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName + AND F.COLUMN_NAME = 'DEFAULT'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vDateFormat := ENV_MANAGER.gvDefaultDateFormat; + END; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vDateFormat; + END GET_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS ( + + pTemplateTableName IN VARCHAR2, + pColumnList OUT CLOB, + pFieldList OUT CLOB + ) + IS + vSchemaName VARCHAR2(200); + vTableName VARCHAR2(200); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vMaxColumnNameLength PLS_INTEGER := 0; + vColType varchar2(200); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnList => '''||nvl(pColumnList, 'NULL')||'''' + ,'pFieldList = '''||nvl(pFieldList, 'NULL')||'''' + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + vSchemaName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.',''); + ENV_MANAGER.LOG_PROCESS_EVENT('vSchemaName','DEBUG', vSchemaName); + vTableName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'\..*'),'.',''); + ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG', vTableName); + FOR rec IN ( + SELECT + '"'||column_name||'"' as quoted_column_name, + column_name, + data_type, + data_length, + char_length, -- MARS-1056: Added for CHAR/BYTE semantics + char_used, -- MARS-1056: Added for CHAR/BYTE semantics + data_precision, + data_scale, + column_id, + max(length(column_name)+1) over (partition by table_name) as max_column_name_length + FROM all_tab_columns + WHERE table_name = UPPER(vTableName) + AND owner = NVL(UPPER(vSchemaName), USER) + ORDER BY column_id + ) LOOP + -- Build the column_list string + rec.quoted_column_name := rpad(rec.quoted_column_name, rec.max_column_name_length+2, ' '); + + vColType := + CASE + -- MARS-1056: Fixed VARCHAR2 definition logic to preserve CHAR/BYTE semantics + WHEN rec.data_type = 'VARCHAR2' THEN + CASE + WHEN rec.char_used = 'C' THEN + rec.quoted_column_name || ' VARCHAR2(' || rec.char_length || ' CHAR)' + WHEN rec.char_used = 'B' THEN + rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ' BYTE)' + ELSE + -- Fallback for NULL char_used (should not occur but handle gracefully) + rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ')' + END + -- Other character types (preserve original logic) + WHEN rec.data_type IN ('CHAR', 'NCHAR', 'NVARCHAR2') THEN + rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')' + WHEN rec.data_type = 'NUMBER' THEN + rec.quoted_column_name || ' ' || rec.data_type || + CASE + WHEN rec.data_precision IS NOT NULL AND rec.data_scale IS NOT NULL THEN + '(' || rec.data_precision || ',' || rec.data_scale || ')' + WHEN rec.data_precision IS NOT NULL THEN + '(' || rec.data_precision || ')' + ELSE + '' + END + WHEN rec.data_type IN ('RAW') THEN + rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')' + WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') IN ('DATE', 'TIMESTAMP') THEN + rec.quoted_column_name || ' ' || rec.data_type + ELSE + rec.quoted_column_name || ' ' || rec.data_type + END; + pColumnList := pColumnList ||vColType ||cgBL|| ','; + -- Build the field_list string + -- Note: field_list uses CHAR() for CSV field definitions - this is correct behavior + pFieldList := pFieldList || + CASE + WHEN rec.data_type = 'DATE' THEN + -- MARS-1046: DATE format - wrap with NORMALIZE_DATE_FORMAT to fix ISO 8601 'T' separator + rec.quoted_column_name || ' DATE ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN rec.data_type LIKE 'TIMESTAMP%WITH TIME ZONE' THEN + -- MARS-1046: TIMESTAMP WITH TIME ZONE format for ISO 8601 with fractional seconds and timezone + -- Syntax: column_name CHAR(length) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK "format" + -- Use fixed length of 50 for ISO 8601 format (e.g., "2012-03-02T14:16:23.798+01:00" = 29 chars) + rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN + -- Other TIMESTAMP types (without timezone) + -- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format') + rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN + -- For CSV field definitions, use data_length for CHAR() specification + rec.quoted_column_name || ' CHAR(' || rec.data_length || ')' + ELSE + rec.quoted_column_name + END ||cgBL|| ','; + + + END LOOP; + + -- Remove the trailing comma and space from the strings + pColumnList := ' '||RTRIM(pColumnList, ','); + pFieldList := ' '||RTRIM(pFieldList, ','); + ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', pColumnList); + -- TO_DO !!! + -- Add check if pColumnList/pFieldList is empty or not + -- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + + -- Output the generated column_list and field_list + ENV_MANAGER.LOG_PROCESS_EVENT('column_list' ,'DEBUG',pColumnList); + ENV_MANAGER.LOG_PROCESS_EVENT('field_list' ,'DEBUG',pFieldList); + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION +-- WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); +-- RAISE_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GENERATE_EXTERNAL_TABLE_PARAMS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_SOURCE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE, + pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + INSERT INTO CT_MRDS.A_SOURCE(A_SOURCE_KEY, SOURCE_NAME) VALUES (pSourceKey, pSourceName); + COMMIT; + EXCEPTION + WHEN DUP_VAL_ON_INDEX THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ADD_SOURCE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DELETE_SOURCE_CASCADE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vSharedTemplateCount PLS_INTEGER := 0; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceKey => '''||nvl(pSourceKey, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- First pass: Delete container files (those that have CONTAINER_FILE_KEY set) + for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG + where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NOT NULL + ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop + -- Delete processed file records + delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for container config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + + -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats + IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN + SELECT COUNT(DISTINCT A_SOURCE_KEY) + INTO vSharedTemplateCount + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME + AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted + + -- Only delete date formats if template table is not shared with other sources + IF vSharedTemplateCount = 0 THEN + delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); + END IF; + END IF; + + -- Delete container file configuration + delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted container A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + end loop; + COMMIT; -- Commit container deletions + + -- Second pass: Delete parent files (those that do NOT have CONTAINER_FILE_KEY set) + for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG + where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NULL + ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop + -- Delete processed file records + delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for parent config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + + -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats + IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN + SELECT COUNT(DISTINCT A_SOURCE_KEY) + INTO vSharedTemplateCount + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME + AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted + + -- Only delete date formats if template table is not shared with other sources + IF vSharedTemplateCount = 0 THEN + delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); + END IF; + END IF; + + -- Delete parent file configuration + delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted parent A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + end loop; + COMMIT; -- Commit parent deletions + + -- Delete source system record + DELETE FROM CT_MRDS.A_SOURCE where A_SOURCE_KEY = pSourceKey; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE record for source key: '||pSourceKey,'DEBUG', vParameters); + COMMIT; -- Final commit for source deletion + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END DELETE_SOURCE_CASCADE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY ( + pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ) RETURN PLS_INTEGER + IS + vSourceFileConfigKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileId => '||nvl(to_char(pSourceFileId), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT A_SOURCE_FILE_CONFIG_KEY + INTO vSourceFileConfigKey + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_ID = pSourceFileId + AND SOURCE_FILE_TYPE = 'CONTAINER'; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileConfigKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG := 'No match in A_SOURCE_FILE_CONFIG where SOURCE_FILE_TYPE=''CONTAINER'' and SOURCE_FILE_ID = '''||pSourceFileId||''''; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG, 'WARNING', vParameters); + RETURN NULL; + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES := 'GET_CONTAINER_SOURCE_FILE_CONFIG_KEY: Multiple SOURCE_FILE_TYPE=''CONTAINER'' matches for SOURCE_FILE_ID: '||pSourceFileId||' in A_SOURCE_FILE_CONFIG'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END GET_CONTAINER_SOURCE_FILE_CONFIG_KEY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_CONFIG_KEY ( + pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT' + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE + ) RETURN PLS_INTEGER + IS + vSourceFileConfigKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileType => '''||nvl(pSourceFileType, 'NULL')||'''' + ,'pSourceFileId => '''||nvl(pSourceFileId, 'NULL')||'''' + ,'pTableId => '''||nvl(pTableId, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT A_SOURCE_FILE_CONFIG_KEY + INTO vSourceFileConfigKey + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = pSourceFileType + AND SOURCE_FILE_ID = pSourceFileId + AND TABLE_ID = pTableId; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG', vParameters); + RETURN vSourceFileConfigKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + WHEN TOO_MANY_ROWS THEN + vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE + ||cgBL||' '||'GET_SOURCE_FILE_CONFIG_KEY: Multiple matches in A_SOURCE_FILE_CONFIG table WHERE' + ||cgBL||' '||'SOURCE_FILE_TYPE: '||pSourceFileType + ||cgBL||' '||'SOURCE_FILE_ID: '||pSourceFileId + ||cgBL||' '||'TABLE_ID: '||pTableId; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, vgMsgTmp); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END GET_SOURCE_FILE_CONFIG_KEY; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_SOURCE_FILE_CONFIG ( + pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE + ,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE + ,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL + ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL + ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ) IS + vSourceFileConfigKey PLS_INTEGER; + vSourceKeyExists PLS_INTEGER := 0; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => '''||nvl(to_char(pSourceKey), 'NULL')||'''' + ,'pSourceFileType => '''||nvl(to_char(pSourceFileType), 'NULL')||'''' + ,'pSourceFileId => '''||nvl(to_char(pSourceFileId), 'NULL')||'''' + ,'pSourceFileDesc => '''||nvl(to_char(pSourceFileDesc), 'NULL')||'''' + ,'pSourceFileNamePattern => '''||nvl(to_char(pSourceFileNamePattern), 'NULL')||'''' + ,'pTableId => '''||nvl(to_char(pTableId), 'NULL')||'''' + ,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||'''' + ,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||'''' + ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING) + VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding); + COMMIT; + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + + WHEN OTHERS THEN + IF SQLCODE = -2291 THEN + ENV_MANAGER.MSG_MISSING_SOURCE_KEY := 'The Source with A_SOURCE_KEY: '''||pSourceKey||''' not found in parent table A_SOURCE.' + ||cgBL||'First add a record to A_SOURCE:' + ||cgBL||'call file_manager.add_source(pSourceKey => '''||pSourceKey||''', pSourceName => ''...'')'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_SOURCE_KEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END IF; + + END ADD_SOURCE_FILE_CONFIG; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_COLUMN_DATE_FORMAT ( + pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE + ,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE + ,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnName => '''||nvl(pColumnName, 'NULL')||'''' + ,'pDateFormat => '''||nvl(pDateFormat, 'NULL')||'''' + )); + + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + INSERT INTO CT_MRDS.A_COLUMN_DATE_FORMAT(TEMPLATE_TABLE_NAME, COLUMN_NAME, DATE_FORMAT) + VALUES (pTemplateTableName, pColumnName, pDateFormat); + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ADD_COLUMN_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2) + RETURN VARCHAR2 + IS + BEGIN + CASE pBucketArea + WHEN 'INBOX' THEN RETURN ENV_MANAGER.gvInboxBucketUri; + WHEN 'ODS' THEN RETURN ENV_MANAGER.gvDataBucketUri; + WHEN 'DATA' THEN RETURN ENV_MANAGER.gvDataBucketUri; + WHEN 'ARCHIVE' THEN RETURN ENV_MANAGER.gvArchiveBucketUri; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_INVALID_BUCKET_AREA, + ENV_MANAGER.MSG_INVALID_BUCKET_AREA || ' Provided: ''' || pBucketArea || ''''); + END CASE; + END; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Configuration entry + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vContainerFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vCount PLS_INTEGER := 0; + addColHeader BOOLEAN := TRUE; + vMsgTmp VARCHAR2(32000):= ''; + CURSOR cColumnFormat(vTableName A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE) IS + SELECT * + FROM CT_MRDS.A_COLUMN_DATE_FORMAT + WHERE TEMPLATE_TABLE_NAME = vTableName; + vColumnDateFormat A_COLUMN_DATE_FORMAT%ROWTYPE; + + FUNCTION FORMAT_CONFIG( pSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE, + pTypeInfo VARCHAR2 DEFAULT 'File Configuration', + pLevel VARCHAR2 DEFAULT '') + RETURN VARCHAR2 + IS + vMsg VARCHAR2(9999); + BEGIN + vMsg := '' + ||cgBL||pLevel||''||'Details about '||pTypeInfo||':' + ||cgBL||pLevel||''||'--------------------------------' + ||cgBL||pLevel||'A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||pLevel||'A_SOURCE_KEY = '||pSourceFileConfig.A_SOURCE_KEY + ||cgBL||pLevel||'SOURCE_FILE_TYPE = '||pSourceFileConfig.SOURCE_FILE_TYPE + ||cgBL||pLevel||'SOURCE_FILE_ID = '||pSourceFileConfig.SOURCE_FILE_ID + ||cgBL||pLevel||'SOURCE_FILE_DESC = '||pSourceFileConfig.SOURCE_FILE_DESC + ||cgBL||pLevel||'SOURCE_FILE_NAME_PATTERN = '||pSourceFileConfig.SOURCE_FILE_NAME_PATTERN + ||cgBL||pLevel||'TABLE_ID = '||pSourceFileConfig.TABLE_ID + ||cgBL||pLevel||'TEMPLATE_TABLE_NAME = '||pSourceFileConfig.TEMPLATE_TABLE_NAME + ||cgBL||pLevel||'CONTAINER_FILE_KEY = '||pSourceFileConfig.CONTAINER_FILE_KEY + ||cgBL||pLevel||'ODS_SCHEMA_NAME = '||pSourceFileConfig.ODS_SCHEMA_NAME + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_DAYS = '||pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_FILES_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT + ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS + + ||cgBL||pLevel||''||'--------------------------------' + ; + RETURN vMsg; + END FORMAT_CONFIG; + + BEGIN + vMsgTmp := ''; + -- Get Main Config + BEGIN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + vMsgTmp := FORMAT_CONFIG(pSourceFileConfig => vSourceFileConfig, pTypeInfo => 'File Configuration'); + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no config entry in A_SOURCE_FILE_CONFIG where A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfigKey; + RETURN vMsgTmp; + END; + + -- Get Container Config + IF pIncludeContainerInfo > 0 THEN + BEGIN + SELECT * + INTO vContainerFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = vSourceFileConfig.CONTAINER_FILE_KEY; + + vMsgTmp := vMsgTmp || cgBL || FORMAT_CONFIG(pSourceFileConfig => vContainerFileConfig, pTypeInfo => 'related Container Config', pLevel => ' '); + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := vMsgTmp|| cgBL || 'There is no CONTAINER config entry in A_SOURCE_FILE_CONFIG.'; + -- RETURN vMsgTmp; + END; + END IF; + + -- Get Column Date Format Config + IF pIncludeColumnFormatInfo > 0 THEN + BEGIN + OPEN cColumnFormat(vTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME); + LOOP + FETCH cColumnFormat INTO vColumnDateFormat; + IF cColumnFormat%FOUND AND addColHeader THEN + vCount := 1; + vMsgTmp := vMsgTmp||cgBL|| cgBL || ' Column Date Format config entries:'; + vMsgTmp := vMsgTmp||cgBL||''||' --------------------------------'; + addColHeader := FALSE; + END IF; + EXIT WHEN cColumnFormat%NOTFOUND; + vMsgTmp := vMsgTmp + ||cgBL||' TEMPLATE_TABLE_NAME = '||vColumnDateFormat.TEMPLATE_TABLE_NAME + ||cgBL||' COLUMN_NAME = '||vColumnDateFormat.COLUMN_NAME + ||cgBL||' DATE_FORMAT = '||vColumnDateFormat.DATE_FORMAT + ||cgBL||''||' --------------------------------'; + END LOOP; + If vCount=0 THEN + vMsgTmp := vMsgTmp || cgBL || 'There is no Column Date Format config entries in A_COLUMN_DATE_FORMAT where TEMPLATE_TABLE_NAME = '||NVL(vSourceFileConfig.TEMPLATE_TABLE_NAME,'NULL'); + END IF; + CLOSE cColumnFormat; + END; + END IF; + + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_SOURCE_FILE_CONFIG_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE + ,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Received entry + IS + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vMsgTmp VARCHAR2(32000):= ''; + BEGIN + + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + vMsgTmp := '' + ||cgBL||''||'Details about received file:' + ||cgBL||''||'--------------------------------' + ||cgBL||'A_SOURCE_FILE_RECEIVED_KEY = '||vSourceFileReceived.A_SOURCE_FILE_RECEIVED_KEY + ||cgBL||'A_SOURCE_FILE_CONFIG_KEY = '||vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||'SOURCE_FILE_NAME = '||vSourceFileReceived.SOURCE_FILE_NAME + ||cgBL||'CHECKSUM = '||vSourceFileReceived.CHECKSUM + ||cgBL||'CREATED = '||vSourceFileReceived.CREATED + ||cgBL||'BYTES = '||vSourceFileReceived.BYTES + ||cgBL||'RECEPTION_DATE = '||vSourceFileReceived.RECEPTION_DATE + ||cgBL||'PROCESSING_STATUS = '||vSourceFileReceived.PROCESSING_STATUS + ||cgBL||'EXTERNAL_TABLE_NAME = '||vSourceFileReceived.EXTERNAL_TABLE_NAME + ||cgBL||'PARTITION_YEAR = '||vSourceFileReceived.PARTITION_YEAR + ||cgBL||'PARTITION_MONTH = '||vSourceFileReceived.PARTITION_MONTH + ||cgBL||''||'--------------------------------' + ; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no data in A_SOURCE_FILE_RECEIVED where A_SOURCE_FILE_RECEIVED_KEY = '||pSourceFileReceivedKey; + RETURN vMsgTmp; + END; + + IF pIncludeConfigInfo>0 THEN + vMsgTmp := vMsgTmp || cgBL || CT_MRDS.FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO(vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY,pIncludeContainerInfo,pIncludeColumnFormatInfo); + END IF; + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_SOURCE_FILE_RECEIVED_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_USER_LOAD_OPERATIONS ( + pOperationId PLS_INTEGER + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Received entry + IS + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vMsgTmp VARCHAR2(32000):= ''; + BEGIN + + BEGIN + SELECT * + INTO vUserLoadOperations + FROM USER_LOAD_OPERATIONS + WHERE id = pOperationId; + + vMsgTmp := '' + ||''||'Details about USER_LOAD_OPERATIONS where ID = '||pOperationId + ||cgBL||''||'--------------------------------' + ||cgBL||'ID = '||vUserLoadOperations.ID + ||cgBL||'TYPE = '||vUserLoadOperations.TYPE + ||cgBL||'SID = '||vUserLoadOperations.SID + ||cgBL||'SERIAL# = '||vUserLoadOperations.SERIAL# + ||cgBL||'START_TIME = '||vUserLoadOperations.START_TIME + ||cgBL||'UPDATE_TIME = '||vUserLoadOperations.UPDATE_TIME + ||cgBL||'STATUS = '||vUserLoadOperations.STATUS + ||cgBL||'OWNER_NAME = '||vUserLoadOperations.OWNER_NAME + ||cgBL||'TABLE_NAME = '||vUserLoadOperations.TABLE_NAME + ||cgBL||'PARTITION_NAME = '||vUserLoadOperations.PARTITION_NAME + ||cgBL||'SUBPARTITION_NAME = '||vUserLoadOperations.SUBPARTITION_NAME + ||cgBL||'FILE_URI_LIST = '||vUserLoadOperations.FILE_URI_LIST + ||cgBL||'ROWS_LOADED = '||vUserLoadOperations.ROWS_LOADED + ||cgBL||'LOGFILE_TABLE = '||vUserLoadOperations.LOGFILE_TABLE + ||cgBL||'BADFILE_TABLE = '||vUserLoadOperations.BADFILE_TABLE + ||cgBL||'STATUS_TABLE = '||vUserLoadOperations.STATUS_TABLE + ||cgBL||'TEMPEXT_TABLE = '||vUserLoadOperations.TEMPEXT_TABLE + ||cgBL||'CREDENTIAL_NAME = '||vUserLoadOperations.CREDENTIAL_NAME + ||cgBL||'EXPIRATION_TIME = '||vUserLoadOperations.EXPIRATION_TIME + ||cgBL||''||'--------------------------------' + ; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no data in USER_LOAD_OPERATIONS where ID = '||pOperationId; + RETURN vMsgTmp; + END; + + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_USER_LOAD_OPERATIONS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ANALYZE_VALIDATION_ERRORS( + pSourceFileReceivedKey IN NUMBER + ) RETURN VARCHAR2 + IS + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vValidationLogTable VARCHAR2(128); + vTemplateSchema VARCHAR2(128); + vTemplateTable VARCHAR2(128); + vCsvFileUri VARCHAR2(4000); + vParameters VARCHAR2(4000); + vResult VARCHAR2(32000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => ' || pSourceFileReceivedKey + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start', 'DEBUG', vParameters); + + -- Get file and config information + BEGIN + -- Get source file received data first + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr + WHERE sfr.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + -- Get source file config data + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc + WHERE sfc.A_SOURCE_FILE_CONFIG_KEY = vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_ERROR('Source file or config not found for key: ' || pSourceFileReceivedKey, vParameters); + RETURN 'Error: Source file with key ' || pSourceFileReceivedKey || ' not found in A_SOURCE_FILE_RECEIVED table'; + END; + + -- Extract template schema and table from template table name + vTemplateSchema := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '^([^.]+)'); + vTemplateTable := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '\.(.+)$', 1, 1, NULL, 1); + + -- Build CSV file URI + vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || + 'INBOX/' || vSourceFileConfig.A_SOURCE_KEY || '/' || + vSourceFileConfig.SOURCE_FILE_ID || '/' || + vSourceFileConfig.TABLE_ID || '/' || + vSourceFileReceived.SOURCE_FILE_NAME; + + -- Find validation log table (most recent one with errors) + -- Look for validation log tables related to this external table + BEGIN + SELECT table_name + INTO vValidationLogTable + FROM ( + SELECT table_name, + REGEXP_SUBSTR(table_name, '\$([0-9]+)_', 1, 1, NULL, 1) as log_number + FROM USER_TABLES + WHERE table_name LIKE 'VALIDATE$%_LOG' + ORDER BY TO_NUMBER(log_number) DESC + ) + WHERE ROWNUM = 1; + EXCEPTION + WHEN NO_DATA_FOUND THEN + -- If no validation log tables found, use a default name + vValidationLogTable := 'VALIDATE$999_LOG'; + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('Calling ENV_MANAGER.ANALYZE_VALIDATION_ERRORS with parameters: ' || + 'LogTable=' || vValidationLogTable || + ', Schema=' || vTemplateSchema || + ', Table=' || vTemplateTable || + ', URI=' || SUBSTR(vCsvFileUri, 1, 100) || '...', 'DEBUG', vParameters); + + -- Call the main function with derived parameters + vResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( + pValidationLogTable => vValidationLogTable, + pTemplateSchema => vTemplateSchema, + pTemplateTable => vTemplateTable, + pCsvFileUri => vCsvFileUri + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End', 'DEBUG', vParameters); + RETURN vResult; + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_ERROR('Error in ANALYZE_VALIDATION_ERRORS: ' || SQLERRM, vParameters); + RETURN 'Error analyzing validation errors: ' || SQLERRM || + cgBL || 'Source File Key: ' || pSourceFileReceivedKey || + cgBL || 'Check A_SOURCE_FILE_RECEIVED and A_SOURCE_FILE_CONFIG tables for data integrity.'; + END ANALYZE_VALIDATION_ERRORS; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_MANAGER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_MANAGER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg new file mode 100644 index 0000000..a820802 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg @@ -0,0 +1,639 @@ +create or replace PACKAGE CT_MRDS.FILE_MANAGER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) || + '3.3.2 (2026-02-20): MARS-828 - Fixed threshold column names in GET_DET_SOURCE_FILE_CONFIG_INFO for MARS-828 compatibility' || CHR(13)||CHR(10) || + '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) || + '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) || + '3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) || + '3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) || + '3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) || + '2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) || + '2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-01): Initial release with file processing and validation capabilities'; + + TYPE tSourceFileReceived IS RECORD + ( + A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE, + A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE, + SOURCE_FILE_PREFIX_INBOX VARCHAR2(430), + SOURCE_FILE_PREFIX_ODS VARCHAR2(430), + SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430), + SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430), + SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE, + RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE, + PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE, + EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE + ); + + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + vgSourceFileConfigKey PLS_INTEGER; + vgMsgTmp VARCHAR2(32000); + + + + --------------------------------------------------------------------------------------------------------------------------- + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_SOURCE_FILE_CONFIG + * @desc Get source file type by matching the source file name against source file type naming patterns + * or by specifying the id of a received source file. + * @example ... + * @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE" + **/ + FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL + , pSourceFileReceivedKey IN NUMBER DEFAULT NULL + , pSourceFileConfigKey IN NUMBER DEFAULT NULL) + RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + + + + /** + * @name REGISTER_SOURCE_FILE_RECEIVED + * @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table. + * This overload automatically determines source file type from the file name. + * It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record. + * @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + * @ex_rslt 3245 + **/ + FUNCTION REGISTER_SOURCE_FILE_RECEIVED ( + pSourceFileReceivedName IN VARCHAR2 + ) + RETURN PLS_INTEGER; + + + + /** + * @name REGISTER_SOURCE_FILE_RECEIVED + * @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig. + * Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record. + * @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED( + * pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv' + * ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... ); + * @ex_rslt 3245 + **/ + FUNCTION REGISTER_SOURCE_FILE_RECEIVED ( + pSourceFileReceivedName IN VARCHAR2, + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) + RETURN PLS_INTEGER; + + + + /** + * @name SET_SOURCE_FILE_RECEIVED_STATUS + * @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column + * based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY + * and provided value of pStatus parameter + * @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION'); + **/ + PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS( + pSourceFileReceivedKey IN PLS_INTEGER, + pStatus IN VARCHAR2 + ); + + + + /** + * @name GET_EXTERNAL_TABLE_COLUMNS + * @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name. + * It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure. + * @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual; + * @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE, + * "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE, + * "REV_NUMBER" NUMBER(28,0), + * "REF_DATE" DATE, + * "FREE_TEXT" VARCHAR2(1000 CHAR), + * "MLF_BS_TOTAL" NUMBER(28,10), + * "DF_BS_TOTAL" NUMBER(28,10), + * "MLF_SF_TOTAL" NUMBER(28,10), + * "DF_SF_TOTAL" NUMBER(28,10) + **/ + FUNCTION GET_EXTERNAL_TABLE_COLUMNS ( + pTargetTableTemplate IN VARCHAR2 + ) + RETURN CLOB; + + + + /** + * @name CREATE_EXTERNAL_TABLE + * @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table + * MARS-1049: Added pEncoding parameter for CSV character set specification + * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252') + * If provided, adds CHARACTERSET clause to external table definition + * @example + * begin + * FILE_MANAGER.CREATE_EXTERNAL_TABLE( + * pTableName => 'STANDING_FACILITIES_HEADER', + * pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER', + * pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/', + * pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/', + * pFileName => NULL, + * pDelimiter => ',', + * pEncoding => 'UTF8' + * ); + * end; + **/ + PROCEDURE CREATE_EXTERNAL_TABLE ( + pTableName IN VARCHAR2, + pTemplateTableName IN VARCHAR2, + pPrefix IN VARCHAR2, + pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri, + pFileName IN VARCHAR2 DEFAULT NULL, + pDelimiter IN VARCHAR2 DEFAULT ',', + pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ); + + + + /** + * @name CREATE_EXTERNAL_TABLE + * @desc Creates External Table for single file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);; + **/ + PROCEDURE CREATE_EXTERNAL_TABLE ( + pSourceFileReceivedKey IN NUMBER + ); + + + + /** + * @name VALIDATE_SOURCE_FILE_RECEIVED + * @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + * It validate External table build upon single file + * provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377); + **/ + PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED + ( + pSourceFileReceivedKey IN NUMBER + ); + + + /** + * @name VALIDATE_EXTERNAL_TABLE + * @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE. + * It validates External Table provided by parameter pTableName. + * It returns: PASSED or FAILED. + * @example + * declare + * vStatus VARCHAR2(100); + * begin + * vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER'); + * DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus); + * end; + * + * @ex_rslt FAILED + **/ + FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2; + + + /** + * @name S_VALIDATE_EXTERNAL_TABLE + * @desc A function which checks if SELECT query reterns any rows. + * It trys to selects External Table provided by parameter pTableName. + * It returns: PASSED or FAILED. + * @example + * declare + * vStatus VARCHAR2(100); + * begin + * vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER'); + * DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus); + * end; + * + * @ex_rslt PASSED + **/ + FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2; + + + + /** + * @name DROP_EXTERNAL_TABLE + * @desc It drops External Table for single file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377); + **/ + PROCEDURE DROP_EXTERNAL_TABLE ( + pSourceFileReceivedKey IN NUMBER + ); + + + + /** + * @name COPY_FILE + * @desc It copies file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * into destination provided by pDestination parameter. + * pDestination parameter allowed values are: 'ODS' + * @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS'); + **/ + PROCEDURE COPY_FILE( + pSourceFileReceivedKey IN NUMBER, + pDestination IN VARCHAR2 + ); + + + + + /** + * @name MOVE_FILE + * @desc It moves file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * into destination provided by pDestination parameter. + * pDestination parameter allowed values are: 'ODS', 'QUARANTINE' + * @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS'); + **/ + PROCEDURE MOVE_FILE( + pSourceFileReceivedKey IN NUMBER, + pDestination IN VARCHAR2 + ); + + + + /** + * @name DELETE_FOLDER_CONTENTS + * @desc It deletes all files from specified folder in the cloud storage. + * The procedure lists all objects in the specified folder prefix and deletes them one by one. + * pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE' + * pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/') + * @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/'); + **/ + PROCEDURE DELETE_FOLDER_CONTENTS( + pBucketArea IN VARCHAR2, + pFolderPrefix IN VARCHAR2 + ); + + + + + /** + * @name PROCESS_SOURCE_FILE + * @desc It process file provided by pSourceFileReceivedName parameter. + * Ubmrella procedure that calls: + * - REGISTER_SOURCE_FILE_RECEIVED; + * - CREATE_EXTERNAL_TABLE; + * - VALIDATE_SOURCE_FILE_RECEIVED; + * - DROP_EXTERNAL_TABLE; + * - MOVE_FILE; + * @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + **/ + PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + ; + + + + /** + * @name PROCESS_SOURCE_FILE + * @desc It process file provided by pSourceFileReceivedName parameter and return processing result value. + * It returns (success/failure) => 0 / -(value). + * Ubmrella function that calls PROCESS_SOURCE_FILE procedure. + * @example + * declare + * vResult PLS_INTEGER; + * begin + * vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + * DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult); + * end; + * @ex_rslt 0 + * -20021 + **/ + FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER; + + + + /** + * @name GET_DATE_FORMAT + * @desc Returns date format for specified template table name and column name. + * Date is taken from configuration A_COLUMN_DATE_FORMAT table. + * @example select FILE_MANAGER.GET_DATE_FORMAT( + * pTemplateTableName => 'STANDING_FACILITIES_HEADER', + * pColumnName => 'SNAPSHOT_DATE') + * from dual; + * @ex_rslt DD/MM/YYYY HH24:MI:SS + **/ + FUNCTION GET_DATE_FORMAT( + pTemplateTableName IN VARCHAR2, + pColumnName IN VARCHAR2 + ) RETURN VARCHAR2; + + + + /** + * @name GENERATE_EXTERNAL_TABLE_PARAMS + * @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName. + * @example + * declare + * vColumnList CLOB; + * vFieldList CLOB; + * begin + * FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS ( + * pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER' + * ,pColumnList => vColumnList + * ,pFieldList => vFieldList + * ); + * DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList); + * DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList); + * end; + * / + **/ + PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS ( + pTemplateTableName IN VARCHAR2, + pColumnList OUT CLOB, + pFieldList OUT CLOB + ); + + + + + + /** + * @name ADD_SOURCE + * @desc Insert a new record to A_SOURCE table. + * pSourceKey is a PRIMARY KEY value. + **/ + PROCEDURE ADD_SOURCE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE, + pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE + ); + + + + /** + * @name DELETE_SOURCE_CASCADE + * @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables: + * - A_SOURCE_FILE_CONFIG + * - A_SOURCE_FILE_RECEIVED + * - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems) + * The procedure checks if template tables are shared before deleting date format configurations. + * If a template table is used by multiple source systems, date formats are preserved. + * @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS'); + **/ + PROCEDURE DELETE_SOURCE_CASCADE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE + ); + + + + /** + * @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY + * @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID) + * it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record. + * @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY( + * pSourceFileId => 'UC_DISSEM') + * from dual; + * @ex_rslt 126 + **/ + FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY ( + pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name GET_SOURCE_FILE_CONFIG_KEY + * @desc For specified input parameters, + * it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY. + * @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY ( + * pSourceFileType => 'INPUT' + * ,pSourceFileId => 'UC_DISSEM' + * ,pTableId => 'UC_NMA_DISSEM') + * from dual; + * @ex_rslt 126 + **/ + FUNCTION GET_SOURCE_FILE_CONFIG_KEY ( + pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT' + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name ADD_SOURCE_FILE_CONFIG + * @desc Insert a new record to A_SOURCE_FILE_CONFIG table. + * MARS-1049: Added pEncoding parameter for CSV character set specification. + * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2') + * If NULL, no CHARACTERSET clause is added to external table definitions + * @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( + * pSourceKey => 'C2D', pSourceFileType => 'INPUT', + * pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS', + * pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS', + * pEncoding => 'UTF8' + * ); + **/ + PROCEDURE ADD_SOURCE_FILE_CONFIG ( + pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE + ,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE + ,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL + ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL + ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ); + + + + /** + * @name ADD_COLUMN_DATE_FORMAT + * @desc Insert a new record to A_COLUMN_DATE_FORMAT table. + **/ + PROCEDURE ADD_COLUMN_DATE_FORMAT ( + pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE + ,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE + ,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE + ); + + + + /** + * @name GET_BUCKET_URI + * @desc Function used to get string with bucket http url. + * Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE' + * @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual; + * @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/ + **/ + FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2) + RETURN VARCHAR2; + + + + /** + * @name GET_DET_SOURCE_FILE_CONFIG_INFO + * @desc Function returns details about A_SOURCE_FILE_CONFIG record + * for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT) + * @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO ( + * pSourceFileConfigKey => 128 + * ,pIncludeContainerInfo => 1 + * ,pIncludeColumnFormatInfo => 1 + * ) from dual; + * @ex_rslt + * Details about File Configuration: + * -------------------------------- + * A_SOURCE_FILE_CONFIG_KEY = 128 + * A_SOURCE_KEY = C2D + * ... + * -------------------------------- + * + * Details about related Container Config: + * -------------------------------- + * A_SOURCE_FILE_CONFIG_KEY = 126 + * A_SOURCE_KEY = C2D + * ... + * -------------------------------- + * + * Column Date Format config entries: + * -------------------------------- + * TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM + * ... + * -------------------------------- + **/ + FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2; + + + + /** + * @name GET_DET_SOURCE_FILE_RECEIVED_INFO + * @desc Function returns details about A_SOURCE_FILE_RECEIVED record + * for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY). + * If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT) + * @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO ( + * pSourceFileReceivedKey => 377 + * ,pIncludeConfigInfo => 1 + * ,pIncludeContainerInfo => 1 + * ,pIncludeColumnFormatInfo => 1 + * ) from dual; + * + **/ + FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE + ,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2; + + + + /** + * @name GET_DET_USER_LOAD_OPERATIONS + * @desc Function returns details from USER_LOAD_OPERATIONS table + * for specified pOperationId. + * @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual; + * @ex_rslt + * Details about USER_LOAD_OPERATIONS where ID = 3608 + * -------------------------------- + * ID = 3608 + * TYPE = VALIDATE + * SID = 31260 + * SERIAL# = 52915 + * START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE + * UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE + * STATUS = FAILED + * OWNER_NAME = CT_MRDS + * TABLE_NAME = STANDING_FACILITIES_HEADER + * PARTITION_NAME = + * SUBPARTITION_NAME = + * FILE_URI_LIST = + * ROWS_LOADED = + * LOGFILE_TABLE = VALIDATE$3608_LOG + * BADFILE_TABLE = VALIDATE$3608_BAD + * STATUS_TABLE = + * TEMPEXT_TABLE = + * CREDENTIAL_NAME = + * EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE + * -------------------------------- + **/ + FUNCTION GET_DET_USER_LOAD_OPERATIONS ( + pOperationId PLS_INTEGER + ) RETURN VARCHAR2; + + /** + * @name ANALYZE_VALIDATION_ERRORS + * @desc Wrapper function that analyzes validation errors for a source file using its received key. + * Automatically derives template schema, table name, CSV URI and validation log table + * from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS. + * @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL; + * @ex_rslt Detailed validation analysis report with column mismatches and solutions + **/ + FUNCTION ANALYZE_VALIDATION_ERRORS( + pSourceFileReceivedKey IN NUMBER + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_MANAGER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL; + * @ex_rslt 3.2.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_MANAGER + * Version: 3.2.0 + * Build Date: 2025-10-22 16:30:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_MANAGER Version History: + * 3.2.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql b/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql new file mode 100644 index 0000000..993109b --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql @@ -0,0 +1,84 @@ +-- ============================================================================ +-- MARS-1409 Master Rollback Script +-- ============================================================================ +-- Author: Grzegorz Michalski +-- Purpose: Rollback A_WORKFLOW_HISTORY_KEY column changes from A_SOURCE_FILE_RECEIVED +-- Target Schema: CT_MRDS +-- Estimated Time: 1-2 minutes +-- Prerequisites: Backup of current FILE_MANAGER package, ADMIN privileges +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK ON +SET ECHO OFF + +-- Create log directory if it doesn't exist +host mkdir log 2>nul + +-- Generate dynamic SPOOL filename with timestamp +var filename VARCHAR2(100) +BEGIN + :filename := 'log/ROLLBACK_MARS_1409_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-1409 Rollback Starting +PROMPT ============================================================================ +PROMPT Package: CT_MRDS.FILE_MANAGER +PROMPT Change: Remove A_WORKFLOW_HISTORY_KEY column and restore previous version +PROMPT Steps: TBD (Restore packages, Drop column, Verify) +PROMPT Timestamp: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; +PROMPT ============================================================================ + +-- Confirm rollback with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20000, 'Rollback aborted by user'); + END IF; +END; +/ + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 1: Restore FILE_MANAGER package body (previous version) +PROMPT ============================================================================ +@@93_MARS_1409_rollback_FILE_MANAGER_BODY.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 2: Restore FILE_MANAGER package specification (previous version) +PROMPT ============================================================================ +@@92_MARS_1409_rollback_FILE_MANAGER_SPEC.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 3: Drop A_WORKFLOW_HISTORY_KEY column +PROMPT ============================================================================ +@@91_MARS_1409_rollback_workflow_history_key_column.sql + +PROMPT +PROMPT ============================================================================ +PROMPT STEP 4: Verify rollback +PROMPT ============================================================================ +@@90_MARS_1409_verify_rollback.sql + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-1409 Rollback Complete +PROMPT ============================================================================ +PROMPT Final Status: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_end FROM DUAL; +PROMPT +PROMPT Review the log file for detailed results: &_filename +PROMPT ============================================================================ + +spool off diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/A_SOURCE_FILE_RECEIVED.sql b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/A_SOURCE_FILE_RECEIVED.sql new file mode 100644 index 0000000..18739dc --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/A_SOURCE_FILE_RECEIVED.sql @@ -0,0 +1,66 @@ +-- ==================================================================== +-- A_SOURCE_FILE_RECEIVED Table +-- ==================================================================== +-- Purpose: Track received files and their processing status +-- ==================================================================== + +CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ( + A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE, + A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE, + SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL, + CHECKSUM VARCHAR2(128), + CREATED TIMESTAMP(6) WITH TIME ZONE, + BYTES NUMBER, + RECEPTION_DATE DATE NOT NULL, + PROCESSING_STATUS VARCHAR2(200), + EXTERNAL_TABLE_NAME VARCHAR2(200), + PARTITION_YEAR VARCHAR2(4), + PARTITION_MONTH VARCHAR2(2), + ARCH_FILE_NAME VARCHAR2(1000), + CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY), + CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')) +) TABLESPACE "DATA"; + +-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation) +CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1 +ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES); + +-- Column comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS + 'Primary key - unique identifier for received file record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS + 'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS + 'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS + 'MD5 checksum of file content for integrity verification and duplicate detection'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS + 'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS + 'File size in bytes'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS + 'Date when file was registered in the system (extracted from CREATED timestamp)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS + 'Name of temporary external table created for file validation (dropped after validation)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS + 'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS + 'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_FILE_NAME IS + 'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)'; + +GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkb b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkb new file mode 100644 index 0000000..856d449 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkb @@ -0,0 +1,1171 @@ +create or replace PACKAGE BODY CT_MRDS.ENV_MANAGER +AS + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE INIT_ERRORS IS + BEGIN + Errors(CODE_EMPTY_FILEURI_AND_RECKEY) := Error_Record(CODE_EMPTY_FILEURI_AND_RECKEY, MSG_EMPTY_FILEURI_AND_RECKEY); -- -20001 + Errors(CODE_NO_CONFIG_MATCH_FOR_FILEURI) := Error_Record(CODE_NO_CONFIG_MATCH_FOR_FILEURI, MSG_NO_CONFIG_MATCH_FOR_FILEURI); -- -20002 + Errors(CODE_MULTIPLE_MATCH_FOR_SRCFILE) := Error_Record(CODE_MULTIPLE_MATCH_FOR_SRCFILE, MSG_MULTIPLE_MATCH_FOR_SRCFILE); -- -20003 + Errors(CODE_MISSING_COLUMN_DATE_FORMAT) := Error_Record(CODE_MISSING_COLUMN_DATE_FORMAT, MSG_MISSING_COLUMN_DATE_FORMAT); -- -20004 + Errors(CODE_MULTIPLE_COLUMN_DATE_FORMAT) := Error_Record(CODE_MULTIPLE_COLUMN_DATE_FORMAT, MSG_MULTIPLE_COLUMN_DATE_FORMAT); -- -20005 + Errors(CODE_DIDNT_GET_LOAD_OPERATION_ID) := Error_Record(CODE_DIDNT_GET_LOAD_OPERATION_ID, MSG_DIDNT_GET_LOAD_OPERATION_ID); -- -20006 + Errors(CODE_NO_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_NO_CONFIG_FOR_RECEIVED_FILE, MSG_NO_CONFIG_FOR_RECEIVED_FILE); -- -20007 + Errors(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); -- -20008 + Errors(CODE_FILE_NOT_FOUND_ON_CLOUD) := Error_Record(CODE_FILE_NOT_FOUND_ON_CLOUD, MSG_FILE_NOT_FOUND_ON_CLOUD); -- -20009 + Errors(CODE_FILE_VALIDATION_FAILED) := Error_Record(CODE_FILE_VALIDATION_FAILED, MSG_FILE_VALIDATION_FAILED); -- -20010 + Errors(CODE_EXCESS_COLUMNS_DETECTED) := Error_Record(CODE_EXCESS_COLUMNS_DETECTED, MSG_EXCESS_COLUMNS_DETECTED); -- -20011 + Errors(CODE_NO_CONFIG_MATCH) := Error_Record(CODE_NO_CONFIG_MATCH, MSG_NO_CONFIG_MATCH); -- -20012 + Errors(CODE_UNKNOWN_PREFIX) := Error_Record(CODE_UNKNOWN_PREFIX, MSG_UNKNOWN_PREFIX); -- -20013 + Errors(CODE_TABLE_NOT_EXISTS) := Error_Record(CODE_TABLE_NOT_EXISTS, MSG_TABLE_NOT_EXISTS); -- -20014 + Errors(CODE_COLUMN_NOT_EXISTS) := Error_Record(CODE_COLUMN_NOT_EXISTS, MSG_COLUMN_NOT_EXISTS); -- -20015 + Errors(CODE_UNSUPPORTED_DATA_TYPE) := Error_Record(CODE_UNSUPPORTED_DATA_TYPE, MSG_UNSUPPORTED_DATA_TYPE); -- -20016 + Errors(CODE_MISSING_SOURCE_KEY) := Error_Record(CODE_MISSING_SOURCE_KEY, MSG_MISSING_SOURCE_KEY); -- -20017 + Errors(CODE_NULL_SOURCE_FILE_CONFIG_KEY) := Error_Record(CODE_NULL_SOURCE_FILE_CONFIG_KEY, MSG_NULL_SOURCE_FILE_CONFIG_KEY); -- -20018 + Errors(CODE_DUPLICATED_SOURCE_KEY) := Error_Record(CODE_DUPLICATED_SOURCE_KEY, MSG_DUPLICATED_SOURCE_KEY); -- -20019 + Errors(CODE_MISSING_CONTAINER_CONFIG) := Error_Record(CODE_MISSING_CONTAINER_CONFIG, MSG_MISSING_CONTAINER_CONFIG); -- -20020 + Errors(CODE_MULTIPLE_CONTAINER_ENTRIES) := Error_Record(CODE_MULTIPLE_CONTAINER_ENTRIES, MSG_MULTIPLE_CONTAINER_ENTRIES); -- -20021 + Errors(CODE_WRONG_DESTINATION_PARAM) := Error_Record(CODE_WRONG_DESTINATION_PARAM, MSG_WRONG_DESTINATION_PARAM); -- -20022 + Errors(CODE_FILE_NOT_EXISTS_ON_CLOUD) := Error_Record(CODE_FILE_NOT_EXISTS_ON_CLOUD, MSG_FILE_NOT_EXISTS_ON_CLOUD); -- -20023 + Errors(CODE_FILE_ALREADY_REGISTERED) := Error_Record(CODE_FILE_ALREADY_REGISTERED, MSG_FILE_ALREADY_REGISTERED); -- -20024 + Errors(CODE_WRONG_DATE_TIMESTAMP_FORMAT) := Error_Record(CODE_WRONG_DATE_TIMESTAMP_FORMAT, MSG_WRONG_DATE_TIMESTAMP_FORMAT); -- -20025 + Errors(CODE_ENVIRONMENT_NOT_SET) := Error_Record(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET); -- -20026 + Errors(CODE_CONFIG_VARIABLE_NOT_SET) := Error_Record(CODE_CONFIG_VARIABLE_NOT_SET, MSG_CONFIG_VARIABLE_NOT_SET); -- -20027 + Errors(CODE_NOT_INPUT_SOURCE_FILE_TYPE) := Error_Record(CODE_NOT_INPUT_SOURCE_FILE_TYPE, MSG_NOT_INPUT_SOURCE_FILE_TYPE); -- -20028 + Errors(CODE_EXP_DATA_FOR_ARCH_FAILED) := Error_Record(CODE_EXP_DATA_FOR_ARCH_FAILED, MSG_EXP_DATA_FOR_ARCH_FAILED); -- -20029 + Errors(CODE_RESTORE_FILE_FROM_TRASH) := Error_Record(CODE_RESTORE_FILE_FROM_TRASH, MSG_RESTORE_FILE_FROM_TRASH); -- -20030 + Errors(CODE_CHANGE_STAT_TO_ARCHIVED_FAILED):= Error_Record(CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); -- -20031 + Errors(CODE_MOVE_FILE_TO_TRASH_FAILED) := Error_Record(CODE_MOVE_FILE_TO_TRASH_FAILED, MSG_MOVE_FILE_TO_TRASH_FAILED); -- -20032 + Errors(CODE_DROP_EXPORTED_FILES_FAILED) := Error_Record(CODE_DROP_EXPORTED_FILES_FAILED, MSG_DROP_EXPORTED_FILES_FAILED); -- -20033 + Errors(CODE_INVALID_BUCKET_AREA) := Error_Record(CODE_INVALID_BUCKET_AREA, MSG_INVALID_BUCKET_AREA); -- -20034 + Errors(CODE_INVALID_PARALLEL_DEGREE) := Error_Record(CODE_INVALID_PARALLEL_DEGREE, MSG_INVALID_PARALLEL_DEGREE); -- -20110 + Errors(CODE_PARALLEL_EXECUTION_FAILED) := Error_Record(CODE_PARALLEL_EXECUTION_FAILED, MSG_PARALLEL_EXECUTION_FAILED); -- -20111 + + Errors(CODE_UNKNOWN) := Error_Record(CODE_UNKNOWN, MSG_UNKNOWN); -- -20999 + + END INIT_ERRORS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DEFAULT_ENV + RETURN VARCHAR2 + IS + vDefaultEnv CT_MRDS.a_file_manager_config.config_variable_value%TYPE; + BEGIN + select config_variable_value + into vDefaultEnv + from CT_MRDS.a_file_manager_config + where lower(environment_id)='default' + and lower(config_variable)='environmentid'; + RETURN vDefaultEnv; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + END; + + ---------------------------------------------------------------------------------------------------- + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE INIT_VARIABLES( + pEnv VARCHAR2 + ) IS + BEGIN + for rec in ( + select + ENVIRONMENT_ID + ,REGION + ,NAMESPACE + ,INBOXBUCKETNAME + ,DATABUCKETNAME + ,ARCHIVEBUCKETNAME + ,CREDENTIALNAME + ,LOGGINGENABLED + ,MINLOGLEVEL + ,DEFAULTDATEFORMAT + ,CONSOLELOGGINGENABLED + from ( + select environment_id, config_variable, config_variable_value from CT_MRDS.A_FILE_MANAGER_CONFIG + where environment_id=pEnv + ) + pivot ( + min(config_variable_value) + for config_variable in ( + 'Region' as Region + ,'NameSpace' as NameSpace + ,'InboxBucketName' as InboxBucketName + ,'DataBucketName' as DataBucketName + ,'ArchiveBucketName' as ArchiveBucketName + ,'CredentialName' as CredentialName + ,'LoggingEnabled' as LoggingEnabled + ,'MinLogLevel' as MinLogLevel + ,'DefaultDateFormat' as DefaultDateFormat + ,'ConsoleLoggingEnabled' as ConsoleLoggingEnabled) + ) + ) loop + if (rec.NAMESPACE is NULL + or rec.REGION is NULL + or rec.NAMESPACE is NULL + or rec.INBOXBUCKETNAME is NULL + or rec.DATABUCKETNAME is NULL + or rec.ARCHIVEBUCKETNAME is NULL + or rec.CREDENTIALNAME is NULL + ) THEN + vgMsgTmp := MSG_CONFIG_VARIABLE_NOT_SET + ||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'Region = '||rec.Region + ||cgBL||' '||'NameSpace = '||rec.NameSpace + ||cgBL||' '||'InboxBucketName = '||rec.InboxBucketName + ||cgBL||' '||'DataBucketName = '||rec.DataBucketName + ||cgBL||' '||'ArchiveBucketName = '||rec.ArchiveBucketName + ||cgBL||' '||'CredentialName = '||rec.CredentialName + ; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR'); + RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, vgMsgTmp); + + elsif (rec.LOGGINGENABLED is NULL + or rec.MINLOGLEVEL is NULL + or rec.DEFAULTDATEFORMAT is NULL + ) THEN + vgMsgTmp := 'Missing configuration variables' + ||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'LoggingEnabled = '||rec.LoggingEnabled + ||cgBL||' '||'MinLogLevel = '||rec.MinLogLevel + ||cgBL||' '||'DefaultDateFormat = '||rec.DefaultDateFormat + ; + LOG_PROCESS_EVENT(vgMsgTmp, 'WARNING'); + + else + gvNameSpace := rec.NAMESPACE; + gvRegion := rec.REGION; + gvInboxBucketName := rec.INBOXBUCKETNAME; + gvDataBucketName := rec.DATABUCKETNAME; + gvArchiveBucketName := rec.ARCHIVEBUCKETNAME; + gvCredentialName := rec.CREDENTIALNAME; + gvInboxBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.INBOXBUCKETNAME||'/o/'; + gvDataBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.DATABUCKETNAME||'/o/'; + gvArchiveBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.ARCHIVEBUCKETNAME||'/o/'; + gvLoggingEnabled := rec.LOGGINGENABLED; + gvMinLogLevel := rec.MINLOGLEVEL; + gvDefaultDateFormat := rec.DEFAULTDATEFORMAT; + gvConsoleLoggingEnabled := NVL(rec.CONSOLELOGGINGENABLED, 'ON'); + end if; + end loop; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vgMsgTmp := MSG_CONFIG_VARIABLE_NOT_SET + ||cgBL||' '||'No configuration found for environment_id='||pEnv||' in A_FILE_MANAGER_CONFIG table'; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', 'pEnv='||pEnv); + RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, vgMsgTmp); + WHEN OTHERS THEN + vgMsgTmp := 'Unexpected error while initializing variables for environment: '||pEnv + ||cgBL||' '||'SQLERRM: '||SQLERRM; + LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', 'pEnv='||pEnv); + RAISE; + END INIT_VARIABLES; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_ERROR_MESSAGE( + pCode PLS_INTEGER + ) RETURN VARCHAR2 + IS + BEGIN + RETURN Errors(pCode).message; + EXCEPTION + WHEN NO_DATA_FOUND THEN + LOG_PROCESS_EVENT('No error message found for pCode='||pCode , 'WARNING', 'pCode='||pCode); + LOG_PROCESS_EVENT('Update ENV_MANAGER package header with new code.' , 'WARNING', 'pCode='||pCode); + RETURN NULL; + WHEN OTHERS THEN + LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pCode='||pCode); + RAISE_APPLICATION_ERROR(CODE_UNKNOWN, MSG_UNKNOWN); + END GET_ERROR_MESSAGE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_ERROR_STACK( + pFormat VARCHAR2 + ,pCode PLS_INTEGER + ,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL + ) RETURN VARCHAR2 + IS + vFullErrorCore VARCHAR2(32000); + vFullErrorMsg VARCHAR2(32000); + BEGIN +-- vgErrorMessage := SQLERRM|| cgBL; +-- vgErrorStack := DBMS_UTILITY.FORMAT_ERROR_STACK; +-- vgErrorBacktrace := DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; + vFullErrorCore :='Error Message:' + ||cgBL|| SQLERRM|| cgBL + ||'-------------------------------------------------------' + ||cgBL||'Error Stack:' + ||cgBL|| DBMS_UTILITY.FORMAT_ERROR_STACK + ||'-------------------------------------------------------' + ||cgBL||'Error Backtrace:' + ||cgBL|| DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; +-- vFullErrorCore := REGEXP_REPLACE (vFullErrorCore, pCode||': ', pCode||': '||GET_ERROR_MESSAGE(pCode) , 1, 1); + IF (pFormat = 'TABLE') THEN + vFullErrorMsg := vFullErrorCore; + ELSE + vFullErrorMsg := cgBL||'------------------------------------------------------+' + ||cgBL||vFullErrorCore + ||'------------------------------------------------------+'; + END IF; +-- IF pSourceFileReceivedKey is not null THEN +-- vFullErrorMsg := vFullErrorMsg ||cgBL||GET_DET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey,1,1,1); +-- END IF; + + RETURN vFullErrorMsg; + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pFormat='||pFormat); + RETURN NULL; + END GET_ERROR_STACK; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FORMAT_PARAMETERS( + pParameterList SYS.ODCIVARCHAR2LIST + ) RETURN VARCHAR2 IS + vResult VARCHAR2(10000); + BEGIN + FOR i IN 1 .. pParameterList.COUNT LOOP +-- dbms_output.put_line('pParameterList(i): '||pParameterList(i)); + if i < pParameterList.COUNT then vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL') ||' ,'|| cgBL; + else vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL'); + end if; + END LOOP; + RETURN vResult; + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_EVENT('Error while formating parameters.' , 'WARNING'); + RETURN NULL; + END FORMAT_PARAMETERS; + + ---------------------------------------------------------------------------------------------------- + + + + PROCEDURE LOG_PROCESS_EVENT ( + pLogMessage VARCHAR2 + ,pLogLevel VARCHAR2 DEFAULT 'ERROR' + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ) IS + PRAGMA AUTONOMOUS_TRANSACTION; + + vLoggingEnabled VARCHAR2(10); + vMinLogLevel VARCHAR2(10); + vCallStack VARCHAR2(10000); + vProcedureName VARCHAR2(100); + vProcedureLevel PLS_INTEGER; + vTotalLines PLS_INTEGER; + vCurrentLine PLS_INTEGER; + + -- Map of priority level + TYPE logLevelMap IS TABLE OF NUMBER INDEX BY VARCHAR2(10); + vLogLevels logLevelMap; + + BEGIN + -- Prority logging level (higher -> more important) + vLogLevels('DEBUG') := 1; + vLogLevels('INFO') := 2; + vLogLevels('WARNING') := 3; + vLogLevels('ERROR') := 4; + + -- Check id logging is TURN-OFF + IF gvLoggingEnabled = 'OFF' THEN + RETURN; + END IF; + -- Check logging level + IF vLogLevels(pLogLevel) < vLogLevels(gvMinLogLevel) THEN + RETURN; + END IF; + + vCallStack := DBMS_UTILITY.FORMAT_CALL_STACK; + vProcedureName := REGEXP_SUBSTR(vCallStack, 'package body\s+\w+\.(\w+\.\w+)', 1, 2, NULL, 1); + vTotalLines := REGEXP_COUNT(vCallStack, CHR(10)) + 1; + vCurrentLine := REGEXP_COUNT(SUBSTR(vCallStack, 1, INSTR(vCallStack, vProcedureName) - 1), CHR(10)) + 1; + vProcedureLevel := (vTotalLines - vCurrentLine + 1) - 3; + vProcedureName := LPAD(vProcedureName, LENGTH(vProcedureName) + 2*vProcedureLevel, ' '); + + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, username, osuser, machine, module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES (guid, gvUsername, gvOsuser, gvMachine, gvModule, pProcessName, vProcedureName, pParameters, pLogLevel, pLogMessage); + + COMMIT; + + -- Also output to console for immediate visibility (if enabled) + IF gvConsoleLoggingEnabled = 'ON' THEN + DBMS_OUTPUT.PUT_LINE('[' || TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') || '] [' || pLogLevel || '] ' || vProcedureName || ': ' || pLogMessage); + END IF; + + END LOG_PROCESS_EVENT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE LOG_PROCESS_ERROR( + pLogMessage IN VARCHAR2, + pParameters IN VARCHAR2 DEFAULT NULL, + pProcessName IN VARCHAR2 DEFAULT 'FILE_MANAGER' + ) IS + PRAGMA AUTONOMOUS_TRANSACTION; + + vCallStack VARCHAR2(32767); + vErrorStack VARCHAR2(32767); + vErrorBacktrace VARCHAR2(32767); + vAdjustedBacktrace VARCHAR2(32767); + vErrorContext VARCHAR2(4000); + vProcName VARCHAR2(100); + vProcedureLevel PLS_INTEGER; + vTotalLines PLS_INTEGER; + vCurrentLine PLS_INTEGER; + vFullErrorMessage CLOB; + vTimestamp VARCHAR2(30); + vSessionInfo VARCHAR2(1000); + + BEGIN + -- Check if logging is disabled + IF gvLoggingEnabled = 'OFF' THEN + RETURN; + END IF; + + -- Capture all available error information + vErrorStack := DBMS_UTILITY.FORMAT_ERROR_STACK; + vErrorBacktrace := DBMS_UTILITY.FORMAT_ERROR_BACKTRACE; + vCallStack := DBMS_UTILITY.FORMAT_CALL_STACK; + vTimestamp := TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'); + + -- Capture session information for better context + vSessionInfo := 'Session ID: ' || SYS_CONTEXT('USERENV', 'SID') || + ', User: ' || SYS_CONTEXT('USERENV', 'SESSION_USER') || + ', Module: ' || SYS_CONTEXT('USERENV', 'MODULE') || + ', Client Info: ' || NVL(SYS_CONTEXT('USERENV', 'CLIENT_INFO'), 'N/A') || + ', Action: ' || NVL(SYS_CONTEXT('USERENV', 'ACTION'), 'N/A'); + + -- Build error context information + vErrorContext := 'Environment: ' || gvEnv || + ', Process: ' || NVL(pProcessName, 'UNKNOWN') || + ', Timestamp: ' || vTimestamp || + ', SQLCODE: ' || SQLCODE || + ', Transaction Active: ' || CASE WHEN DBMS_TRANSACTION.STEP_ID IS NOT NULL THEN 'YES' ELSE 'NO' END; + + -- Extract procedure name and nesting level from call stack + -- Always extract actual procedure name from call stack for precise error location + vProcName := REGEXP_SUBSTR(vCallStack, 'package body\s+\w+\.(\w+\.\w+)', 1, 2, NULL, 1); + + -- If we couldn't extract procedure name from call stack, use provided process name + IF vProcName IS NULL THEN + vProcName := NVL(pProcessName, 'UNKNOWN'); + END IF; + + vTotalLines := REGEXP_COUNT(vCallStack, CHR(10)) + 1; + vCurrentLine := REGEXP_COUNT(SUBSTR(vCallStack, 1, INSTR(vCallStack, vProcName) - 1), CHR(10)) + 1; + vProcedureLevel := (vTotalLines - vCurrentLine + 1) - 3; + vProcName := LPAD(vProcName, LENGTH(vProcName) + 2*vProcedureLevel, ' '); + + -- Enhance line number display to show direct _BODY.sql file line numbers + -- Since packages are now split into separate _SPEC and _BODY files, line numbers map directly + vAdjustedBacktrace := REGEXP_REPLACE(vErrorBacktrace, + 'at "CT_MRDS\.FILE_MANAGER", line ([0-9]+)', + 'at "CT_MRDS.FILE_MANAGER", line \1 (-> FILE_MANAGER_BODY.sql:line \1)', 1, 0, 'i'); + + vAdjustedBacktrace := REGEXP_REPLACE(vAdjustedBacktrace, + 'at "CT_MRDS\.ENV_MANAGER", line ([0-9]+)', + 'at "CT_MRDS.ENV_MANAGER", line \1 (-> ENV_MANAGER_BODY.sql:line \1)', 1, 0, 'i'); + + -- Build comprehensive error message with professional formatting + vFullErrorMessage := 'ERROR REPORT' || cgBL || + '-------------------------------------------------------' || cgBL || + 'ERROR SUMMARY' || cgBL || + ' Message: ' || pLogMessage || cgBL || + ' Context: ' || vErrorContext || cgBL || + '-------------------------------------------------------' || cgBL || + 'SESSION INFORMATION' || cgBL || + ' ' || vSessionInfo || cgBL || + '-------------------------------------------------------' || cgBL || + 'ERROR STACK (Oracle Internal)' || cgBL || + vErrorStack || + '-------------------------------------------------------' || cgBL || + 'BACKTRACE INFORMATION (Oracle Internal)' || cgBL || + vErrorBacktrace || + '-------------------------------------------------------' || cgBL || + 'CALL STACK (Execution Path)' || cgBL || + vCallStack || + '-------------------------------------------------------' || cgBL || + 'QUICK REFERENCE' || cgBL || + ' SQLCODE: ' || SQLCODE || cgBL || + ' SQLERRM: ' || SQLERRM || cgBL || + ' Timestamp: ' || vTimestamp || cgBL || + ' Parameters: ' || NVL(pParameters, 'None provided') || cgBL || + '-------------------------------------------------------'; + + -- Insert comprehensive error record into log table + -- Note: LOG_MESSAGE is VARCHAR2(4000), so we'll truncate if needed but include key info + INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, username, osuser, machine, module, process_name, procedure_name, procedure_parameters, log_level, log_message) + VALUES (guid, gvUsername, gvOsuser, gvMachine, gvModule, NVL(pProcessName, 'FILE_MANAGER'), vProcName, pParameters, 'ERROR', + CASE + WHEN LENGTH(vFullErrorMessage) <= 4000 THEN vFullErrorMessage + ELSE SUBSTR(vFullErrorMessage, 1, 3950) || '... [TRUNCATED]' + END); + + COMMIT; + + -- Enhanced console output for immediate visibility (if enabled) + IF gvConsoleLoggingEnabled = 'ON' THEN + DBMS_OUTPUT.PUT_LINE('======================================================='); + DBMS_OUTPUT.PUT_LINE('ERROR DETECTED AT: ' || vTimestamp); + DBMS_OUTPUT.PUT_LINE('PROCEDURE: ' || NVL(vProcName, 'UNKNOWN')); + DBMS_OUTPUT.PUT_LINE('MESSAGE: ' || pLogMessage); + DBMS_OUTPUT.PUT_LINE('SQLCODE: ' || SQLCODE || ' | ENVIRONMENT: ' || gvEnv); + -- Extract and show the most relevant file and line number + IF INSTR(vAdjustedBacktrace, '-> ') > 0 THEN + DBMS_OUTPUT.PUT_LINE('SOURCE FILE LOCATION: ' || REGEXP_SUBSTR(vAdjustedBacktrace, '-> [^)]+')); + END IF; + DBMS_OUTPUT.PUT_LINE('FULL DETAILS: Query A_PROCESS_LOG table for complete diagnostic info'); + DBMS_OUTPUT.PUT_LINE('QUERY (This Error): SELECT * FROM CT_MRDS.A_PROCESS_LOG WHERE GUID = ''' || guid || ''' ORDER BY LOG_TIMESTAMP DESC;'); + DBMS_OUTPUT.PUT_LINE('QUERY (Recent All): SELECT * FROM CT_MRDS.A_PROCESS_LOG WHERE LOG_TIMESTAMP >= SYSDATE - 1/1440 ORDER BY LOG_TIMESTAMP DESC;'); + DBMS_OUTPUT.PUT_LINE('======================================================='); + END IF; + + END LOG_PROCESS_ERROR; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ANALYZE_VALIDATION_ERRORS( + pValidationLogTable VARCHAR2, + pTemplateSchema VARCHAR2, + pTemplateTable VARCHAR2, + pCsvFileUri VARCHAR2 + ) RETURN VARCHAR2 + IS + vAnalysisReport CLOB := ''; + vCsvHeader VARCHAR2(4000); + vExpectedOrder VARCHAR2(4000); + vCsvOrder VARCHAR2(4000); + vErrorDetails VARCHAR2(32000) := ''; + vSolutions VARCHAR2(4000); + vColumnMismatch VARCHAR2(1000); + vErrorCount NUMBER := 0; + vFirstDataError VARCHAR2(1000); + vErrorColumn VARCHAR2(100); + vErrorValue VARCHAR2(500); + vExpectedType VARCHAR2(100); + vTemplateColCount NUMBER := 0; + vCsvColCount NUMBER := 0; + vExcessColumns VARCHAR2(2000); + vCsvFirstLine VARCHAR2(4000); + + -- Cursor for template table columns + CURSOR c_template_columns IS + SELECT COLUMN_NAME, DATA_TYPE, COLUMN_ID + FROM ALL_TAB_COLUMNS + WHERE OWNER = UPPER(REGEXP_SUBSTR(pTemplateSchema || '.' || pTemplateTable, '^([^.]+)')) + AND TABLE_NAME = UPPER(REGEXP_SUBSTR(pTemplateSchema || '.' || pTemplateTable, '\.(.+)$', 1, 1, NULL, 1)) + ORDER BY COLUMN_ID; + + BEGIN + -- Build expected column order from template table and count columns + FOR rec IN c_template_columns LOOP + IF vExpectedOrder IS NOT NULL THEN + vExpectedOrder := vExpectedOrder || ', '; + END IF; + vExpectedOrder := vExpectedOrder || rec.COLUMN_NAME; + vTemplateColCount := vTemplateColCount + 1; + END LOOP; + + -- Parse validation log table for errors and CSV structure + BEGIN + -- Try to extract error information from the validation log table + EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ' || pValidationLogTable || + ' WHERE record LIKE ''error processing column%''' + INTO vErrorCount; + + -- Get first error details + IF vErrorCount > 0 THEN + EXECUTE IMMEDIATE 'SELECT record FROM ' || pValidationLogTable || + ' WHERE record LIKE ''error processing column%'' AND ROWNUM = 1' + INTO vFirstDataError; + + -- Parse error to extract column name and error type + vErrorColumn := REGEXP_SUBSTR(vFirstDataError, 'error processing column ([A-Z_]+)', 1, 1, NULL, 1); + + -- Try to get the actual error value from ORA-01722 message + BEGIN + EXECUTE IMMEDIATE 'SELECT record FROM ' || pValidationLogTable || + ' WHERE record LIKE ''ORA-01722%'' AND ROWNUM = 1' + INTO vFirstDataError; + vErrorValue := REGEXP_SUBSTR(vFirstDataError, 'string value containing ''([^'']+)''', 1, 1, NULL, 1); + EXCEPTION + WHEN NO_DATA_FOUND THEN + vErrorValue := 'unknown value'; + WHEN OTHERS THEN + vErrorValue := 'parsing error'; + END; + END IF; + + -- Try to extract CSV structure from validation log field definitions + BEGIN + EXECUTE IMMEDIATE ' + SELECT LISTAGG( + REGEXP_SUBSTR(record, ''^\s+([A-Z_]+)\s+'', 1, 1, NULL, 1), + '', '' + ) WITHIN GROUP (ORDER BY ROWNUM) + FROM ' || pValidationLogTable || ' + WHERE record LIKE '' %CHAR%'' + AND record NOT LIKE ''%Fields in Data Source%'' + AND REGEXP_SUBSTR(record, ''^\s+([A-Z_]+)\s+'') IS NOT NULL' + INTO vCsvOrder; + + -- Count CSV columns from parsed structure + IF vCsvOrder IS NOT NULL THEN + vCsvColCount := REGEXP_COUNT(vCsvOrder, ',') + 1; + END IF; + + EXCEPTION + WHEN OTHERS THEN + vCsvOrder := 'Unable to determine CSV column order from validation log'; + END; + + -- Alternative method: Try to read first line of CSV directly for column count + IF vCsvColCount = 0 THEN + BEGIN + -- This is a fallback - try to get CSV header from external source if possible + -- Note: This would require DBMS_CLOUD.GET_OBJECT or similar approach + -- For now, we'll rely on the validation log parsing + NULL; + EXCEPTION + WHEN OTHERS THEN + NULL; + END; + END IF; + + EXCEPTION + WHEN OTHERS THEN + vErrorDetails := 'Error analyzing validation log: ' || SQLERRM; + END; + + -- Detect column order mismatch and excess columns + IF vCsvOrder IS NOT NULL AND vExpectedOrder IS NOT NULL THEN + IF UPPER(REPLACE(vCsvOrder, ' ', '')) != UPPER(REPLACE(vExpectedOrder, ' ', '')) THEN + vColumnMismatch := 'YES'; + ELSE + vColumnMismatch := 'NO'; + END IF; + END IF; + + -- Check for excess columns + IF vCsvColCount > vTemplateColCount THEN + -- Try to identify which columns are excess + IF vCsvOrder IS NOT NULL THEN + -- Parse CSV columns and compare with template + DECLARE + vCsvCols SYS.ODCIVARCHAR2LIST; + vTemplateCols SYS.ODCIVARCHAR2LIST; + vExcessFound VARCHAR2(1) := 'N'; + i NUMBER; + BEGIN + -- Split CSV columns + SELECT TRIM(REGEXP_SUBSTR(vCsvOrder, '[^,]+', 1, LEVEL)) + BULK COLLECT INTO vCsvCols + FROM DUAL + CONNECT BY REGEXP_SUBSTR(vCsvOrder, '[^,]+', 1, LEVEL) IS NOT NULL; + + -- Split template columns + SELECT TRIM(REGEXP_SUBSTR(vExpectedOrder, '[^,]+', 1, LEVEL)) + BULK COLLECT INTO vTemplateCols + FROM DUAL + CONNECT BY REGEXP_SUBSTR(vExpectedOrder, '[^,]+', 1, LEVEL) IS NOT NULL; + + -- Find excess columns (those in CSV but not in template) + FOR i IN 1..vCsvCols.COUNT LOOP + DECLARE + vFoundInTemplate BOOLEAN := FALSE; + j NUMBER; + BEGIN + -- Check if CSV column exists in template + FOR j IN 1..vTemplateCols.COUNT LOOP + IF UPPER(TRIM(vCsvCols(i))) = UPPER(TRIM(vTemplateCols(j))) THEN + vFoundInTemplate := TRUE; + EXIT; + END IF; + END LOOP; + + -- If not found in template, it's an excess column + IF NOT vFoundInTemplate THEN + IF vExcessFound = 'Y' THEN + vExcessColumns := vExcessColumns || ', '; + END IF; + vExcessColumns := vExcessColumns || vCsvCols(i); + vExcessFound := 'Y'; + END IF; + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + vExcessColumns := 'Unable to determine specific excess columns'; + END; + END IF; + END IF; + + -- Build comprehensive analysis report + vAnalysisReport := 'FILE VALIDATION FAILED - DETAILED ANALYSIS' || cgBL || + '=================================================' || cgBL || cgBL; + + -- Column structure analysis + vAnalysisReport := vAnalysisReport || + 'COLUMN STRUCTURE ANALYSIS:' || cgBL || + '---------------------------------------------------' || cgBL || + 'Template Expected Order: ' || vExpectedOrder || cgBL || + 'Template Column Count: ' || vTemplateColCount || cgBL || + 'CSV Detected Order: ' || NVL(vCsvOrder, 'Unknown') || cgBL || + 'CSV Column Count: ' || vCsvColCount || cgBL || cgBL; + + -- Report column count issues + IF vCsvColCount > vTemplateColCount THEN + vAnalysisReport := vAnalysisReport || + 'EXCESS COLUMNS DETECTED!' || cgBL || + 'CSV file has ' || (vCsvColCount - vTemplateColCount) || ' more columns than template allows.' || cgBL; + IF vExcessColumns IS NOT NULL THEN + vAnalysisReport := vAnalysisReport || + 'Excess columns found: ' || vExcessColumns || cgBL; + END IF; + vAnalysisReport := vAnalysisReport || cgBL; + END IF; + + -- Report column order issues + IF vColumnMismatch = 'YES' THEN + vAnalysisReport := vAnalysisReport || + 'COLUMN ORDER MISMATCH DETECTED!' || cgBL || + 'CSV columns are in different order than template expects.' || cgBL || cgBL; + END IF; + + -- Specific error analysis + IF vErrorCount > 0 THEN + vAnalysisReport := vAnalysisReport || + 'SPECIFIC ERRORS FOUND:' || cgBL || + '---------------------------------------------------' || cgBL; + + IF vErrorColumn IS NOT NULL THEN + -- Get expected data type for error column + FOR rec IN c_template_columns LOOP + IF rec.COLUMN_NAME = vErrorColumn THEN + vExpectedType := rec.DATA_TYPE; + EXIT; + END IF; + END LOOP; + + vAnalysisReport := vAnalysisReport || + '1. Column ' || vErrorColumn || ': Expected ' || vExpectedType || + ', received "' || NVL(vErrorValue, 'unknown value') || '" (TEXT)' || cgBL || + ' → CSV position contains different data type than expected' || cgBL; + END IF; + + vAnalysisReport := vAnalysisReport || + 'Total validation errors found: ' || vErrorCount || cgBL || cgBL; + END IF; + + -- Solutions section + vAnalysisReport := vAnalysisReport || + 'SUGGESTED SOLUTIONS:' || cgBL || + '---------------------------------------------------' || cgBL; + + -- Solutions for excess columns + IF vCsvColCount > vTemplateColCount THEN + vAnalysisReport := vAnalysisReport || + 'FOR EXCESS COLUMNS:' || cgBL || + '• Remove extra columns from CSV file' || cgBL || + '• Keep only these columns in this order: ' || vExpectedOrder || cgBL; + IF vExcessColumns IS NOT NULL THEN + vAnalysisReport := vAnalysisReport || + '• Specifically remove: ' || vExcessColumns || cgBL; + END IF; + vAnalysisReport := vAnalysisReport || cgBL; + END IF; + + -- Solutions for column order + IF vColumnMismatch = 'YES' THEN + vAnalysisReport := vAnalysisReport || + 'FOR COLUMN ORDER:' || cgBL || + '• Reorder CSV columns to match template: ' || vExpectedOrder || cgBL || + '• Or update template table column order to match CSV file' || cgBL || cgBL; + END IF; + + -- General solutions + vAnalysisReport := vAnalysisReport || + 'GENERAL RECOMMENDATIONS:' || cgBL || + '• Ensure CSV has exactly ' || vTemplateColCount || ' columns' || cgBL || + '• Verify column names match template table exactly' || cgBL || + '• Check data types in each column match expectations' || cgBL || cgBL; + + -- Validation log reference + vAnalysisReport := vAnalysisReport || + 'TECHNICAL DETAILS:' || cgBL || + '---------------------------------------------------' || cgBL || + 'Validation Log Table: ' || pValidationLogTable || cgBL || + 'Template Table: ' || pTemplateSchema || '.' || pTemplateTable || cgBL || + 'CSV File: ' || pCsvFileUri || cgBL || + 'Query validation details: SELECT * FROM ' || pValidationLogTable || ';' || cgBL; + + RETURN vAnalysisReport; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error generating validation analysis: ' || SQLERRM || cgBL || + 'Validation Log Table: ' || pValidationLogTable || cgBL || + 'Check table manually: SELECT * FROM ' || pValidationLogTable || ';'; + END ANALYZE_VALIDATION_ERRORS; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN GET_PACKAGE_VERSION_INFO( + pPackageName => 'ENV_MANAGER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN FORMAT_VERSION_HISTORY( + pPackageName => 'ENV_MANAGER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_PACKAGE_VERSION_INFO( + pPackageName VARCHAR2, + pVersion VARCHAR2, + pBuildDate VARCHAR2, + pAuthor VARCHAR2 + ) RETURN VARCHAR2 + IS + BEGIN + RETURN 'Package: ' || pPackageName || cgBL || + 'Version: ' || pVersion || cgBL || + 'Build Date: ' || pBuildDate || cgBL || + 'Author: ' || pAuthor; + END GET_PACKAGE_VERSION_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FORMAT_VERSION_HISTORY( + pPackageName VARCHAR2, + pVersionHistory VARCHAR2 + ) RETURN VARCHAR2 + IS + BEGIN + RETURN pPackageName || ' Version History:' || cgBL || pVersionHistory; + END FORMAT_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE HASH + CHANGE DETECTION FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION CALCULATE_PACKAGE_HASH( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageType VARCHAR2 + ) RETURN VARCHAR2 + IS + vSourceCode CLOB; + vHash VARCHAR2(64); + vRawHash RAW(32); + BEGIN + -- Build complete source code from ALL_SOURCE using XMLAGG (no 4000 char limit) + -- CRITICAL: Cannot use LISTAGG due to VARCHAR2 limit + SELECT XMLAGG(XMLELEMENT(E, TEXT) ORDER BY LINE).GETCLOBVAL() + INTO vSourceCode + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = UPPER(pPackageType); + + -- If empty, return NULL + IF vSourceCode IS NULL OR DBMS_LOB.GETLENGTH(vSourceCode) = 0 THEN + RETURN NULL; + END IF; + + -- Calculate SHA256 hash directly from CLOB + -- DBMS_CRYPTO.HASH has overload for CLOB in Oracle 19c+ + vRawHash := DBMS_CRYPTO.HASH( + src => vSourceCode, + typ => DBMS_CRYPTO.HASH_SH256 + ); + + -- Convert to hex string + vHash := LOWER(RAWTOHEX(vRawHash)); + + RETURN vHash; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + WHEN OTHERS THEN + LOG_PROCESS_ERROR('Error calculating package hash: ' || SQLERRM, + 'pPackageOwner=' || pPackageOwner || ', pPackageName=' || pPackageName); + RETURN NULL; + END CALCULATE_PACKAGE_HASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE TRACK_PACKAGE_VERSION( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageVersion VARCHAR2, + pPackageBuildDate VARCHAR2, + pPackageAuthor VARCHAR2 + ) + IS + vHashSpec VARCHAR2(64); + vHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLineCountSpec NUMBER; + vLineCountBody NUMBER; + vChangeDetected CHAR(1) := 'N'; + vChangeMessage VARCHAR2(4000); + vParameters VARCHAR2(4000); + BEGIN + vParameters := FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pPackageOwner => ''' || pPackageOwner || '''', + 'pPackageName => ''' || pPackageName || '''', + 'pPackageVersion => ''' || pPackageVersion || '''' + )); + + LOG_PROCESS_EVENT('Start TRACK_PACKAGE_VERSION', 'INFO', vParameters); + + -- Calculate current hashes + vHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get line counts + BEGIN + SELECT COUNT(*) + INTO vLineCountSpec + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = 'PACKAGE'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vLineCountSpec := 0; + END; + + BEGIN + SELECT COUNT(*) + INTO vLineCountBody + FROM ALL_SOURCE + WHERE OWNER = UPPER(pPackageOwner) + AND NAME = UPPER(pPackageName) + AND TYPE = 'PACKAGE BODY'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vLineCountBody := 0; + END; + + -- Get last tracked version and hashes + BEGIN + SELECT PACKAGE_VERSION, SOURCE_CODE_HASH_SPEC, SOURCE_CODE_HASH_BODY + INTO vLastVersion, vLastHashSpec, vLastHashBody + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + + -- Check if hash changed but version didn't + IF (vHashSpec != vLastHashSpec OR NVL(vHashBody,'X') != NVL(vLastHashBody,'X')) + AND pPackageVersion = vLastVersion THEN + + vChangeDetected := 'Y'; + vChangeMessage := 'WARNING: Source code changed without version update!' || cgBL || + 'Last Version: ' || vLastVersion || cgBL || + 'Current Version: ' || pPackageVersion || cgBL; + + IF vHashSpec != vLastHashSpec THEN + vChangeMessage := vChangeMessage || + 'SPEC Changed - Hash: ' || SUBSTR(vHashSpec, 1, 16) || '... (was: ' || + SUBSTR(vLastHashSpec, 1, 16) || '...)' || cgBL; + END IF; + + IF NVL(vHashBody,'X') != NVL(vLastHashBody,'X') THEN + vChangeMessage := vChangeMessage || + 'BODY Changed - Hash: ' || SUBSTR(vHashBody, 1, 16) || '... (was: ' || + SUBSTR(NVL(vLastHashBody,'NULL'), 1, 16) || '...)' || cgBL; + END IF; + + vChangeMessage := vChangeMessage || + 'RECOMMENDATION: Update PACKAGE_VERSION constant and PACKAGE_BUILD_DATE'; + + LOG_PROCESS_EVENT(vChangeMessage, 'WARNING', vParameters); + END IF; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + -- First time tracking this package + vChangeDetected := 'N'; + vChangeMessage := 'First tracking record for this package'; + LOG_PROCESS_EVENT(vChangeMessage, 'INFO', vParameters); + END; + + -- Insert tracking record + INSERT INTO CT_MRDS.A_PACKAGE_VERSION_TRACKING ( + PACKAGE_OWNER, + PACKAGE_NAME, + PACKAGE_TYPE, + PACKAGE_VERSION, + PACKAGE_BUILD_DATE, + PACKAGE_AUTHOR, + SOURCE_CODE_HASH_SPEC, + SOURCE_CODE_HASH_BODY, + LINE_COUNT_SPEC, + LINE_COUNT_BODY, + DETECTED_CHANGE_WITHOUT_VERSION, + CHANGE_DETECTION_MESSAGE + ) VALUES ( + UPPER(pPackageOwner), + UPPER(pPackageName), + 'BOTH', + pPackageVersion, + pPackageBuildDate, + pPackageAuthor, + vHashSpec, + vHashBody, + vLineCountSpec, + vLineCountBody, + vChangeDetected, + vChangeMessage + ); + + COMMIT; + + LOG_PROCESS_EVENT('End TRACK_PACKAGE_VERSION - Record inserted', 'INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + LOG_PROCESS_ERROR('Error in TRACK_PACKAGE_VERSION: ' || SQLERRM, vParameters); + RAISE; + END TRACK_PACKAGE_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION CHECK_PACKAGE_CHANGES( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2 + IS + vCurrentHashSpec VARCHAR2(64); + vCurrentHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLastTrackingDate TIMESTAMP; + vChangeReport VARCHAR2(4000); + vSpecChanged BOOLEAN := FALSE; + vBodyChanged BOOLEAN := FALSE; + BEGIN + -- Get current hashes + vCurrentHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vCurrentHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get last tracked hashes + BEGIN + SELECT PACKAGE_VERSION, SOURCE_CODE_HASH_SPEC, SOURCE_CODE_HASH_BODY, TRACKING_DATE + INTO vLastVersion, vLastHashSpec, vLastHashBody, vLastTrackingDate + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN 'Package ' || pPackageOwner || '.' || pPackageName || ' has never been tracked.' || cgBL || + 'Run TRACK_PACKAGE_VERSION to establish baseline.'; + END; + + -- Check for changes + IF vCurrentHashSpec != vLastHashSpec THEN + vSpecChanged := TRUE; + END IF; + + IF NVL(vCurrentHashBody, 'X') != NVL(vLastHashBody, 'X') THEN + vBodyChanged := TRUE; + END IF; + + -- Build report + IF vSpecChanged OR vBodyChanged THEN + vChangeReport := 'WARNING: Package ' || pPackageOwner || '.' || pPackageName || ' has changed!' || cgBL || + '========================================' || cgBL || + 'Last Tracked Version: ' || vLastVersion || cgBL || + 'Last Tracked Date: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + cgBL; + + IF vSpecChanged THEN + vChangeReport := vChangeReport || + 'SPECIFICATION Changed:' || cgBL || + ' Current Hash: ' || SUBSTR(vCurrentHashSpec, 1, 16) || '...' || cgBL || + ' Last Hash: ' || SUBSTR(vLastHashSpec, 1, 16) || '...' || cgBL || + cgBL; + END IF; + + IF vBodyChanged THEN + vChangeReport := vChangeReport || + 'BODY Changed:' || cgBL || + ' Current Hash: ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 16) || '...' || cgBL || + ' Last Hash: ' || SUBSTR(NVL(vLastHashBody, 'NULL'), 1, 16) || '...' || cgBL || + cgBL; + END IF; + + vChangeReport := vChangeReport || + 'RECOMMENDATION:' || cgBL || + '1. Update PACKAGE_VERSION constant' || cgBL || + '2. Update PACKAGE_BUILD_DATE constant' || cgBL || + '3. Add entry to VERSION_HISTORY' || cgBL || + '4. Call TRACK_PACKAGE_VERSION to update tracking'; + ELSE + vChangeReport := 'OK: Package ' || pPackageOwner || '.' || pPackageName || ' has not changed.' || cgBL || + 'Last Tracked: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + 'Version: ' || vLastVersion; + END IF; + + RETURN vChangeReport; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error checking package changes: ' || SQLERRM; + END CHECK_PACKAGE_CHANGES; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_PACKAGE_HASH_INFO( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2 + IS + vCurrentHashSpec VARCHAR2(64); + vCurrentHashBody VARCHAR2(64); + vLastHashSpec VARCHAR2(64); + vLastHashBody VARCHAR2(64); + vLastVersion VARCHAR2(10); + vLastTrackingDate TIMESTAMP; + vLastChangeDetected CHAR(1); + vInfo VARCHAR2(4000); + BEGIN + -- Get current hashes + vCurrentHashSpec := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE'); + vCurrentHashBody := CALCULATE_PACKAGE_HASH(pPackageOwner, pPackageName, 'PACKAGE BODY'); + + -- Get last tracking info + BEGIN + SELECT PACKAGE_VERSION, + SOURCE_CODE_HASH_SPEC, + SOURCE_CODE_HASH_BODY, + TRACKING_DATE, + DETECTED_CHANGE_WITHOUT_VERSION + INTO vLastVersion, vLastHashSpec, vLastHashBody, vLastTrackingDate, vLastChangeDetected + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING + WHERE PACKAGE_OWNER = UPPER(pPackageOwner) + AND PACKAGE_NAME = UPPER(pPackageName) + ORDER BY TRACKING_DATE DESC + FETCH FIRST 1 ROW ONLY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN 'Package: ' || pPackageOwner || '.' || pPackageName || cgBL || + 'Status: Never tracked' || cgBL || + 'Current Hash (SPEC): ' || SUBSTR(vCurrentHashSpec, 1, 16) || '...' || cgBL || + 'Current Hash (BODY): ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 16) || '...'; + END; + + -- Build info report + vInfo := 'Package: ' || pPackageOwner || '.' || pPackageName || cgBL || + 'Current Version: ' || vLastVersion || cgBL || + 'Last Tracked: ' || TO_CHAR(vLastTrackingDate, 'YYYY-MM-DD HH24:MI:SS') || cgBL || + cgBL || + 'Current Hash (SPEC): ' || SUBSTR(vCurrentHashSpec, 1, 32) || '...' || cgBL || + 'Last Hash (SPEC): ' || SUBSTR(vLastHashSpec, 1, 32) || '...' || cgBL; + + IF vCurrentHashBody IS NOT NULL OR vLastHashBody IS NOT NULL THEN + vInfo := vInfo || + 'Current Hash (BODY): ' || SUBSTR(NVL(vCurrentHashBody, 'NULL'), 1, 32) || '...' || cgBL || + 'Last Hash (BODY): ' || SUBSTR(NVL(vLastHashBody, 'NULL'), 1, 32) || '...' || cgBL; + END IF; + + vInfo := vInfo || cgBL; + + -- Status + IF vCurrentHashSpec = vLastHashSpec AND NVL(vCurrentHashBody, 'X') = NVL(vLastHashBody, 'X') THEN + vInfo := vInfo || 'Status: OK - No changes detected'; + ELSE + vInfo := vInfo || 'Status: CHANGED - Source code modified since last tracking'; + END IF; + + IF vLastChangeDetected = 'Y' THEN + vInfo := vInfo || cgBL || 'Last Tracking Warning: Change detected without version update'; + END IF; + + RETURN vInfo; + + EXCEPTION + WHEN OTHERS THEN + RETURN 'Error getting package hash info: ' || SQLERRM; + END GET_PACKAGE_HASH_INFO; + + ---------------------------------------------------------------------------------------------------- + +BEGIN + INIT_ERRORS; + guid := sys_guid(); + gvUsername := SYS_CONTEXT('USERENV', 'SESSION_USER'); + gvOsuser := SYS_CONTEXT('USERENV', 'OS_USER'); + gvMachine := SYS_CONTEXT('USERENV', 'HOST'); + gvModule := SYS_CONTEXT('USERENV', 'MODULE'); + + -- Get info about EnvironmentID. Without it package cannot proceed further. + -- Information about environment is needed to get proper configuration values + -- It can be set up in two different ways : + -- 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => 'dev'); + -- 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values ('default','environment_id','dev'); + -- Session level setup (1.) takes precedence over configuration level one (2.) + + gvEnv := nvl(SYS_CONTEXT ('USERENV', 'CLIENT_IDENTIFIER'), GET_DEFAULT_ENV()); + if gvEnv is null then + dbms_output.put_line(MSG_ENVIRONMENT_NOT_SET); + LOG_PROCESS_EVENT(MSG_ENVIRONMENT_NOT_SET, 'ERROR'); + RAISE_APPLICATION_ERROR(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET); + else + dbms_output.put_line('EnvironmentID set to: '||gvEnv); + end if; + + INIT_VARIABLES(pEnv => gvEnv); +END ENV_MANAGER; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkg b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkg new file mode 100644 index 0000000..fded944 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/ENV_MANAGER.pkg @@ -0,0 +1,625 @@ +create or replace PACKAGE CT_MRDS.ENV_MANAGER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select ENV_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-12-20 10:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.2.0 (2025-12-20): Added error codes for parallel execution support (CODE_INVALID_PARALLEL_DEGREE -20110, CODE_PARALLEL_EXECUTION_FAILED -20111)' || CHR(13)||CHR(10) || + '3.1.0 (2025-10-22): Added package hash tracking and automatic change detection system (SHA256 hashing)' || CHR(13)||CHR(10) || + '3.0.0 (2025-10-22): Added package versioning system with centralized version management functions' || CHR(13)||CHR(10) || + '2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function for comprehensive CSV validation analysis' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-01): Added LOG_PROCESS_ERROR procedure with enhanced error diagnostics and stack traces' || CHR(13)||CHR(10) || + '1.5.0 (2025-09-20): Added console logging support with gvConsoleLoggingEnabled configuration' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-01): Initial release with error management and configuration system'; + + TYPE Error_Record IS RECORD ( + code PLS_INTEGER, + message VARCHAR2(4000) + ); + + TYPE tErrorList IS TABLE OF Error_Record INDEX BY PLS_INTEGER; + + Errors tErrorList; + + + guid VARCHAR2(32); + gvEnv VARCHAR2(200); + gvUsername VARCHAR2(128); + gvOsuser VARCHAR2(128); + gvMachine VARCHAR2(64); + gvModule VARCHAR2(64); + + gvNameSpace VARCHAR2(200); + gvRegion VARCHAR2(200); + gvDataBucketName VARCHAR2(200); + gvInboxBucketName VARCHAR2(200); + gvArchiveBucketName VARCHAR2(200); + gvDataBucketUri VARCHAR2(200); + gvInboxBucketUri VARCHAR2(200); + gvArchiveBucketUri VARCHAR2(200); + gvCredentialName VARCHAR2(200); + + -- Overwritten by variable "LoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF' + + -- Overwritten by variable "MinLogLevel" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + -- Possible values: DEBUG ,INFO ,WARNING ,ERROR + gvMinLogLevel VARCHAR2(10) := 'DEBUG'; + + -- Overwritten by variable "DefaultDateFormat" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvDefaultDateFormat VARCHAR2(200) := 'DD/MM/YYYY HH24:MI:SS'; + + -- Overwritten by variable "ConsoleLoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table + gvConsoleLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF' + + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + + vgSourceFileConfigKey PLS_INTEGER; + + vgMsgTmp VARCHAR2(32000); + --Exceptions + ERR_EMPTY_FILEURI_AND_RECKEY EXCEPTION; + CODE_EMPTY_FILEURI_AND_RECKEY CONSTANT PLS_INTEGER := -20001; + MSG_EMPTY_FILEURI_AND_RECKEY VARCHAR2(4000) := 'Either pFileUri or pSourceFileReceivedKey must be not null'; + PRAGMA EXCEPTION_INIT( ERR_EMPTY_FILEURI_AND_RECKEY + ,CODE_EMPTY_FILEURI_AND_RECKEY); + + + ERR_NO_CONFIG_MATCH_FOR_FILEURI EXCEPTION; + CODE_NO_CONFIG_MATCH_FOR_FILEURI CONSTANT PLS_INTEGER := -20002; + MSG_NO_CONFIG_MATCH_FOR_FILEURI VARCHAR2(4000) := 'No match for source file in A_SOURCE_FILE_CONFIG table' + ||cgBL||' The file provided in parameter: pFileUri does not have ' + ||cgBL||' coresponding configuration in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH_FOR_FILEURI + ,CODE_NO_CONFIG_MATCH_FOR_FILEURI); + + ERR_MULTIPLE_MATCH_FOR_SRCFILE EXCEPTION; + CODE_MULTIPLE_MATCH_FOR_SRCFILE CONSTANT PLS_INTEGER := -20003; + MSG_MULTIPLE_MATCH_FOR_SRCFILE VARCHAR2(4000) := 'Multiple match for source file in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_MATCH_FOR_SRCFILE + ,CODE_MULTIPLE_MATCH_FOR_SRCFILE); + + ERR_MISSING_COLUMN_DATE_FORMAT EXCEPTION; + CODE_MISSING_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20004; + MSG_MISSING_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Missing entry in config table: A_COLUMN_DATE_FORMAT primary key(TEMPLATE_TABLE_NAME, COLUMN_NAME)' + ||cgBL||' Remember: each column which data_type IN (''DATE'', ''TIMESTAMP'')' + ||cgBL||' should have DateFormat specified in A_COLUMN_DATE_FORMAT table ' + ||cgBL||' for example: ''YYYY-MM-DD'''; + PRAGMA EXCEPTION_INIT( ERR_MISSING_COLUMN_DATE_FORMAT + ,CODE_MISSING_COLUMN_DATE_FORMAT); + + ERR_MULTIPLE_COLUMN_DATE_FORMAT EXCEPTION; + CODE_MULTIPLE_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20005; + MSG_MULTIPLE_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Multiple records for date format in A_COLUMN_DATE_FORMAT table' + ||cgBL||' There should be only one format specified for each DAT/TIMESTAMP column'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_COLUMN_DATE_FORMAT + ,CODE_MULTIPLE_COLUMN_DATE_FORMAT); + + + ERR_DIDNT_GET_LOAD_OPERATION_ID EXCEPTION; + CODE_DIDNT_GET_LOAD_OPERATION_ID CONSTANT PLS_INTEGER := -20006; + MSG_DIDNT_GET_LOAD_OPERATION_ID VARCHAR2(4000) := 'Didnt get load operation id from external table validation'; + PRAGMA EXCEPTION_INIT( ERR_DIDNT_GET_LOAD_OPERATION_ID + ,CODE_DIDNT_GET_LOAD_OPERATION_ID); + + ERR_NO_CONFIG_FOR_RECEIVED_FILE EXCEPTION; + CODE_NO_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20007; + MSG_NO_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'No match for received source file in A_SOURCE_FILE_CONFIG ' + ||cgBL||' or missing data in A_SOURCE_FILE_RECEIVED table for provided pSourceFileReceivedKey parameter'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_FOR_RECEIVED_FILE + ,CODE_NO_CONFIG_FOR_RECEIVED_FILE); + + ERR_MULTI_CONFIG_FOR_RECEIVED_FILE EXCEPTION; + CODE_MULTI_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20008; + MSG_MULTI_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'Multiple matchs for received source file in A_SOURCE_FILE_CONFIG'; + PRAGMA EXCEPTION_INIT( ERR_MULTI_CONFIG_FOR_RECEIVED_FILE + ,CODE_MULTI_CONFIG_FOR_RECEIVED_FILE); + + ERR_FILE_NOT_FOUND_ON_CLOUD EXCEPTION; + CODE_FILE_NOT_FOUND_ON_CLOUD CONSTANT PLS_INTEGER := -20009; + MSG_FILE_NOT_FOUND_ON_CLOUD VARCHAR2(4000) := 'File not found on the cloud'; + PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_FOUND_ON_CLOUD + ,CODE_FILE_NOT_FOUND_ON_CLOUD); + + ERR_FILE_VALIDATION_FAILED EXCEPTION; + CODE_FILE_VALIDATION_FAILED CONSTANT PLS_INTEGER := -20010; + MSG_FILE_VALIDATION_FAILED VARCHAR2(4000) := 'File validation failed'; + PRAGMA EXCEPTION_INIT( ERR_FILE_VALIDATION_FAILED + ,CODE_FILE_VALIDATION_FAILED); + + ERR_EXCESS_COLUMNS_DETECTED EXCEPTION; + CODE_EXCESS_COLUMNS_DETECTED CONSTANT PLS_INTEGER := -20011; + MSG_EXCESS_COLUMNS_DETECTED VARCHAR2(4000) := 'CSV file contains more columns than template allows'; + PRAGMA EXCEPTION_INIT( ERR_EXCESS_COLUMNS_DETECTED + ,CODE_EXCESS_COLUMNS_DETECTED); + + ERR_NO_CONFIG_MATCH EXCEPTION; + CODE_NO_CONFIG_MATCH CONSTANT PLS_INTEGER := -20012; + MSG_NO_CONFIG_MATCH VARCHAR2(4000) := 'No match for specified parameters in A_SOURCE_FILE_CONFIG table'; + PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH + ,CODE_NO_CONFIG_MATCH); + + ERR_UNKNOWN_PREFIX EXCEPTION; + CODE_UNKNOWN_PREFIX CONSTANT PLS_INTEGER := -20013; + MSG_UNKNOWN_PREFIX VARCHAR2(4000) := 'Unknown prefix'; + PRAGMA EXCEPTION_INIT( ERR_UNKNOWN_PREFIX + ,CODE_UNKNOWN_PREFIX); + + ERR_TABLE_NOT_EXISTS EXCEPTION; + CODE_TABLE_NOT_EXISTS CONSTANT PLS_INTEGER := -20014; + MSG_TABLE_NOT_EXISTS VARCHAR2(4000) := 'Table does not exist'; + PRAGMA EXCEPTION_INIT( ERR_TABLE_NOT_EXISTS + ,CODE_TABLE_NOT_EXISTS); + + ERR_COLUMN_NOT_EXISTS EXCEPTION; + CODE_COLUMN_NOT_EXISTS CONSTANT PLS_INTEGER := -20015; + MSG_COLUMN_NOT_EXISTS VARCHAR2(4000) := 'Column does not exist in table'; + PRAGMA EXCEPTION_INIT( ERR_COLUMN_NOT_EXISTS + ,CODE_COLUMN_NOT_EXISTS); + + ERR_UNSUPPORTED_DATA_TYPE EXCEPTION; + CODE_UNSUPPORTED_DATA_TYPE CONSTANT PLS_INTEGER := -20016; + MSG_UNSUPPORTED_DATA_TYPE VARCHAR2(4000) := 'Unsupported data type'; + PRAGMA EXCEPTION_INIT( ERR_UNSUPPORTED_DATA_TYPE + ,CODE_UNSUPPORTED_DATA_TYPE); + + ERR_MISSING_SOURCE_KEY EXCEPTION; + CODE_MISSING_SOURCE_KEY CONSTANT PLS_INTEGER := -20017; + MSG_MISSING_SOURCE_KEY VARCHAR2(4000) := 'The Source was not found in parent table A_SOURCE'; + PRAGMA EXCEPTION_INIT( ERR_MISSING_SOURCE_KEY + ,CODE_MISSING_SOURCE_KEY); + + ERR_NULL_SOURCE_FILE_CONFIG_KEY EXCEPTION; + CODE_NULL_SOURCE_FILE_CONFIG_KEY CONSTANT PLS_INTEGER := -20018; + MSG_NULL_SOURCE_FILE_CONFIG_KEY VARCHAR2(4000) := 'No entry in A_SOURCE_FILE_CONFIG table for specified A_SOURCE_FILE_CONFIG_KEY'; + PRAGMA EXCEPTION_INIT( ERR_NULL_SOURCE_FILE_CONFIG_KEY + ,CODE_NULL_SOURCE_FILE_CONFIG_KEY); + + ERR_DUPLICATED_SOURCE_KEY EXCEPTION; + CODE_DUPLICATED_SOURCE_KEY CONSTANT PLS_INTEGER := -20019; + MSG_DUPLICATED_SOURCE_KEY VARCHAR2(4000) := 'The Source already exists in the A_SOURCE table'; + PRAGMA EXCEPTION_INIT( ERR_DUPLICATED_SOURCE_KEY + ,CODE_DUPLICATED_SOURCE_KEY); + + ERR_MISSING_CONTAINER_CONFIG EXCEPTION; + CODE_MISSING_CONTAINER_CONFIG CONSTANT PLS_INTEGER := -20020; + MSG_MISSING_CONTAINER_CONFIG VARCHAR2(4000) := 'No match in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID'; + PRAGMA EXCEPTION_INIT( ERR_MISSING_CONTAINER_CONFIG + ,CODE_MISSING_CONTAINER_CONFIG); + + ERR_MULTIPLE_CONTAINER_ENTRIES EXCEPTION; + CODE_MULTIPLE_CONTAINER_ENTRIES CONSTANT PLS_INTEGER := -20021; + MSG_MULTIPLE_CONTAINER_ENTRIES VARCHAR2(4000) := 'Multiple matches in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID'; + PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_CONTAINER_ENTRIES + ,CODE_MULTIPLE_CONTAINER_ENTRIES); + + ERR_WRONG_DESTINATION_PARAM EXCEPTION; + CODE_WRONG_DESTINATION_PARAM CONSTANT PLS_INTEGER := -20022; + MSG_WRONG_DESTINATION_PARAM VARCHAR2(4000) := 'Wrong destination parameter provided.'; + PRAGMA EXCEPTION_INIT( ERR_WRONG_DESTINATION_PARAM + ,CODE_WRONG_DESTINATION_PARAM); + + ERR_FILE_NOT_EXISTS_ON_CLOUD EXCEPTION; + CODE_FILE_NOT_EXISTS_ON_CLOUD CONSTANT PLS_INTEGER := -20023; + MSG_FILE_NOT_EXISTS_ON_CLOUD VARCHAR2(4000) := 'File not exists on cloud.'; + PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_EXISTS_ON_CLOUD + ,CODE_FILE_NOT_EXISTS_ON_CLOUD); + + ERR_FILE_ALREADY_REGISTERED EXCEPTION; + CODE_FILE_ALREADY_REGISTERED CONSTANT PLS_INTEGER := -20024; + MSG_FILE_ALREADY_REGISTERED VARCHAR2(4000) := 'File already registered in A_SOURCE_FILE_RECEIVED table.'; + PRAGMA EXCEPTION_INIT( ERR_FILE_ALREADY_REGISTERED + ,CODE_FILE_ALREADY_REGISTERED); + + ERR_WRONG_DATE_TIMESTAMP_FORMAT EXCEPTION; + CODE_WRONG_DATE_TIMESTAMP_FORMAT CONSTANT PLS_INTEGER := -20025; + MSG_WRONG_DATE_TIMESTAMP_FORMAT VARCHAR2(4000) := 'Provided DATE or TIMESTAMP format has errors (possible duplicated codes, ex: ''DD'').'; + PRAGMA EXCEPTION_INIT( ERR_WRONG_DATE_TIMESTAMP_FORMAT + ,CODE_WRONG_DATE_TIMESTAMP_FORMAT); + + ERR_ENVIRONMENT_NOT_SET EXCEPTION; + CODE_ENVIRONMENT_NOT_SET CONSTANT PLS_INTEGER := -20026; + MSG_ENVIRONMENT_NOT_SET VARCHAR2(4000) := 'EnvironmentID not set' + ||cgBL||' Information about environment is needed to get proper configuration values.' + ||cgBL||' It can be set up in two different ways:' + ||cgBL||' 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => ''dev'')' + ||cgBL||' 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values (''default'',''environment_id'',''dev'')' + ||cgBL||' Session level setup (1.) takes precedence over configuration level one (2.)' + ; + PRAGMA EXCEPTION_INIT( ERR_ENVIRONMENT_NOT_SET + ,CODE_ENVIRONMENT_NOT_SET); + + + ERR_CONFIG_VARIABLE_NOT_SET EXCEPTION; + CODE_CONFIG_VARIABLE_NOT_SET CONSTANT PLS_INTEGER := -20027; + MSG_CONFIG_VARIABLE_NOT_SET VARCHAR2(4000) := 'Missing configuration value in A_FILE_MANAGER_CONFIG'; + PRAGMA EXCEPTION_INIT( ERR_CONFIG_VARIABLE_NOT_SET + ,CODE_CONFIG_VARIABLE_NOT_SET); + + ERR_NOT_INPUT_SOURCE_FILE_TYPE EXCEPTION; + CODE_NOT_INPUT_SOURCE_FILE_TYPE CONSTANT PLS_INTEGER := -20028; + MSG_NOT_INPUT_SOURCE_FILE_TYPE VARCHAR2(4000) := 'Archival can be executed only for A_SOURCE_FILE_CONFIG_KEY where SOURCE_FILE_TYPE=''INPUT'''; + PRAGMA EXCEPTION_INIT( ERR_NOT_INPUT_SOURCE_FILE_TYPE + ,CODE_NOT_INPUT_SOURCE_FILE_TYPE); + + ERR_EXP_DATA_FOR_ARCH_FAILED EXCEPTION; + CODE_EXP_DATA_FOR_ARCH_FAILED CONSTANT PLS_INTEGER := -20029; + MSG_EXP_DATA_FOR_ARCH_FAILED VARCHAR2(4000) := 'Export data for archival failed.'; + PRAGMA EXCEPTION_INIT( ERR_EXP_DATA_FOR_ARCH_FAILED + ,CODE_EXP_DATA_FOR_ARCH_FAILED); + + ERR_RESTORE_FILE_FROM_TRASH EXCEPTION; + CODE_RESTORE_FILE_FROM_TRASH CONSTANT PLS_INTEGER := -20030; + MSG_RESTORE_FILE_FROM_TRASH VARCHAR2(4000) := 'Unexpected issues occured while archival process. Restoration of exported files failed.'; + PRAGMA EXCEPTION_INIT( ERR_RESTORE_FILE_FROM_TRASH + ,CODE_RESTORE_FILE_FROM_TRASH); + + ERR_CHANGE_STAT_TO_ARCHIVED_FAILED EXCEPTION; + CODE_CHANGE_STAT_TO_ARCHIVED_FAILED CONSTANT PLS_INTEGER := -20031; + MSG_CHANGE_STAT_TO_ARCHIVED_FAILED VARCHAR2(4000) := 'Failed to change file status to: ARCHIVED in A_SOURCE_FILE_RECEIVED table.'; + PRAGMA EXCEPTION_INIT( ERR_CHANGE_STAT_TO_ARCHIVED_FAILED + ,CODE_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ERR_MOVE_FILE_TO_TRASH_FAILED EXCEPTION; + CODE_MOVE_FILE_TO_TRASH_FAILED CONSTANT PLS_INTEGER := -20032; + MSG_MOVE_FILE_TO_TRASH_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.'; + PRAGMA EXCEPTION_INIT( ERR_MOVE_FILE_TO_TRASH_FAILED + ,CODE_MOVE_FILE_TO_TRASH_FAILED); + + ERR_DROP_EXPORTED_FILES_FAILED EXCEPTION; + CODE_DROP_EXPORTED_FILES_FAILED CONSTANT PLS_INTEGER := -20033; + MSG_DROP_EXPORTED_FILES_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.'; + PRAGMA EXCEPTION_INIT( ERR_DROP_EXPORTED_FILES_FAILED + ,CODE_DROP_EXPORTED_FILES_FAILED); + + ERR_INVALID_BUCKET_AREA EXCEPTION; + CODE_INVALID_BUCKET_AREA CONSTANT PLS_INTEGER := -20034; + MSG_INVALID_BUCKET_AREA VARCHAR2(4000) := 'Invalid bucket area specified. Valid values: INBOX, ODS, DATA, ARCHIVE'; + PRAGMA EXCEPTION_INIT( ERR_INVALID_BUCKET_AREA + ,CODE_INVALID_BUCKET_AREA); + + ERR_INVALID_PARALLEL_DEGREE EXCEPTION; + CODE_INVALID_PARALLEL_DEGREE CONSTANT PLS_INTEGER := -20110; + MSG_INVALID_PARALLEL_DEGREE VARCHAR2(4000) := 'Invalid parallel degree parameter. Must be between 1 and 16'; + PRAGMA EXCEPTION_INIT( ERR_INVALID_PARALLEL_DEGREE + ,CODE_INVALID_PARALLEL_DEGREE); + + ERR_PARALLEL_EXECUTION_FAILED EXCEPTION; + CODE_PARALLEL_EXECUTION_FAILED CONSTANT PLS_INTEGER := -20111; + MSG_PARALLEL_EXECUTION_FAILED VARCHAR2(4000) := 'Parallel execution failed'; + PRAGMA EXCEPTION_INIT( ERR_PARALLEL_EXECUTION_FAILED + ,CODE_PARALLEL_EXECUTION_FAILED); + + ERR_UNKNOWN EXCEPTION; + CODE_UNKNOWN CONSTANT PLS_INTEGER := -20999; + MSG_UNKNOWN VARCHAR2(4000) := 'Unknown Error Occured'; + PRAGMA EXCEPTION_INIT( ERR_UNKNOWN + ,CODE_UNKNOWN); + + --------------------------------------------------------------------------------------------------------------------------- + --------------------------------------------------------------------------------------------------------------------------- + + + + + /** + * @name LOG_PROCESS_EVENT + * @desc Insert a new log record into A_PROCESS_LOG table. + * Also outputs to console if gvConsoleLoggingEnabled = 'ON'. + * Respects logging level configuration (gvMinLogLevel). + * @example ENV_MANAGER.LOG_PROCESS_EVENT('Process completed successfully', 'INFO', 'pParam1=value1'); + * @ex_rslt Record inserted into A_PROCESS_LOG table and optionally displayed in console output + **/ + PROCEDURE LOG_PROCESS_EVENT ( + pLogMessage VARCHAR2 + ,pLogLevel VARCHAR2 DEFAULT 'ERROR' + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ); + + /** + * @name LOG_PROCESS_ERROR + * @desc Insert a detailed error record into A_PROCESS_LOG table with full stack trace, backtrace, and call stack. + * This procedure captures comprehensive error information for debugging purposes while + * allowing clean user-facing error messages to be raised separately. + * @param pLogMessage - Base error message description + * @param pParameters - Procedure parameters for context + * @param pProcessName - Name of the calling process/package + * @ex_rslt Record inserted into A_PROCESS_LOG table with complete error stack information + */ + PROCEDURE LOG_PROCESS_ERROR ( + pLogMessage VARCHAR2 + ,pParameters VARCHAR2 DEFAULT NULL + ,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER' + ); + + /** + * @name INIT_ERRORS + * @desc Loads data into Errors array. + * Errors array is a list of Record(Error_Code, Error_Message) index by Error_Code. + * Called automatically during package initialization. + * @example Called automatically when package is first referenced + * @ex_rslt Errors array populated with all error codes and messages + **/ + PROCEDURE INIT_ERRORS; + + + + /** + * @name GET_DEFAULT_ENV + * @desc It returns string with name of default environment. + * Return string is A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID value. + * @example select ENV_MANAGER.GET_DEFAULT_ENV() from dual; + * @ex_rslt dev + **/ + FUNCTION GET_DEFAULT_ENV + RETURN VARCHAR2; + + + + /** + * @name INIT_VARIABLES + * @desc For specified pEnv parameter (A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID) + * Assign values to following global package variables: + * - gvNameSpace + * - gvRegion + * - gvCredentialName + * - gvInboxBucketName + * - gvDataBucketName + * - gvArchiveBucketName + * - gvInboxBucketUri + * - gvDataBucketUri + * - gvArchiveBucketUri + * - gvLoggingEnabled + * - gvMinLogLevel + * - gvDefaultDateFormat + * - gvConsoleLoggingEnabled + **/ + PROCEDURE INIT_VARIABLES( + pEnv VARCHAR2 + ); + + + + /** + * @name GET_ERROR_MESSAGE + * @desc It returns string with error message for specified pCode (Error_Code). + * Error message is take from Errors Array loaded by INIT_ERRORS procedure + * @example select ENV_MANAGER.GET_ERROR_MESSAGE(pCode => -20009) from dual; + * @ex_rslt File not found on the cloud + **/ + FUNCTION GET_ERROR_MESSAGE( + pCode PLS_INTEGER + ) RETURN VARCHAR2; + + + + /** + * @name GET_ERROR_STACK + * @desc It returns string with all possible error stack info. + * Error message is take from Errors Array loaded by INIT_ERRORS procedure + * @example + * select ENV_MANAGER.GET_ERROR_STACK( + * pFormat => 'OUTPUT' + * ,pCode => -20009 + * ,pSourceFileReceivedKey => NULL) + * from dual + * @ex_rslt + * ------------------------------------------------------+ + * Error Message: + * ORA-0000: normal, successful completion + * ------------------------------------------------------- + * Error Stack: + * ------------------------------------------------------- + * Error Backtrace: + * ------------------------------------------------------+ + **/ + FUNCTION GET_ERROR_STACK( + pFormat VARCHAR2 + ,pCode PLS_INTEGER + ,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL + ) RETURN VARCHAR2; + + /** + * @name FORMAT_PARAMETERS + * @desc Formats parameter list for logging purposes. + * Converts SYS.ODCIVARCHAR2LIST to formatted string with proper NULL handling. + * @example select ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('param1=value1', 'param2=NULL')) from dual; + * @ex_rslt param1=value1 , + * param2=NULL + **/ + FUNCTION FORMAT_PARAMETERS( + pParameterList SYS.ODCIVARCHAR2LIST + ) RETURN VARCHAR2; + + /** + * @name ANALYZE_VALIDATION_ERRORS + * @desc Analyzes CSV validation errors and generates detailed diagnostic report. + * Compares CSV structure with template table and provides specific error analysis. + * Includes suggested solutions for common validation issues. + * @param pValidationLogTable - Name of validation log table (e.g., VALIDATE$242_LOG) + * @param pTemplateSchema - Schema of template table (e.g., CT_ET_TEMPLATES) + * @param pTemplateTable - Name of template table (e.g., MOCK_PROC_TABLE) + * @param pCsvFileUri - URI of CSV file being validated + * @example SELECT ENV_MANAGER.ANALYZE_VALIDATION_ERRORS('VALIDATE$242_LOG', 'CT_ET_TEMPLATES', 'MOCK_PROC_TABLE', 'https://...') FROM DUAL; + * @ex_rslt Detailed validation analysis report with column mismatches and solutions + **/ + FUNCTION ANALYZE_VALIDATION_ERRORS( + pValidationLogTable VARCHAR2, + pTemplateSchema VARCHAR2, + pTemplateTable VARCHAR2, + pCsvFileUri VARCHAR2 + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the ENV_MANAGER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT ENV_MANAGER.GET_VERSION() FROM DUAL; + * @ex_rslt 3.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Formatted for display in logs or monitoring systems. + * @example SELECT ENV_MANAGER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: ENV_MANAGER + * Version: 3.0.0 + * Build Date: 2025-10-22 16:00:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Shows evolution of package features over time. + * @example SELECT ENV_MANAGER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt ENV_MANAGER Version History: + * 3.0.0 (2025-10-22): Added package versioning system... + * 2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + + /** + * @name GET_PACKAGE_VERSION_INFO + * @desc Universal function to get formatted version information for any package. + * This centralized function is used by all packages in the system. + * @param pPackageName - Name of the package + * @param pVersion - Version string (MAJOR.MINOR.PATCH format) + * @param pBuildDate - Build date timestamp + * @param pAuthor - Package author name + * @example SELECT ENV_MANAGER.GET_PACKAGE_VERSION_INFO('FILE_MANAGER', '2.1.0', '2025-10-22 15:00:00', 'Grzegorz Michalski') FROM DUAL; + * @ex_rslt Package: FILE_MANAGER + * Version: 2.1.0 + * Build Date: 2025-10-22 15:00:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_PACKAGE_VERSION_INFO( + pPackageName VARCHAR2, + pVersion VARCHAR2, + pBuildDate VARCHAR2, + pAuthor VARCHAR2 + ) RETURN VARCHAR2; + + /** + * @name FORMAT_VERSION_HISTORY + * @desc Universal function to format version history for any package. + * Adds package name header and proper formatting. + * @param pPackageName - Name of the package + * @param pVersionHistory - Complete version history text + * @example SELECT ENV_MANAGER.FORMAT_VERSION_HISTORY('FILE_MANAGER', '2.1.0 (2025-10-22): Export procedures...') FROM DUAL; + * @ex_rslt FILE_MANAGER Version History: + * 2.1.0 (2025-10-22): Export procedures... + **/ + FUNCTION FORMAT_VERSION_HISTORY( + pPackageName VARCHAR2, + pVersionHistory VARCHAR2 + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE HASH + CHANGE DETECTION FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name CALCULATE_PACKAGE_HASH + * @desc Calculates SHA256 hash of package source code from ALL_SOURCE. + * Returns hash for both SPEC and BODY (if exists). + * Used for automatic change detection. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @param pPackageType - Type of package code ('PACKAGE' for SPEC, 'PACKAGE BODY' for BODY) + * @example SELECT ENV_MANAGER.CALCULATE_PACKAGE_HASH('CT_MRDS', 'FILE_MANAGER', 'PACKAGE') FROM DUAL; + * @ex_rslt A7B3C5D9E8F1234567890ABCDEF... (64-character SHA256 hash) + **/ + FUNCTION CALCULATE_PACKAGE_HASH( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageType VARCHAR2 -- 'PACKAGE' or 'PACKAGE BODY' + ) RETURN VARCHAR2; + + /** + * @name TRACK_PACKAGE_VERSION + * @desc Records package version and source code hash in A_PACKAGE_VERSION_TRACKING table. + * Automatically detects if source code changed without version update. + * Should be called after every package deployment. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @param pPackageVersion - Current version from PACKAGE_VERSION constant + * @param pPackageBuildDate - Build date from PACKAGE_BUILD_DATE constant + * @param pPackageAuthor - Author from PACKAGE_AUTHOR constant + * @example EXEC ENV_MANAGER.TRACK_PACKAGE_VERSION('CT_MRDS', 'FILE_MANAGER', '3.2.0', '2025-10-22 16:30:00', 'Grzegorz Michalski'); + * @ex_rslt Record inserted into A_PACKAGE_VERSION_TRACKING with change detection status + **/ + PROCEDURE TRACK_PACKAGE_VERSION( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2, + pPackageVersion VARCHAR2, + pPackageBuildDate VARCHAR2, + pPackageAuthor VARCHAR2 + ); + + /** + * @name CHECK_PACKAGE_CHANGES + * @desc Checks if package source code has changed since last tracking. + * Compares current hash with last recorded hash in A_PACKAGE_VERSION_TRACKING. + * Returns detailed change detection report. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @example SELECT ENV_MANAGER.CHECK_PACKAGE_CHANGES('CT_MRDS', 'FILE_MANAGER') FROM DUAL; + * @ex_rslt WARNING: Package changed without version update! + * Last Version: 3.2.0 + * Current Hash (SPEC): A7B3C5D9... + * Last Hash (SPEC): B8C4D6E0... + * RECOMMENDATION: Update PACKAGE_VERSION and PACKAGE_BUILD_DATE + **/ + FUNCTION CHECK_PACKAGE_CHANGES( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2; + + /** + * @name GET_PACKAGE_HASH_INFO + * @desc Returns formatted information about package hash and tracking history. + * Includes current hash, last tracked hash, and change detection status. + * @param pPackageOwner - Schema owner of the package + * @param pPackageName - Name of the package + * @example SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('CT_MRDS', 'FILE_MANAGER') FROM DUAL; + * @ex_rslt Package: CT_MRDS.FILE_MANAGER + * Current Version: 3.2.0 + * Current Hash (SPEC): A7B3C5D9... + * Last Tracked: 2025-10-22 16:30:00 + * Status: OK - No changes detected + **/ + FUNCTION GET_PACKAGE_HASH_INFO( + pPackageOwner VARCHAR2, + pPackageName VARCHAR2 + ) RETURN VARCHAR2; + +END ENV_MANAGER; +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..1c9f880 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkb @@ -0,0 +1,1285 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE + ---------------------------------------------------------------------------------------------------- + /** + * @name GET_ARCHIVAL_WHERE_CLAUSE + * @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration. + * Supports three strategies: THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID. + * @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY + * @return VARCHAR2 - WHERE clause for filtering archival candidates + **/ + FUNCTION GET_ARCHIVAL_WHERE_CLAUSE( + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) RETURN VARCHAR2 + IS + vWhereClause VARCHAR2(4000); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + CASE pSourceFileConfig.ARCHIVAL_STRATEGY + -- Legacy threshold-based strategy (backward compatible) + WHEN 'THRESHOLD_BASED' THEN + vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS; + + -- Archive data older than X months (0 = current month only) + WHEN 'MINIMUM_AGE_MONTHS' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); + END IF; + vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + -- Hybrid: Current month exclusion AND minimum age requirement + WHEN 'HYBRID' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy'); + END IF; + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' || + 'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + ELSE + RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY); + END CASE; + + RETURN vWhereClause; + END GET_ARCHIVAL_WHERE_CLAUSE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + vKeepInTrash BOOLEAN; -- Derived from config + + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + -- Check if archiving is enabled for this configuration + IF vSourceFileConfig.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archiving disabled for this configuration (IS_ARCHIVE_ENABLED=N). Skipping.', 'WARNING', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN; + END IF; + + -- Get TRASH policy from configuration + vKeepInTrash := (vSourceFileConfig.IS_KEEP_IN_TRASH = 'Y'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEEP_IN_TRASH=' || vSourceFileConfig.IS_KEEP_IN_TRASH, 'INFO', vParameters); + + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + -- Strategy-based trigger logic (MARS-828) + IF vSourceFileConfig.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN + -- MINIMUM_AGE_MONTHS: Archive based on age only, ignore thresholds + vArchivalTriggeredBy := 'AGE_BASED'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival strategy: MINIMUM_AGE_MONTHS (threshold-independent)','INFO'); + ELSE + -- THRESHOLD_BASED and HYBRID: Check thresholds + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + END IF; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + + -- Use strategy-based WHERE clause (MARS-828) + -- Using GROUP BY instead of DISTINCT to avoid ORA-22950 (object type ordering issue) + vQuery := ' + select CT_MRDS.t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' + and h.WORKFLOW_SUCCESSFUL = ''Y'' + group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')' + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||''' + and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||''' + and h.WORKFLOW_SUCCESSFUL = ''Y'' + ' + ; + vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month,'INFO', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + -- Note: DBMS_CLOUD.EXPORT_DATA may create multiple parquet files (parallel execution) + -- Instead of tracking individual files, we store the archive directory prefix + -- ARCH_PATH contain the directory URI where all parquet files are located + vFilename := vUri; -- Store directory prefix instead of individual filename + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status to ARCHIVED_AND_TRASHED (file will be moved to TRASH folder) + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -- Status reflects file is archived and kept in TRASH + ,ARCH_PATH = vFilename -- Now contains directory prefix, not individual file + ,PARTITION_YEAR = ym_loop.year -- Record which partition year the data was archived to + ,PARTITION_MONTH = ym_loop.month -- Record which partition month the data was archived to + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE'; + + -- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) + -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) + IF vProcessControlStatus = 'OK' THEN + IF NOT vKeepInTrash THEN + -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + -- Update status to ARCHIVED_AND_PURGED + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + END LOOP; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEEP_IN_TRASH=N).','INFO'); + ELSE + -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO'); + END IF; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED') + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_PATH = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + -- ROLLBACK: Delete all parquet files from archive directory + FOR arch_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vFilename -- vFilename now contains directory prefix + ) + ) LOOP + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vFilename || arch_file.object_name + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name); + END LOOP; + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN CT_MRDS.ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN CT_MRDS.ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + vWhereClause VARCHAR2(4000); + vOdsBucketUri VARCHAR2(1000); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + + -- Get WHERE clause based on archival strategy (MARS-828) + vWhereClause := GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vWhereClause','DEBUG',vWhereClause); + + -- Get ODS bucket URI before building query + vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + -- Use strategy-based WHERE clause for statistics (MARS-828) + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES + ,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''', + location_uri => '''||vOdsBucketUri||''' + ) + ) r + on t.filename = r.object_name' + ; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into CT_MRDS.A_TABLE_STAT_HIST values vStats; + delete from CT_MRDS.A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into CT_MRDS.A_TABLE_STAT values vStats; + COMMIT; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- TRASH FOLDER MANAGEMENT PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashPath VARCHAR2(1000); + vOdsPath VARCHAR2(1000); + vFilesRestored PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vRestoreLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath); + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Restore all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + FOR file_rec IN ( + SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pRestoreAll THEN + vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations + FOR file_rec IN ( + SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME, + c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_PATH = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashLocationUri VARCHAR2(1000); + vFilesDeleted PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vPurgeLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY); + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri + ); + vFilesDeleted := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Delete all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pPurgeAll THEN + vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations + FOR config_rec IN ( + SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_CONFIG c + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PURGE_TRASH_FOLDER( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pPurgeAll => pPurgeAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + PRAGMA AUTONOMOUS_TRANSACTION; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + RESTORE_FILE_FROM_TRASH( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pRestoreAll => pRestoreAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN CT_MRDS.ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL') + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- BATCH ARCHIVAL PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesArchived PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pArchiveAll THEN + vProcessingLevel := 'LEVEL 3: Archive All'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Archive All Enabled Tables', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pArchiveAll=TRUE'); + END IF; + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + IS_ARCHIVE_ENABLED, + IS_KEEP_IN_TRASH, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pArchiveAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pArchiveAll = TRUE) + ) + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF config_rec.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (IS_ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEEP_IN_TRASH=' || config_rec.IS_KEEP_IN_TRASH || ']', + 'INFO' + ); + + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesArchived := vTablesArchived + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully archived table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to archive table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch archival summary: Archived=' || vTablesArchived || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed, + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pArchiveAll => pArchiveAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_ARCHIVE_ALL; + + ---------------------------------------------------------------------------------------------------- + -- BATCH STATISTICS GATHERING PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTablesProcessed PLS_INTEGER := 0; + vTablesSkipped PLS_INTEGER := 0; + vTablesFailed PLS_INTEGER := 0; + vProcessingLevel VARCHAR2(50); + vEnabledFilter VARCHAR2(50); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine processing level and validate parameters + IF pSourceFileConfigKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 1: Single Config Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO'); + ELSIF pSourceKey IS NOT NULL THEN + vProcessingLevel := 'LEVEL 2: Source Key'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO'); + ELSIF pGatherAll THEN + vProcessingLevel := 'LEVEL 3: Gather All Statistics'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Gather All Enabled Statistics', 'INFO'); + ELSE + RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pGatherAll=TRUE'); + END IF; + + -- Set enabled filter info + vEnabledFilter := CASE WHEN pOnlyEnabled THEN 'IS_ARCHIVE_ENABLED=Y only' ELSE 'All tables' END; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Filter mode: ' || vEnabledFilter, 'INFO'); + + FOR config_rec IN ( + SELECT + A_SOURCE_FILE_CONFIG_KEY, + TABLE_ID, + IS_ARCHIVE_ENABLED, + A_SOURCE_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ( + -- Level 1: Specific config key + (pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey) + OR + -- Level 2: All configs for source key + (pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey) + OR + -- Level 3: All configs when pGatherAll = TRUE + (pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pGatherAll = TRUE) + ) + -- Apply IS_ARCHIVE_ENABLED filter if pOnlyEnabled = TRUE + AND (pOnlyEnabled = FALSE OR IS_ARCHIVE_ENABLED = 'Y') + ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY + ) LOOP + IF pOnlyEnabled AND config_rec.IS_ARCHIVE_ENABLED = 'N' THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Skipping table ' || config_rec.TABLE_ID || ' (IS_ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']', + 'INFO' + ); + vTablesSkipped := vTablesSkipped + 1; + ELSE + BEGIN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Gathering statistics for table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_ARCHIVE_ENABLED=' || config_rec.IS_ARCHIVE_ENABLED || ']', + 'INFO' + ); + + GATHER_TABLE_STAT(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY); + + vTablesProcessed := vTablesProcessed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Successfully gathered statistics for table ' || config_rec.TABLE_ID, + 'INFO' + ); + EXCEPTION + WHEN OTHERS THEN + vTablesFailed := vTablesFailed + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + 'Failed to gather statistics for table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM, + 'ERROR' + ); + -- Continue with next table + END; + END IF; + END LOOP; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT( + vProcessingLevel || ' - Batch statistics gathering summary: Processed=' || vTablesProcessed || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed || ' [Filter: ' || vEnabledFilter || ']', + 'INFO' + ); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT_ALL; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''', + 'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''', + 'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''', + 'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||'''' + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT_ALL( + pSourceFileConfigKey => pSourceFileConfigKey, + pSourceKey => pSourceKey, + pGatherAll => pGatherAll, + pOnlyEnabled => pOnlyEnabled + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END FN_GATHER_TABLE_STAT_ALL; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..8b14196 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_ARCHIVER.pkg @@ -0,0 +1,279 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) || + '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || + '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || + '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || + '3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) || + '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name GET_TABLE_STAT + * @desc Private function to retrieve table statistics for archival processing. + * Returns A_TABLE_STAT record with table metadata and row counts. + * @param pSourceFileConfigKey - Configuration key for source file + * @return CT_MRDS.A_TABLE_STAT%ROWTYPE - Table statistics record + * @private Internal function for archival operations + **/ + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name FN_ARCHIVE_TABLE_DATA + * @desc Function wrapper for ARCHIVE_TABLE_DATA procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. + * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately). + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name FN_GATHER_TABLE_STAT + * @desc Function wrapper for GATHER_TABLE_STAT procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT procedure and captures execution result. + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + /** + * @name GATHER_TABLE_STAT_ALL + * @desc Multi-level batch statistics gathering procedure with three granularity levels. + * Processes configurations based on IS_ARCHIVE_ENABLED setting (when pOnlyEnabled=TRUE). + * Gathers statistics for external tables and inserts data into A_TABLE_STAT and A_TABLE_STAT_HIST. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (e.g., 'LM', 'C2D') (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y' + * @example -- Level 1: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE); + * @example -- All tables regardless of IS_ARCHIVE_ENABLED: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE, pOnlyEnabled => FALSE); + **/ + PROCEDURE GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ); + + /** + * @name FN_GATHER_TABLE_STAT_ALL + * @desc Function wrapper for GATHER_TABLE_STAT_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (medium priority) + * @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority) + * @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y' + * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_GATHER_TABLE_STAT_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pGatherAll IN BOOLEAN DEFAULT FALSE, + pOnlyEnabled IN BOOLEAN DEFAULT TRUE + ) RETURN PLS_INTEGER; + + /** + * @name ARCHIVE_ALL + * @desc Multi-level batch archival procedure with three granularity levels. + * Only processes configurations where IS_ARCHIVE_ENABLED='Y'. + * TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example -- Level 1: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceFileConfigKey => 123); + * @example -- Level 2: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceKey => 'LM'); + * @example -- Level 3: CALL FILE_ARCHIVER.ARCHIVE_ALL(pArchiveAll => TRUE); + **/ + PROCEDURE ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name FN_ARCHIVE_ALL + * @desc Function wrapper for ARCHIVE_ALL procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_ALL procedure and captures execution result. + * @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority) + * @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (medium priority) + * @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority) + * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_ALL(pSourceKey => 'LM') FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION FN_ARCHIVE_ALL ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL, + pArchiveAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Restores files from TRASH folder back to ODS at three different granularity levels. + * Moves files from TRASH subfolder back to ODS subfolder in DATA bucket. + * Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345); + * @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341); + * @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE); + **/ + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Function overload for RESTORE_FILE_FROM_TRASH procedure. + * Returns SQLCODE for Python library integration. + * Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name PURGE_TRASH_FOLDER + * @desc Deletes files from TRASH folder at three different granularity levels. + * Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345); + * @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341); + * @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE); + **/ + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name PURGE_TRASH_FOLDER + * @desc Function overload for PURGE_TRASH_FOLDER procedure. + * Returns SQLCODE for Python library integration. + * Calls the main PURGE_TRASH_FOLDER procedure and captures execution result. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkb b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkb new file mode 100644 index 0000000..3daac26 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkb @@ -0,0 +1,2009 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_MANAGER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: NORMALIZE_DATE_FORMAT + ---------------------------------------------------------------------------------------------------- + /** + * Purpose: Normalize Oracle date format strings for use in external tables + * + * Problem: ISO 8601 formats like 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' fail because + * literal character 'T' must be enclosed in double quotes for Oracle + * external table DATE column definitions. + * + * Solution: Detect unquoted 'T' separator and wrap it in double quotes + * + * Parameters: + * pDateFormat - Original date format from A_COLUMN_DATE_FORMAT table + * + * Returns: Normalized format with quoted 'T' if applicable + * + * Examples: + * Input: 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' + * Output: 'YYYY-MM-DD"T"HH24:MI:SS.FF3TZH:TZM' + * + * Input: 'DD/MM/YYYY HH24:MI:SS' (no T) + * Output: 'DD/MM/YYYY HH24:MI:SS' (unchanged) + * + * Input: 'YYYY-MM-DD"T"HH24:MI:SS' (already quoted) + * Output: 'YYYY-MM-DD"T"HH24:MI:SS' (unchanged) + * + * Author: Grzegorz Michalski + * Date: 2025-11-27 + * Version: 1.0.0 (MARS-1046) + */ + FUNCTION NORMALIZE_DATE_FORMAT(pDateFormat VARCHAR2) RETURN VARCHAR2 IS + vNormalizedFormat VARCHAR2(500); + BEGIN + -- Return NULL if input is NULL + IF pDateFormat IS NULL THEN + RETURN NULL; + END IF; + + vNormalizedFormat := pDateFormat; + + -- Check if 'T' separator exists and is NOT already quoted + -- Pattern: [YMD]T[HM] (date component + T + time component) + IF INSTR(vNormalizedFormat, '"T"') = 0 AND + REGEXP_LIKE(vNormalizedFormat, '[YMD]T[HM]') THEN + + -- Wrap 'T' in double quotes using regex replace + -- Pattern matches: (date format char) + T + (time format char) + -- Replacement: \1 + "T" + \2 + vNormalizedFormat := REGEXP_REPLACE(vNormalizedFormat, '([YMD])T([HM])', '\1"T"\2'); + END IF; + + RETURN vNormalizedFormat; + + EXCEPTION + WHEN OTHERS THEN + -- If normalization fails, return original format (safety fallback) + RETURN pDateFormat; + END NORMALIZE_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL + , pSourceFileReceivedKey IN NUMBER DEFAULT NULL + , pSourceFileConfigKey IN NUMBER DEFAULT NULL) + RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pFileUri => '''||nvl(pFileUri,'NULL')||'''' + ,'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') + ,'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + + BEGIN + IF pFileUri IS NOT NULL THEN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE REGEXP_LIKE(pFileUri, A_SOURCE_KEY||'/'||SOURCE_FILE_ID||'/'||TABLE_ID||'/'||SOURCE_FILE_NAME_PATTERN); + ELSIF pSourceFileReceivedKey IS NOT NULL THEN + SELECT T.* + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG T, CT_MRDS.A_SOURCE_FILE_RECEIVED R + WHERE T.A_SOURCE_FILE_CONFIG_KEY = R.A_SOURCE_FILE_CONFIG_KEY + AND R.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + ELSIF pSourceFileConfigKey IS NOT NULL THEN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + END IF; + -- Set global package variable vgSourceFileConfigKey - used in error messages + vgSourceFileConfigKey := vSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY; + EXCEPTION + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); + + WHEN OTHERS THEN + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileConfig; + + END GET_SOURCE_FILE_CONFIG; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey IN NUMBER DEFAULT NULL) + -- + -- Get source file received info + -- + RETURN tSourceFileReceived + IS + vSourceFileReceived tSourceFileReceived; + vBucket VARCHAR2(400); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + BEGIN + SELECT R.A_SOURCE_FILE_RECEIVED_KEY, R.A_SOURCE_FILE_CONFIG_KEY, + 'INBOX'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_INBOX, + 'ODS'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_ODS, + 'QUARANTINE'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_QUARANTINE, + 'ARCHIVE'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/' as SOURCE_FILE_PREFIX_ARCHIVE, + R.SOURCE_FILE_NAME, + R.RECEPTION_DATE, R.PROCESSING_STATUS, R.EXTERNAL_TABLE_NAME + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED R, CT_MRDS.A_SOURCE_FILE_CONFIG T + WHERE R.A_SOURCE_FILE_CONFIG_KEY = T.A_SOURCE_FILE_CONFIG_KEY + AND A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + RETURN vSourceFileReceived; + + END GET_SOURCE_FILE_RECEIVED_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER + -- + -- Register a newly received source file A_SOURCE_FILE_RECEIVED + -- This overload automatically determines source file type from the file name + -- + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceivedKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO',vParameters); + + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedName); + vSourceFileReceivedKey := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName, vSourceFileConfig); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN vSourceFileReceivedKey; + EXCEPTION + + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + + WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); + + WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END REGISTER_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION REGISTER_SOURCE_FILE_RECEIVED( + pSourceFileReceivedName IN VARCHAR2 + ,pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE) + RETURN PLS_INTEGER + -- + -- Register a newly received source file A_SOURCE_FILE_RECEIVED + -- + IS + vExternalTableName VARCHAR2(200); + vDirName VARCHAR2(1000); + vFileName VARCHAR2(1000); + vChecksum A_SOURCE_FILE_RECEIVED.CHECKSUM%TYPE; + vCreated A_SOURCE_FILE_RECEIVED.CREATED%TYPE; + vBytes A_SOURCE_FILE_RECEIVED.BYTES%TYPE; + vSourceFileReceivedKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vRow CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''' + ,'pSourceFileConfig => '||'tSourceFileConfig record type')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + vDirName := REGEXP_SUBSTR(pSourceFileReceivedName, '(.*/)(.*)', 1, 1, NULL, 1); + -- Remove prefix from file name + vFileName := REGEXP_SUBSTR(pSourceFileReceivedName,'[^/]*$'); + + ENV_MANAGER.LOG_PROCESS_EVENT('gvCredentialName','DEBUG',ENV_MANAGER.gvCredentialName); + ENV_MANAGER.LOG_PROCESS_EVENT('gvInboxBucketUri','DEBUG',ENV_MANAGER.gvInboxBucketUri); + ENV_MANAGER.LOG_PROCESS_EVENT('vDirName','DEBUG',vDirName); + + SELECT + checksum, created, bytes + INTO + vChecksum, vCreated, vBytes + FROM DBMS_CLOUD.LIST_OBJECTS(ENV_MANAGER.gvCredentialName, + ENV_MANAGER.gvInboxBucketUri || vDirName + ) + WHERE object_name = vFileName + ; + vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL; + vExternalTableName := REPLACE( + REGEXP_SUBSTR(pSourceFileConfig.TEMPLATE_TABLE_NAME||'_'||vSourceFileReceivedKey, + '\..*'), + '.',''); + + INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED + (A_SOURCE_FILE_RECEIVED_KEY, A_SOURCE_FILE_CONFIG_KEY, + SOURCE_FILE_NAME, RECEPTION_DATE, + PROCESSING_STATUS, EXTERNAL_TABLE_NAME, + CHECKSUM, CREATED, BYTES) + VALUES (vSourceFileReceivedKey, pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY, + vFileName, SYSDATE, + 'RECEIVED', vExternalTableName, + vChecksum, vCreated, vBytes); + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileReceivedKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD + ||cgBL||' '||'File: '||ENV_MANAGER.gvInboxBucketUri || vDirName || vFileName; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, vgMsgTmp); + + WHEN DUP_VAL_ON_INDEX THEN + select * into vRow + from CT_MRDS.A_SOURCE_FILE_RECEIVED + where CHECKSUM = vChecksum + and CREATED = vCreated + and BYTES = vBytes + ; + vgMsgTmp := ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED + ||cgBL||' '||'Details about existing File: ' + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'A_SOURCE_FILE_RECEIVED_KEY = '||vRow.A_SOURCE_FILE_RECEIVED_KEY + ||cgBL||' '||'A_SOURCE_FILE_CONFIG_KEY = '||vRow.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||' '||'SOURCE_FILE_NAME = '||vRow.SOURCE_FILE_NAME + ||cgBL||' '||'CHECKSUM = '||vRow.CHECKSUM + ||cgBL||' '||'CREATED = '||vRow.CREATED + ||cgBL||' '||'BYTES = '||vRow.BYTES + ||cgBL||' '||'RECEPTION_DATE = '||vRow.RECEPTION_DATE + ||cgBL||' '||'PROCESSING_STATUS = '||vRow.PROCESSING_STATUS + ||cgBL||' '||'EXTERNAL_TABLE_NAME = '||vRow.EXTERNAL_TABLE_NAME + ||cgBL||' '||'-------------------------' + ||cgBL||' '||'There cannot be two files with the same values for (CHECKSUM, CREATED, BYTES)' + ; + + +-- vChecksum, vCreated, vBytes + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, vgMsgTmp); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END REGISTER_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey IN PLS_INTEGER, pStatus IN VARCHAR2) + -- + -- Change status of file in the A_SOURCE_FILE_RECEIVED table + -- + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') + ,'pStatus => '''||nvl(pStatus, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS=pStatus + WHERE A_SOURCE_FILE_RECEIVED_KEY=pSourceFileReceivedKey; + COMMIT; + ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to '||pStatus,'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END SET_SOURCE_FILE_RECEIVED_STATUS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate IN VARCHAR2) + RETURN CLOB + -- + -- Create list of columns for DBMS_CLOUD.CREATE_EXTERNAL_TABLE from existing template table + -- + IS + vColumnList CLOB; + vTableName VARCHAR2(200); + vSchemaName VARCHAR2(200); + BEGIN + vSchemaName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'.*\.'),'.',''); + vTableName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'\..*'),'.',''); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SQLTERMINATOR', True); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'COLLATION_CLAUSE', 'NEVER'); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'REF_CONSTRAINTS', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'STORAGE', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'TABLESPACE', False); + DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SEGMENT_ATTRIBUTES', False); + vColumnList := RTRIM( + LTRIM( + REGEXP_SUBSTR(DBMS_METADATA.GET_DDL('TABLE', vTableName, vSchemaName),'\(.*\)',1,1,'mn'), + '('), + ')'); + RETURN vColumnList; + END GET_EXTERNAL_TABLE_COLUMNS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE CREATE_EXTERNAL_TABLE ( + pTableName IN VARCHAR2, + pTemplateTableName IN VARCHAR2, + pPrefix IN VARCHAR2, + pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri, + pFileName IN VARCHAR2 DEFAULT NULL, + pDelimiter IN VARCHAR2 DEFAULT ',', + pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NEW PARAMETER FOR FILE ENCODING + ) + -- + -- Create external table for a single source file to validate the file structure + -- + IS + vTableName VARCHAR2(200); + vColumnList CLOB; + vFieldList CLOB; + vFormat VARCHAR2(200); + + vPrefix VARCHAR2(200); + vFileName VARCHAR2(1000); + vFileExtension VARCHAR2(200); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTableName => '''||nvl(pTableName, 'NULL')||'''' + ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pPrefix => '''||nvl(pPrefix, 'NULL')||'''' + ,'pBucketUri => '''||nvl(pBucketUri, 'NULL')||'''' + ,'pFileName => '''||nvl(pFileName, 'NULL')||'''' + ,'pDelimiter => '''||nvl(pDelimiter, 'NULL')||'''' + ,'pEncoding => '''||nvl(pEncoding, 'NULL')||'''' -- MARS-1049: NOWY + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + -- Strip off leading and trailing slashes from prefix + vPrefix := TRIM(BOTH '/' FROM pPrefix); + + -- Generate column and field list from template table + GENERATE_EXTERNAL_TABLE_PARAMS (pTemplateTableName, vColumnList, vFieldList); + + --vFormat evaluation based on pBucketUri first, then pPrefix + -- Archive bucket should use parquet + implicit partitioning regardless of prefix + IF INSTR(pBucketUri, ENV_MANAGER.gvArchiveBucketName)>0 THEN + vFormat := '{"type": "parquet" + ,"implicit_partition_type": "hive" + ,"implicit_partition_columns":["PARTITION_YEAR","PARTITION_MONTH"]}'; + vColumnList := vColumnList||cgBL||' , "PARTITION_YEAR" varchar2(4)'||cgBL||', "PARTITION_MONTH" varchar2(2)'; + vFieldList := NULL; + vFileExtension := '.parquet'; + + -- For INBOX, ODS, and other ARCHIVE prefixes (not in archive bucket) use CSV + ELSIF SUBSTR(pPrefix,1,5) = 'INBOX' OR SUBSTR(pPrefix,1,3) = 'ODS' + OR SUBSTR(pPrefix,1,7) = 'ARCHIVE' + THEN + -- MARS-1049: Create format with encoding if specified + IF pDelimiter = '|' THEN + IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN + vFormat := json_object( + 'delimiter' VALUE '|', + 'skipheaders' VALUE '1', + 'characterset' VALUE pEncoding + ); + ELSE + vFormat := json_object('delimiter' VALUE '|', 'skipheaders' VALUE '1'); + END IF; + ELSE + IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN + vFormat := json_object( + 'type' VALUE 'CSV', + 'skipheaders' VALUE '1', + 'ignoremissingcolumns' VALUE 'true', + 'characterset' VALUE pEncoding + ); + ELSE + vFormat := json_object('type' VALUE 'CSV', 'skipheaders' VALUE '1', 'ignoremissingcolumns' value 'true'); + END IF; + END IF; + + vFileExtension := '.csv'; + + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); + END IF; + + -- No filename give: Match all csv files + IF pFileName IS NOT NULL THEN + vFileName := pFileName; + ELSE + vFileName := pBucketUri||vPrefix||'/*'||vFileExtension; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('pTableName', 'DEBUG', pTableName); + ENV_MANAGER.LOG_PROCESS_EVENT('ENV_MANAGER.vpCredentialName', 'DEBUG', ENV_MANAGER.gvCredentialName); + ENV_MANAGER.LOG_PROCESS_EVENT('vFileName', 'DEBUG', vFileName); + ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', vColumnList); + ENV_MANAGER.LOG_PROCESS_EVENT('vFieldList', 'DEBUG', vFieldList); + ENV_MANAGER.LOG_PROCESS_EVENT('vFormat', 'DEBUG', vFormat); + + -- Pre-validation: Check CSV column count for CSV files only + IF SUBSTR(pPrefix,1,5) = 'INBOX' AND pFileName IS NOT NULL THEN + DECLARE + vCsvFirstLine VARCHAR2(4000); + vCsvColCount NUMBER := 0; + vTemplateColCount NUMBER := 0; + vExcessColumns VARCHAR2(2000); + + -- Get template column count + CURSOR c_template_count IS + SELECT COUNT(*) as col_count + FROM ALL_TAB_COLUMNS + WHERE OWNER = UPPER(REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.','')) + AND TABLE_NAME = UPPER(REGEXP_REPLACE(pTemplateTableName,'^.*\.','')); + + BEGIN + -- Get template column count + FOR rec IN c_template_count LOOP + vTemplateColCount := rec.col_count; + END LOOP; + + -- Read first line of CSV to count columns + BEGIN + SELECT UTL_RAW.CAST_TO_VARCHAR2( + DBMS_LOB.SUBSTR( + DBMS_CLOUD.GET_OBJECT( + credential_name => ENV_MANAGER.gvCredentialName, + object_uri => pFileName + ), + 4000, 1 + ) + ) INTO vCsvFirstLine FROM DUAL; + + -- Count commas in header line + 1 for total columns + vCsvColCount := REGEXP_COUNT(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), ',') + 1; + + ENV_MANAGER.LOG_PROCESS_EVENT('CSV Column Count: ' || vCsvColCount || ', Template Column Count: ' || vTemplateColCount, 'INFO', vParameters); + + -- Check for excess columns + IF vCsvColCount > vTemplateColCount THEN + vgMsgTmp := ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED + ||cgBL||'EXCESS COLUMNS DETECTED!' + ||cgBL||'CSV file has ' || vCsvColCount || ' columns but template expects only ' || vTemplateColCount + ||cgBL||'Excess columns: ' || (vCsvColCount - vTemplateColCount) + ||cgBL||'CSV header: ' || SUBSTR(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), 1, 200) + ||cgBL||'POSSIBLE SOLUTIONS:' + ||cgBL||' 1. Remove excess columns from CSV file before processing' + ||cgBL||' 2. Add excess columns to template table: ' || pTemplateTableName; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, vgMsgTmp); + END IF; + + EXCEPTION + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + RAISE; -- Re-raise the excess columns error + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + RAISE; -- Re-raise the validation error + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not perform pre-validation column count check: ' || SQLERRM, 'WARN', vParameters); + -- Continue with normal processing if pre-validation fails + END; + END; + END IF; + + DBMS_CLOUD.CREATE_EXTERNAL_TABLE( + TABLE_NAME => pTableName, + CREDENTIAL_NAME => ENV_MANAGER.gvCredentialName, + FILE_URI_LIST => vFileName, + COLUMN_LIST => vColumnList, + FIELD_LIST => vFieldList, + FORMAT => vFormat + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + RAISE; -- Re-raise the excess columns error with specific code -20011 + WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN + vgMsgTmp := ENV_MANAGER.MSG_UNKNOWN_PREFIX || ': ' || pPrefix; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, vgMsgTmp); + WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END CREATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) + -- + -- Create external table for a single source file to validate the file structure + -- + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived tSourceFileReceived; + vTableName VARCHAR2(200); + vFileName VARCHAR2(1000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + vTableName := vSourceFileConfig.TEMPLATE_TABLE_NAME; + + vFileName := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX||vSourceFileReceived.SOURCE_FILE_NAME; + + CREATE_EXTERNAL_TABLE( + pTableName => vSourceFileReceived.EXTERNAL_TABLE_NAME, + pTemplateTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME, + pPrefix => vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX, + pBucketUri => ENV_MANAGER.gvInboxBucketUri, + pFileName => vFileName, + pDelimiter => ',', + pEncoding => vSourceFileConfig.ENCODING -- MARS-1049: NOWY PARAMETR + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + END CREATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey IN NUMBER) + -- + -- Check the structure of the received file using DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + -- + IS + vSourceFileReceived tSourceFileReceived; + vOperationId NUMBER := -1; + vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; + vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; + vErrors NUMBER := 0; + vNumRows NUMBER := 0; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + ENV_MANAGER.LOG_PROCESS_EVENT('vSourceFileReceived.EXTERNAL_TABLE_NAME: '||vSourceFileReceived.EXTERNAL_TABLE_NAME,'DEBUG', vParameters); + BEGIN + DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(vSourceFileReceived.EXTERNAL_TABLE_NAME, vOperationId); + EXCEPTION + WHEN OTHERS THEN + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_VALIDATION_FAILED, vParameters, 'FILE_MANAGER'); + + -- Call detailed validation error analysis and log the results + DECLARE + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vValidationLogTable VARCHAR2(200); + vTemplateSchema VARCHAR2(200); + vTemplateTable VARCHAR2(200); + vCsvFileUri VARCHAR2(2000); + vAnalysisResult VARCHAR2(32000); + vFailedOperationId NUMBER; + BEGIN + -- Get source file configuration + vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); + + -- Extract template schema and table from template table name + vTemplateSchema := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'.*\.'),'.',''); + vTemplateTable := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'\..*'),'.',''); + + -- Construct CSV file URI + vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX || vSourceFileReceived.SOURCE_FILE_NAME; + + -- Find the failed validation operation ID + SELECT MAX(ID) INTO vFailedOperationId + FROM USER_LOAD_OPERATIONS + WHERE TABLE_NAME = vSourceFileReceived.EXTERNAL_TABLE_NAME + AND TYPE = 'VALIDATE' + AND STATUS != 'COMPLETED'; + + -- Get validation log table name + IF vFailedOperationId IS NOT NULL THEN + SELECT LOGFILE_TABLE INTO vValidationLogTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vFailedOperationId; + + -- Call detailed error analysis + vAnalysisResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( + pValidationLogTable => vValidationLogTable, + pTemplateSchema => vTemplateSchema, + pTemplateTable => vTemplateTable, + pCsvFileUri => vCsvFileUri + ); + + -- Log detailed analysis results + ENV_MANAGER.LOG_PROCESS_EVENT('DETAILED VALIDATION ERROR ANALYSIS:', 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(vAnalysisResult, 'ERROR', vParameters); + END IF; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error during validation analysis: ' || SQLERRM, 'ERROR', vParameters); + END; + + MOVE_FILE(pSourceFileReceivedKey => pSourceFileReceivedKey, pDestination => 'QUARANTINE'); + -- Ensure the status change is committed before raising exception + COMMIT; + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); + END; + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); + IF vOperationId = -1 + THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + END IF; + + SELECT BADFILE_TABLE, ROWS_LOADED, STATUS + INTO vBadfileTable, vNumRows, vStatus + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + +-- DBMS_OUTPUT.PUT_LINE(vStatus); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATED'); + ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to VALIDATED','DEBUG', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED; + ENV_MANAGER.LOG_PROCESS_ERROR(vgMsgTmp, vParameters, 'FILE_MANAGER'); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); + + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD||cgBL||SQLERRM; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, vgMsgTmp); + + ELSIF SQLCODE = -20003 THEN + execute immediate 'select LISTAGG(record, '''||cgBL||''') from (select * from '||REGEXP_SUBSTR(SQLERRM, '"([^"]+)"."([^"]+)"')||' order by rownum desc) where rownum <=2' + into vgMsgTmp; + vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||vgMsgTmp; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); +-- ELSIF SQLCODE = -20000 THEN + -- TO_DO Add additional info about current config + -- ENV_MANAGER.MSG_FILE_VALIDATION_FAILED := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||FILE_MANAGER.OUTPUT_SOURCE_FILE_CONFIG_INFO( ..config key value.. ); +-- dbms_output.put_line(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT, 'ERROR', vParameters); +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); +-- RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); + + ELSE + -- Log complete error details including full stack trace and backtrace + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END IF; + END VALIDATE_SOURCE_FILE_RECEIVED; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2 + -- + -- wrapper for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + -- + IS + vOperationId NUMBER := -1; + vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; + vLogfileTable USER_LOAD_OPERATIONS.LOGFILE_TABLE%TYPE; + vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vDetails clob; + TYPE TCURSOR is REF CURSOR; + vCursor TCURSOR; + vQuery VARCHAR2(1000); + vRecord VARCHAR2(10000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTableName => '''||nvl(pTableName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(pTableName, vOperationId); + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); + IF vOperationId = -1 + THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + END IF; + + SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE + INTO vStatus, vLogfileTable, vBadfileTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + + RETURN vStatus; + EXCEPTION + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + + WHEN OTHERS THEN + SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE + INTO vStatus, vLogfileTable, vBadfileTable + FROM USER_LOAD_OPERATIONS + WHERE ID = vOperationId; + vQuery := 'select record from ( + select + nvl(l.record,''----------------------------------------------------'') as record + ,rownum as lp + ,max(case when nvl(instr(l.record, ''error'' ),0) > 0 then rownum else 0 end) over (partition by 1) as ENV_MANAGER.ERR_row + from '||vLogfileTable||' l + ) + where lp >= ENV_MANAGER.ERR_row + order by rownum'; + + vDetails := vStatus||cgBL||'----------------------------------------------------'||cgBL; + + OPEN vCursor for vQuery; + loop + fetch vCursor into vRecord; + EXIT WHEN vCursor%NOTFOUND; + vDetails := vDetails ||vRecord ||cgBL; +-- for i in loop +-- vDetails := vDetails ||i.record ||cgBL; + end loop; + CLOSE vCursor; + vDetails := vDetails||'More details can be found in below tables:'||cgBL|| + ' SELECT * FROM USER_LOAD_OPERATIONS WHERE ID = '||vOperationId||';'||cgBL|| + ' SELECT * FROM '||vLogfileTable||';'||cgBL|| + ' SELECT * FROM '||vBadfileTable||';' + ; + + RETURN vDetails; + + END VALIDATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2 + -- + -- Simple + -- + IS + vCount PLS_INTEGER; + BEGIN + execute immediate 'select count(1) from '||pTableName into vCount; + IF vCount >= 0 + THEN + RETURN 'PASSED'; + END IF; + + RETURN 'FAILED'; + EXCEPTION + WHEN OTHERS THEN + RETURN 'FAILED'; + END S_VALIDATE_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DROP_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) + -- + -- Drop external table created to validate the file structure + -- + IS + vSourceFileReceived tSourceFileReceived; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + EXECUTE IMMEDIATE 'DROP TABLE '||vSourceFileReceived.EXTERNAL_TABLE_NAME; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END DROP_EXTERNAL_TABLE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE COPY_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) + -- + -- Possible pDestination values are: 'ODS' or 'ARCHIVE' + -- + IS + vSourceFileReceivedInfo tSourceFileReceived; + vSourceObject VARCHAR2(2000); + vTargetObject VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; +-- vStatus VARCHAR2(20); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + + IF pDestination = 'ODS' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + END IF; + + DBMS_CLOUD.COPY_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => vSourceObject, + target_object_uri => vTargetObject, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File copied to '||pDestination||' target location','DEBUG', vTargetObject); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END COPY_FILE; + + ---------------------------------------------------------------------------------------------------- + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE MOVE_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) + -- + -- Possible pDestination values are: 'ODS' or 'ARCHIVE' + -- + IS + vSourceFileReceivedInfo tSourceFileReceived; + vSourceObject VARCHAR2(2000); + vTargetObject VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStatus VARCHAR2(20); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + + vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); + + IF pDestination = 'ODS' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vStatus := 'READY_FOR_INGESTION'; + ELSIF pDestination = 'QUARANTINE' THEN + vSourceObject := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vTargetObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_QUARANTINE||vSourceFileReceivedInfo.SOURCE_FILE_NAME; + vStatus := 'VALIDATION_FAILED'; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + END IF; + + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => vSourceObject, + target_object_uri => vTargetObject, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File moved to '||pDestination||' target location','DEBUG', vTargetObject); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => vStatus); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN + ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM := ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM + ||cgBL||' '||'Possible parameters are: ''ODS'' or ''ARCHIVE''' + ||cgBL||' '||'Provided destination parameter: '''||pDestination||''''; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END MOVE_FILE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DELETE_FOLDER_CONTENTS(pBucketArea IN VARCHAR2, pFolderPrefix IN VARCHAR2) + -- + -- Delete all files from specified folder in cloud storage + -- pBucketArea: 'INBOX', 'DATA', 'ARCHIVE' + -- pFolderPrefix: folder path within bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/') + -- + IS + vBucketUri VARCHAR2(2000); + vFolderUri VARCHAR2(2000); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vFilesDeleted PLS_INTEGER := 0; + vObjectName VARCHAR2(4000); + vFullObjectUri VARCHAR2(4000); + + -- Cursor to list all objects in the folder + CURSOR c_objects IS + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => ENV_MANAGER.gvCredentialName, + location_uri => vBucketUri + )) + WHERE object_name IS NOT NULL + AND object_name LIKE pFolderPrefix || '%'; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pBucketArea => '''||nvl(pBucketArea, 'NULL')||'''', + 'pFolderPrefix => '''||nvl(pFolderPrefix, 'NULL')||'''' + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Get bucket URI based on bucket area + vBucketUri := GET_BUCKET_URI(pBucketArea); + + ENV_MANAGER.LOG_PROCESS_EVENT('Listing objects in bucket with prefix: ' || pFolderPrefix, 'DEBUG', vBucketUri); + + -- List and delete all objects in the folder + FOR obj_rec IN c_objects LOOP + vObjectName := obj_rec.object_name; + vFullObjectUri := vBucketUri || vObjectName; + + BEGIN + ENV_MANAGER.LOG_PROCESS_EVENT('Deleting object', 'DEBUG', vFullObjectUri); + + DBMS_CLOUD.DELETE_OBJECT( + credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vFullObjectUri + ); + + vFilesDeleted := vFilesDeleted + 1; + ENV_MANAGER.LOG_PROCESS_EVENT('Object deleted successfully', 'DEBUG', vObjectName); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error deleting object: ' || vObjectName || ' - ' || SQLERRM, 'ERROR', vParameters); + -- Continue with next file instead of stopping the whole process + END; + END LOOP; + + ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted: ' || vFilesDeleted, 'INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error in DELETE_FOLDER_CONTENTS: ' || SQLERRM, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + END DELETE_FOLDER_CONTENTS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + -- + -- Ubmrella procedure that calls + -- - REGISTER_SOURCE_FILE_RECEIVED + -- - CREATE_EXTERNAL_TABLE + -- - VALIDATE_SOURCE_FILE_RECEIVED + -- - DROP_EXTERNAL_TABLE + -- - MOVE_FILE + IS + vSourceFileId NUMBER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + vSourceFileId := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName); + CREATE_EXTERNAL_TABLE(vSourceFileId); + VALIDATE_SOURCE_FILE_RECEIVED(vSourceFileId); + DROP_EXTERNAL_TABLE(vSourceFileId); +-- COPY_FILE(vSourceFileId, 'ODS'); + MOVE_FILE(vSourceFileId, 'ODS'); + SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => vSourceFileId, pStatus => 'READY_FOR_INGESTION'); + + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + + EXCEPTION + -- -20001 + WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); + -- -20002 + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); + -- -20003 + WHEN ENV_MANAGER.ERR_MULTIPLE_MATCH_FOR_SRCFILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); + -- -20004 + WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); + -- -20005 + WHEN ENV_MANAGER.ERR_MULTIPLE_COLUMN_DATE_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); + -- -20006 + WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); + -- -20007 + WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); + -- -20008 + WHEN ENV_MANAGER.ERR_MULTI_CONFIG_FOR_RECEIVED_FILE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); + -- -20009 + WHEN ENV_MANAGER.ERR_FILE_NOT_FOUND_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD); + -- -20010 + WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_VALIDATION_FAILED), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); + -- -20011 + WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NO_CONFIG_MATCH), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH, ENV_MANAGER.MSG_NO_CONFIG_MATCH); + -- -20012 + WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN_PREFIX), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); + -- -20013 + WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_TABLE_NOT_EXISTS), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS); + -- -20014 + WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_COLUMN_NOT_EXISTS), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS); + -- -20015 + WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE); + -- -20016 + WHEN ENV_MANAGER.ERR_MISSING_SOURCE_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_SOURCE_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); + -- -20017 + WHEN ENV_MANAGER.ERR_NULL_SOURCE_FILE_CONFIG_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY, ENV_MANAGER.MSG_NULL_SOURCE_FILE_CONFIG_KEY); + -- -20018 + WHEN ENV_MANAGER.ERR_DUPLICATED_SOURCE_KEY THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); + -- -20019 + WHEN ENV_MANAGER.ERR_MISSING_CONTAINER_CONFIG THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG, ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG); + -- -20020 + WHEN ENV_MANAGER.ERR_MULTIPLE_CONTAINER_ENTRIES THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); + -- -20021 + WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); + -- -20022 + WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); + -- -20023 + WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); + -- -20024 + WHEN ENV_MANAGER.ERR_WRONG_DATE_TIMESTAMP_FORMAT THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); + + -- -20011 + WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN + ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED, vParameters, 'FILE_MANAGER'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED); + + -- -20999 + WHEN ENV_MANAGER.ERR_UNKNOWN THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END PROCESS_SOURCE_FILE; + + +---------------------------------------------------------------------------------------------------- + + FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PROCESS_SOURCE_FILE(pSourceFileReceivedName => pSourceFileReceivedName); + ---- + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PROCESS_SOURCE_FILE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DATE_FORMAT( + pTemplateTableName IN VARCHAR2, + pColumnName IN VARCHAR2 + ) + RETURN VARCHAR2 + IS + vDateFormat A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vColumnName VARCHAR2(200); + vGetDefault BOOLEAN := FALSE; + BEGIN + vColumnName := trim(pColumnName); + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnName => '''||nvl(vColumnName, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + BEGIN + SELECT DATE_FORMAT + INTO vDateFormat + FROM CT_MRDS.A_COLUMN_DATE_FORMAT F + WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName + AND F.COLUMN_NAME = vColumnName; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vGetDefault := TRUE; + WHEN TOO_MANY_ROWS THEN + -- Below error should not happened because: + -- Unique constraint added on table A_COLUMN_DATE_FORMAT on columns: (TEMPLATE_TABLE_NAME, COLUMN_NAME) + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); + END; + IF vGetDefault THEN + BEGIN + SELECT DATE_FORMAT + INTO vDateFormat + FROM CT_MRDS.A_COLUMN_DATE_FORMAT F + WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName + AND F.COLUMN_NAME = 'DEFAULT'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vDateFormat := ENV_MANAGER.gvDefaultDateFormat; + END; + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vDateFormat; + END GET_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS ( + + pTemplateTableName IN VARCHAR2, + pColumnList OUT CLOB, + pFieldList OUT CLOB + ) + IS + vSchemaName VARCHAR2(200); + vTableName VARCHAR2(200); + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vMaxColumnNameLength PLS_INTEGER := 0; + vColType varchar2(200); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnList => '''||nvl(pColumnList, 'NULL')||'''' + ,'pFieldList = '''||nvl(pFieldList, 'NULL')||'''' + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + vSchemaName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.',''); + ENV_MANAGER.LOG_PROCESS_EVENT('vSchemaName','DEBUG', vSchemaName); + vTableName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'\..*'),'.',''); + ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG', vTableName); + FOR rec IN ( + SELECT + '"'||column_name||'"' as quoted_column_name, + column_name, + data_type, + data_length, + char_length, -- MARS-1056: Added for CHAR/BYTE semantics + char_used, -- MARS-1056: Added for CHAR/BYTE semantics + data_precision, + data_scale, + column_id, + max(length(column_name)+1) over (partition by table_name) as max_column_name_length + FROM all_tab_columns + WHERE table_name = UPPER(vTableName) + AND owner = NVL(UPPER(vSchemaName), USER) + ORDER BY column_id + ) LOOP + -- Build the column_list string + rec.quoted_column_name := rpad(rec.quoted_column_name, rec.max_column_name_length+2, ' '); + + vColType := + CASE + -- MARS-1056: Fixed VARCHAR2 definition logic to preserve CHAR/BYTE semantics + WHEN rec.data_type = 'VARCHAR2' THEN + CASE + WHEN rec.char_used = 'C' THEN + rec.quoted_column_name || ' VARCHAR2(' || rec.char_length || ' CHAR)' + WHEN rec.char_used = 'B' THEN + rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ' BYTE)' + ELSE + -- Fallback for NULL char_used (should not occur but handle gracefully) + rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ')' + END + -- Other character types (preserve original logic) + WHEN rec.data_type IN ('CHAR', 'NCHAR', 'NVARCHAR2') THEN + rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')' + WHEN rec.data_type = 'NUMBER' THEN + rec.quoted_column_name || ' ' || rec.data_type || + CASE + WHEN rec.data_precision IS NOT NULL AND rec.data_scale IS NOT NULL THEN + '(' || rec.data_precision || ',' || rec.data_scale || ')' + WHEN rec.data_precision IS NOT NULL THEN + '(' || rec.data_precision || ')' + ELSE + '' + END + WHEN rec.data_type IN ('RAW') THEN + rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')' + WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') IN ('DATE', 'TIMESTAMP') THEN + rec.quoted_column_name || ' ' || rec.data_type + ELSE + rec.quoted_column_name || ' ' || rec.data_type + END; + pColumnList := pColumnList ||vColType ||cgBL|| ','; + -- Build the field_list string + -- Note: field_list uses CHAR() for CSV field definitions - this is correct behavior + pFieldList := pFieldList || + CASE + WHEN rec.data_type = 'DATE' THEN + -- MARS-1046: DATE format - wrap with NORMALIZE_DATE_FORMAT to fix ISO 8601 'T' separator + rec.quoted_column_name || ' DATE ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN rec.data_type LIKE 'TIMESTAMP%WITH TIME ZONE' THEN + -- MARS-1046: TIMESTAMP WITH TIME ZONE format for ISO 8601 with fractional seconds and timezone + -- Syntax: column_name CHAR(length) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK "format" + -- Use fixed length of 50 for ISO 8601 format (e.g., "2012-03-02T14:16:23.798+01:00" = 29 chars) + rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN + -- Other TIMESTAMP types (without timezone) + -- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format') + rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) + WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN + -- For CSV field definitions, use data_length for CHAR() specification + rec.quoted_column_name || ' CHAR(' || rec.data_length || ')' + ELSE + rec.quoted_column_name + END ||cgBL|| ','; + + + END LOOP; + + -- Remove the trailing comma and space from the strings + pColumnList := ' '||RTRIM(pColumnList, ','); + pFieldList := ' '||RTRIM(pFieldList, ','); + ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', pColumnList); + -- TO_DO !!! + -- Add check if pColumnList/pFieldList is empty or not + -- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + + -- Output the generated column_list and field_list + ENV_MANAGER.LOG_PROCESS_EVENT('column_list' ,'DEBUG',pColumnList); + ENV_MANAGER.LOG_PROCESS_EVENT('field_list' ,'DEBUG',pFieldList); + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + + EXCEPTION +-- WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN +-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); +-- RAISE_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GENERATE_EXTERNAL_TABLE_PARAMS; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_SOURCE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE, + pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + INSERT INTO CT_MRDS.A_SOURCE(A_SOURCE_KEY, SOURCE_NAME) VALUES (pSourceKey, pSourceName); + COMMIT; + EXCEPTION + WHEN DUP_VAL_ON_INDEX THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ADD_SOURCE; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE DELETE_SOURCE_CASCADE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vSharedTemplateCount PLS_INTEGER := 0; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceKey => '''||nvl(pSourceKey, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- First pass: Delete container files (those that have CONTAINER_FILE_KEY set) + for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG + where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NOT NULL + ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop + -- Delete processed file records + delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for container config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + + -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats + IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN + SELECT COUNT(DISTINCT A_SOURCE_KEY) + INTO vSharedTemplateCount + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME + AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted + + -- Only delete date formats if template table is not shared with other sources + IF vSharedTemplateCount = 0 THEN + delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); + END IF; + END IF; + + -- Delete container file configuration + delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted container A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + end loop; + COMMIT; -- Commit container deletions + + -- Second pass: Delete parent files (those that do NOT have CONTAINER_FILE_KEY set) + for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG + where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NULL + ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop + -- Delete processed file records + delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for parent config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + + -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats + IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN + SELECT COUNT(DISTINCT A_SOURCE_KEY) + INTO vSharedTemplateCount + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME + AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted + + -- Only delete date formats if template table is not shared with other sources + IF vSharedTemplateCount = 0 THEN + delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); + END IF; + END IF; + + -- Delete parent file configuration + delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted parent A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); + end loop; + COMMIT; -- Commit parent deletions + + -- Delete source system record + DELETE FROM CT_MRDS.A_SOURCE where A_SOURCE_KEY = pSourceKey; + ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE record for source key: '||pSourceKey,'DEBUG', vParameters); + COMMIT; -- Final commit for source deletion + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END DELETE_SOURCE_CASCADE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY ( + pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ) RETURN PLS_INTEGER + IS + vSourceFileConfigKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileId => '||nvl(to_char(pSourceFileId), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT A_SOURCE_FILE_CONFIG_KEY + INTO vSourceFileConfigKey + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_ID = pSourceFileId + AND SOURCE_FILE_TYPE = 'CONTAINER'; + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vSourceFileConfigKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG := 'No match in A_SOURCE_FILE_CONFIG where SOURCE_FILE_TYPE=''CONTAINER'' and SOURCE_FILE_ID = '''||pSourceFileId||''''; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG, 'WARNING', vParameters); + RETURN NULL; + WHEN TOO_MANY_ROWS THEN + ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES := 'GET_CONTAINER_SOURCE_FILE_CONFIG_KEY: Multiple SOURCE_FILE_TYPE=''CONTAINER'' matches for SOURCE_FILE_ID: '||pSourceFileId||' in A_SOURCE_FILE_CONFIG'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END GET_CONTAINER_SOURCE_FILE_CONFIG_KEY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_SOURCE_FILE_CONFIG_KEY ( + pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT' + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE + ) RETURN PLS_INTEGER + IS + vSourceFileConfigKey PLS_INTEGER; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileType => '''||nvl(pSourceFileType, 'NULL')||'''' + ,'pSourceFileId => '''||nvl(pSourceFileId, 'NULL')||'''' + ,'pTableId => '''||nvl(pTableId, 'NULL')||'''')); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT A_SOURCE_FILE_CONFIG_KEY + INTO vSourceFileConfigKey + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE SOURCE_FILE_TYPE = pSourceFileType + AND SOURCE_FILE_ID = pSourceFileId + AND TABLE_ID = pTableId; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG', vParameters); + RETURN vSourceFileConfigKey; + + EXCEPTION + WHEN NO_DATA_FOUND THEN + RETURN NULL; + WHEN TOO_MANY_ROWS THEN + vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE + ||cgBL||' '||'GET_SOURCE_FILE_CONFIG_KEY: Multiple matches in A_SOURCE_FILE_CONFIG table WHERE' + ||cgBL||' '||'SOURCE_FILE_TYPE: '||pSourceFileType + ||cgBL||' '||'SOURCE_FILE_ID: '||pSourceFileId + ||cgBL||' '||'TABLE_ID: '||pTableId; + ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, vgMsgTmp); + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); + + END GET_SOURCE_FILE_CONFIG_KEY; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_SOURCE_FILE_CONFIG ( + pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE + ,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE + ,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL + ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL + ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ) IS + vSourceFileConfigKey PLS_INTEGER; + vSourceKeyExists PLS_INTEGER := 0; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => '''||nvl(to_char(pSourceKey), 'NULL')||'''' + ,'pSourceFileType => '''||nvl(to_char(pSourceFileType), 'NULL')||'''' + ,'pSourceFileId => '''||nvl(to_char(pSourceFileId), 'NULL')||'''' + ,'pSourceFileDesc => '''||nvl(to_char(pSourceFileDesc), 'NULL')||'''' + ,'pSourceFileNamePattern => '''||nvl(to_char(pSourceFileNamePattern), 'NULL')||'''' + ,'pTableId => '''||nvl(to_char(pTableId), 'NULL')||'''' + ,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||'''' + ,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||'''' + ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING) + VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding); + COMMIT; + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + + WHEN OTHERS THEN + IF SQLCODE = -2291 THEN + ENV_MANAGER.MSG_MISSING_SOURCE_KEY := 'The Source with A_SOURCE_KEY: '''||pSourceKey||''' not found in parent table A_SOURCE.' + ||cgBL||'First add a record to A_SOURCE:' + ||cgBL||'call file_manager.add_source(pSourceKey => '''||pSourceKey||''', pSourceName => ''...'')'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_SOURCE_KEY, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END IF; + + END ADD_SOURCE_FILE_CONFIG; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ADD_COLUMN_DATE_FORMAT ( + pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE + ,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE + ,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' + ,'pColumnName => '''||nvl(pColumnName, 'NULL')||'''' + ,'pDateFormat => '''||nvl(pDateFormat, 'NULL')||'''' + )); + + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + INSERT INTO CT_MRDS.A_COLUMN_DATE_FORMAT(TEMPLATE_TABLE_NAME, COLUMN_NAME, DATE_FORMAT) + VALUES (pTemplateTableName, pColumnName, pDateFormat); + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END ADD_COLUMN_DATE_FORMAT; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2) + RETURN VARCHAR2 + IS + BEGIN + CASE pBucketArea + WHEN 'INBOX' THEN RETURN ENV_MANAGER.gvInboxBucketUri; + WHEN 'ODS' THEN RETURN ENV_MANAGER.gvDataBucketUri; + WHEN 'DATA' THEN RETURN ENV_MANAGER.gvDataBucketUri; + WHEN 'ARCHIVE' THEN RETURN ENV_MANAGER.gvArchiveBucketUri; + ELSE + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_INVALID_BUCKET_AREA, + ENV_MANAGER.MSG_INVALID_BUCKET_AREA || ' Provided: ''' || pBucketArea || ''''); + END CASE; + END; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Configuration entry + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vContainerFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vCount PLS_INTEGER := 0; + addColHeader BOOLEAN := TRUE; + vMsgTmp VARCHAR2(32000):= ''; + CURSOR cColumnFormat(vTableName A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE) IS + SELECT * + FROM CT_MRDS.A_COLUMN_DATE_FORMAT + WHERE TEMPLATE_TABLE_NAME = vTableName; + vColumnDateFormat A_COLUMN_DATE_FORMAT%ROWTYPE; + + FUNCTION FORMAT_CONFIG( pSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE, + pTypeInfo VARCHAR2 DEFAULT 'File Configuration', + pLevel VARCHAR2 DEFAULT '') + RETURN VARCHAR2 + IS + vMsg VARCHAR2(9999); + BEGIN + vMsg := '' + ||cgBL||pLevel||''||'Details about '||pTypeInfo||':' + ||cgBL||pLevel||''||'--------------------------------' + ||cgBL||pLevel||'A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||pLevel||'A_SOURCE_KEY = '||pSourceFileConfig.A_SOURCE_KEY + ||cgBL||pLevel||'SOURCE_FILE_TYPE = '||pSourceFileConfig.SOURCE_FILE_TYPE + ||cgBL||pLevel||'SOURCE_FILE_ID = '||pSourceFileConfig.SOURCE_FILE_ID + ||cgBL||pLevel||'SOURCE_FILE_DESC = '||pSourceFileConfig.SOURCE_FILE_DESC + ||cgBL||pLevel||'SOURCE_FILE_NAME_PATTERN = '||pSourceFileConfig.SOURCE_FILE_NAME_PATTERN + ||cgBL||pLevel||'TABLE_ID = '||pSourceFileConfig.TABLE_ID + ||cgBL||pLevel||'TEMPLATE_TABLE_NAME = '||pSourceFileConfig.TEMPLATE_TABLE_NAME + ||cgBL||pLevel||'CONTAINER_FILE_KEY = '||pSourceFileConfig.CONTAINER_FILE_KEY + ||cgBL||pLevel||'ODS_SCHEMA_NAME = '||pSourceFileConfig.ODS_SCHEMA_NAME + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_DAYS = '||pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_FILES_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM + ||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT + ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS + + ||cgBL||pLevel||''||'--------------------------------' + ; + RETURN vMsg; + END FORMAT_CONFIG; + + BEGIN + vMsgTmp := ''; + -- Get Main Config + BEGIN + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + vMsgTmp := FORMAT_CONFIG(pSourceFileConfig => vSourceFileConfig, pTypeInfo => 'File Configuration'); + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no config entry in A_SOURCE_FILE_CONFIG where A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfigKey; + RETURN vMsgTmp; + END; + + -- Get Container Config + IF pIncludeContainerInfo > 0 THEN + BEGIN + SELECT * + INTO vContainerFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_FILE_CONFIG_KEY = vSourceFileConfig.CONTAINER_FILE_KEY; + + vMsgTmp := vMsgTmp || cgBL || FORMAT_CONFIG(pSourceFileConfig => vContainerFileConfig, pTypeInfo => 'related Container Config', pLevel => ' '); + + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := vMsgTmp|| cgBL || 'There is no CONTAINER config entry in A_SOURCE_FILE_CONFIG.'; + -- RETURN vMsgTmp; + END; + END IF; + + -- Get Column Date Format Config + IF pIncludeColumnFormatInfo > 0 THEN + BEGIN + OPEN cColumnFormat(vTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME); + LOOP + FETCH cColumnFormat INTO vColumnDateFormat; + IF cColumnFormat%FOUND AND addColHeader THEN + vCount := 1; + vMsgTmp := vMsgTmp||cgBL|| cgBL || ' Column Date Format config entries:'; + vMsgTmp := vMsgTmp||cgBL||''||' --------------------------------'; + addColHeader := FALSE; + END IF; + EXIT WHEN cColumnFormat%NOTFOUND; + vMsgTmp := vMsgTmp + ||cgBL||' TEMPLATE_TABLE_NAME = '||vColumnDateFormat.TEMPLATE_TABLE_NAME + ||cgBL||' COLUMN_NAME = '||vColumnDateFormat.COLUMN_NAME + ||cgBL||' DATE_FORMAT = '||vColumnDateFormat.DATE_FORMAT + ||cgBL||''||' --------------------------------'; + END LOOP; + If vCount=0 THEN + vMsgTmp := vMsgTmp || cgBL || 'There is no Column Date Format config entries in A_COLUMN_DATE_FORMAT where TEMPLATE_TABLE_NAME = '||NVL(vSourceFileConfig.TEMPLATE_TABLE_NAME,'NULL'); + END IF; + CLOSE cColumnFormat; + END; + END IF; + + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_SOURCE_FILE_CONFIG_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE + ,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Received entry + IS + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vMsgTmp VARCHAR2(32000):= ''; + BEGIN + + BEGIN + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + vMsgTmp := '' + ||cgBL||''||'Details about received file:' + ||cgBL||''||'--------------------------------' + ||cgBL||'A_SOURCE_FILE_RECEIVED_KEY = '||vSourceFileReceived.A_SOURCE_FILE_RECEIVED_KEY + ||cgBL||'A_SOURCE_FILE_CONFIG_KEY = '||vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY + ||cgBL||'SOURCE_FILE_NAME = '||vSourceFileReceived.SOURCE_FILE_NAME + ||cgBL||'CHECKSUM = '||vSourceFileReceived.CHECKSUM + ||cgBL||'CREATED = '||vSourceFileReceived.CREATED + ||cgBL||'BYTES = '||vSourceFileReceived.BYTES + ||cgBL||'RECEPTION_DATE = '||vSourceFileReceived.RECEPTION_DATE + ||cgBL||'PROCESSING_STATUS = '||vSourceFileReceived.PROCESSING_STATUS + ||cgBL||'EXTERNAL_TABLE_NAME = '||vSourceFileReceived.EXTERNAL_TABLE_NAME + ||cgBL||'PARTITION_YEAR = '||vSourceFileReceived.PARTITION_YEAR + ||cgBL||'PARTITION_MONTH = '||vSourceFileReceived.PARTITION_MONTH + ||cgBL||''||'--------------------------------' + ; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no data in A_SOURCE_FILE_RECEIVED where A_SOURCE_FILE_RECEIVED_KEY = '||pSourceFileReceivedKey; + RETURN vMsgTmp; + END; + + IF pIncludeConfigInfo>0 THEN + vMsgTmp := vMsgTmp || cgBL || CT_MRDS.FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO(vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY,pIncludeContainerInfo,pIncludeColumnFormatInfo); + END IF; + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_SOURCE_FILE_RECEIVED_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_DET_USER_LOAD_OPERATIONS ( + pOperationId PLS_INTEGER + ) RETURN VARCHAR2 + ---- + -- Function to get info about File Received entry + IS + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vMsgTmp VARCHAR2(32000):= ''; + BEGIN + + BEGIN + SELECT * + INTO vUserLoadOperations + FROM USER_LOAD_OPERATIONS + WHERE id = pOperationId; + + vMsgTmp := '' + ||''||'Details about USER_LOAD_OPERATIONS where ID = '||pOperationId + ||cgBL||''||'--------------------------------' + ||cgBL||'ID = '||vUserLoadOperations.ID + ||cgBL||'TYPE = '||vUserLoadOperations.TYPE + ||cgBL||'SID = '||vUserLoadOperations.SID + ||cgBL||'SERIAL# = '||vUserLoadOperations.SERIAL# + ||cgBL||'START_TIME = '||vUserLoadOperations.START_TIME + ||cgBL||'UPDATE_TIME = '||vUserLoadOperations.UPDATE_TIME + ||cgBL||'STATUS = '||vUserLoadOperations.STATUS + ||cgBL||'OWNER_NAME = '||vUserLoadOperations.OWNER_NAME + ||cgBL||'TABLE_NAME = '||vUserLoadOperations.TABLE_NAME + ||cgBL||'PARTITION_NAME = '||vUserLoadOperations.PARTITION_NAME + ||cgBL||'SUBPARTITION_NAME = '||vUserLoadOperations.SUBPARTITION_NAME + ||cgBL||'FILE_URI_LIST = '||vUserLoadOperations.FILE_URI_LIST + ||cgBL||'ROWS_LOADED = '||vUserLoadOperations.ROWS_LOADED + ||cgBL||'LOGFILE_TABLE = '||vUserLoadOperations.LOGFILE_TABLE + ||cgBL||'BADFILE_TABLE = '||vUserLoadOperations.BADFILE_TABLE + ||cgBL||'STATUS_TABLE = '||vUserLoadOperations.STATUS_TABLE + ||cgBL||'TEMPEXT_TABLE = '||vUserLoadOperations.TEMPEXT_TABLE + ||cgBL||'CREDENTIAL_NAME = '||vUserLoadOperations.CREDENTIAL_NAME + ||cgBL||'EXPIRATION_TIME = '||vUserLoadOperations.EXPIRATION_TIME + ||cgBL||''||'--------------------------------' + ; + EXCEPTION + WHEN NO_DATA_FOUND THEN + vMsgTmp := 'There is no data in USER_LOAD_OPERATIONS where ID = '||pOperationId; + RETURN vMsgTmp; + END; + + RETURN vMsgTmp; + EXCEPTION + + WHEN OTHERS THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END GET_DET_USER_LOAD_OPERATIONS; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ANALYZE_VALIDATION_ERRORS( + pSourceFileReceivedKey IN NUMBER + ) RETURN VARCHAR2 + IS + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vValidationLogTable VARCHAR2(128); + vTemplateSchema VARCHAR2(128); + vTemplateTable VARCHAR2(128); + vCsvFileUri VARCHAR2(4000); + vParameters VARCHAR2(4000); + vResult VARCHAR2(32000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => ' || pSourceFileReceivedKey + )); + ENV_MANAGER.LOG_PROCESS_EVENT('Start', 'DEBUG', vParameters); + + -- Get file and config information + BEGIN + -- Get source file received data first + SELECT * + INTO vSourceFileReceived + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr + WHERE sfr.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; + + -- Get source file config data + SELECT * + INTO vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc + WHERE sfc.A_SOURCE_FILE_CONFIG_KEY = vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY; + EXCEPTION + WHEN NO_DATA_FOUND THEN + ENV_MANAGER.LOG_PROCESS_ERROR('Source file or config not found for key: ' || pSourceFileReceivedKey, vParameters); + RETURN 'Error: Source file with key ' || pSourceFileReceivedKey || ' not found in A_SOURCE_FILE_RECEIVED table'; + END; + + -- Extract template schema and table from template table name + vTemplateSchema := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '^([^.]+)'); + vTemplateTable := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '\.(.+)$', 1, 1, NULL, 1); + + -- Build CSV file URI + vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || + 'INBOX/' || vSourceFileConfig.A_SOURCE_KEY || '/' || + vSourceFileConfig.SOURCE_FILE_ID || '/' || + vSourceFileConfig.TABLE_ID || '/' || + vSourceFileReceived.SOURCE_FILE_NAME; + + -- Find validation log table (most recent one with errors) + -- Look for validation log tables related to this external table + BEGIN + SELECT table_name + INTO vValidationLogTable + FROM ( + SELECT table_name, + REGEXP_SUBSTR(table_name, '\$([0-9]+)_', 1, 1, NULL, 1) as log_number + FROM USER_TABLES + WHERE table_name LIKE 'VALIDATE$%_LOG' + ORDER BY TO_NUMBER(log_number) DESC + ) + WHERE ROWNUM = 1; + EXCEPTION + WHEN NO_DATA_FOUND THEN + -- If no validation log tables found, use a default name + vValidationLogTable := 'VALIDATE$999_LOG'; + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('Calling ENV_MANAGER.ANALYZE_VALIDATION_ERRORS with parameters: ' || + 'LogTable=' || vValidationLogTable || + ', Schema=' || vTemplateSchema || + ', Table=' || vTemplateTable || + ', URI=' || SUBSTR(vCsvFileUri, 1, 100) || '...', 'DEBUG', vParameters); + + -- Call the main function with derived parameters + vResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( + pValidationLogTable => vValidationLogTable, + pTemplateSchema => vTemplateSchema, + pTemplateTable => vTemplateTable, + pCsvFileUri => vCsvFileUri + ); + + ENV_MANAGER.LOG_PROCESS_EVENT('End', 'DEBUG', vParameters); + RETURN vResult; + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_ERROR('Error in ANALYZE_VALIDATION_ERRORS: ' || SQLERRM, vParameters); + RETURN 'Error analyzing validation errors: ' || SQLERRM || + cgBL || 'Source File Key: ' || pSourceFileReceivedKey || + cgBL || 'Check A_SOURCE_FILE_RECEIVED and A_SOURCE_FILE_CONFIG tables for data integrity.'; + END ANALYZE_VALIDATION_ERRORS; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_MANAGER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_MANAGER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkg b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkg new file mode 100644 index 0000000..a820802 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkg @@ -0,0 +1,639 @@ +create or replace PACKAGE CT_MRDS.FILE_MANAGER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) || + '3.3.2 (2026-02-20): MARS-828 - Fixed threshold column names in GET_DET_SOURCE_FILE_CONFIG_INFO for MARS-828 compatibility' || CHR(13)||CHR(10) || + '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) || + '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) || + '3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) || + '3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) || + '3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) || + '2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) || + '2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-01): Initial release with file processing and validation capabilities'; + + TYPE tSourceFileReceived IS RECORD + ( + A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE, + A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE, + SOURCE_FILE_PREFIX_INBOX VARCHAR2(430), + SOURCE_FILE_PREFIX_ODS VARCHAR2(430), + SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430), + SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430), + SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE, + RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE, + PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE, + EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE + ); + + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + vgSourceFileConfigKey PLS_INTEGER; + vgMsgTmp VARCHAR2(32000); + + + + --------------------------------------------------------------------------------------------------------------------------- + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_SOURCE_FILE_CONFIG + * @desc Get source file type by matching the source file name against source file type naming patterns + * or by specifying the id of a received source file. + * @example ... + * @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE" + **/ + FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL + , pSourceFileReceivedKey IN NUMBER DEFAULT NULL + , pSourceFileConfigKey IN NUMBER DEFAULT NULL) + RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + + + + /** + * @name REGISTER_SOURCE_FILE_RECEIVED + * @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table. + * This overload automatically determines source file type from the file name. + * It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record. + * @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + * @ex_rslt 3245 + **/ + FUNCTION REGISTER_SOURCE_FILE_RECEIVED ( + pSourceFileReceivedName IN VARCHAR2 + ) + RETURN PLS_INTEGER; + + + + /** + * @name REGISTER_SOURCE_FILE_RECEIVED + * @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig. + * Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record. + * @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED( + * pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv' + * ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... ); + * @ex_rslt 3245 + **/ + FUNCTION REGISTER_SOURCE_FILE_RECEIVED ( + pSourceFileReceivedName IN VARCHAR2, + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) + RETURN PLS_INTEGER; + + + + /** + * @name SET_SOURCE_FILE_RECEIVED_STATUS + * @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column + * based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY + * and provided value of pStatus parameter + * @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION'); + **/ + PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS( + pSourceFileReceivedKey IN PLS_INTEGER, + pStatus IN VARCHAR2 + ); + + + + /** + * @name GET_EXTERNAL_TABLE_COLUMNS + * @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name. + * It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure. + * @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual; + * @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE, + * "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE, + * "REV_NUMBER" NUMBER(28,0), + * "REF_DATE" DATE, + * "FREE_TEXT" VARCHAR2(1000 CHAR), + * "MLF_BS_TOTAL" NUMBER(28,10), + * "DF_BS_TOTAL" NUMBER(28,10), + * "MLF_SF_TOTAL" NUMBER(28,10), + * "DF_SF_TOTAL" NUMBER(28,10) + **/ + FUNCTION GET_EXTERNAL_TABLE_COLUMNS ( + pTargetTableTemplate IN VARCHAR2 + ) + RETURN CLOB; + + + + /** + * @name CREATE_EXTERNAL_TABLE + * @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table + * MARS-1049: Added pEncoding parameter for CSV character set specification + * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252') + * If provided, adds CHARACTERSET clause to external table definition + * @example + * begin + * FILE_MANAGER.CREATE_EXTERNAL_TABLE( + * pTableName => 'STANDING_FACILITIES_HEADER', + * pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER', + * pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/', + * pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/', + * pFileName => NULL, + * pDelimiter => ',', + * pEncoding => 'UTF8' + * ); + * end; + **/ + PROCEDURE CREATE_EXTERNAL_TABLE ( + pTableName IN VARCHAR2, + pTemplateTableName IN VARCHAR2, + pPrefix IN VARCHAR2, + pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri, + pFileName IN VARCHAR2 DEFAULT NULL, + pDelimiter IN VARCHAR2 DEFAULT ',', + pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ); + + + + /** + * @name CREATE_EXTERNAL_TABLE + * @desc Creates External Table for single file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);; + **/ + PROCEDURE CREATE_EXTERNAL_TABLE ( + pSourceFileReceivedKey IN NUMBER + ); + + + + /** + * @name VALIDATE_SOURCE_FILE_RECEIVED + * @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE + * It validate External table build upon single file + * provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377); + **/ + PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED + ( + pSourceFileReceivedKey IN NUMBER + ); + + + /** + * @name VALIDATE_EXTERNAL_TABLE + * @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE. + * It validates External Table provided by parameter pTableName. + * It returns: PASSED or FAILED. + * @example + * declare + * vStatus VARCHAR2(100); + * begin + * vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER'); + * DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus); + * end; + * + * @ex_rslt FAILED + **/ + FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2; + + + /** + * @name S_VALIDATE_EXTERNAL_TABLE + * @desc A function which checks if SELECT query reterns any rows. + * It trys to selects External Table provided by parameter pTableName. + * It returns: PASSED or FAILED. + * @example + * declare + * vStatus VARCHAR2(100); + * begin + * vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER'); + * DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus); + * end; + * + * @ex_rslt PASSED + **/ + FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) + RETURN VARCHAR2; + + + + /** + * @name DROP_EXTERNAL_TABLE + * @desc It drops External Table for single file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377); + **/ + PROCEDURE DROP_EXTERNAL_TABLE ( + pSourceFileReceivedKey IN NUMBER + ); + + + + /** + * @name COPY_FILE + * @desc It copies file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * into destination provided by pDestination parameter. + * pDestination parameter allowed values are: 'ODS' + * @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS'); + **/ + PROCEDURE COPY_FILE( + pSourceFileReceivedKey IN NUMBER, + pDestination IN VARCHAR2 + ); + + + + + /** + * @name MOVE_FILE + * @desc It moves file provided by + * pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY) + * into destination provided by pDestination parameter. + * pDestination parameter allowed values are: 'ODS', 'QUARANTINE' + * @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS'); + **/ + PROCEDURE MOVE_FILE( + pSourceFileReceivedKey IN NUMBER, + pDestination IN VARCHAR2 + ); + + + + /** + * @name DELETE_FOLDER_CONTENTS + * @desc It deletes all files from specified folder in the cloud storage. + * The procedure lists all objects in the specified folder prefix and deletes them one by one. + * pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE' + * pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/') + * @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/'); + **/ + PROCEDURE DELETE_FOLDER_CONTENTS( + pBucketArea IN VARCHAR2, + pFolderPrefix IN VARCHAR2 + ); + + + + + /** + * @name PROCESS_SOURCE_FILE + * @desc It process file provided by pSourceFileReceivedName parameter. + * Ubmrella procedure that calls: + * - REGISTER_SOURCE_FILE_RECEIVED; + * - CREATE_EXTERNAL_TABLE; + * - VALIDATE_SOURCE_FILE_RECEIVED; + * - DROP_EXTERNAL_TABLE; + * - MOVE_FILE; + * @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + **/ + PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + ; + + + + /** + * @name PROCESS_SOURCE_FILE + * @desc It process file provided by pSourceFileReceivedName parameter and return processing result value. + * It returns (success/failure) => 0 / -(value). + * Ubmrella function that calls PROCESS_SOURCE_FILE procedure. + * @example + * declare + * vResult PLS_INTEGER; + * begin + * vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'); + * DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult); + * end; + * @ex_rslt 0 + * -20021 + **/ + FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) + RETURN PLS_INTEGER; + + + + /** + * @name GET_DATE_FORMAT + * @desc Returns date format for specified template table name and column name. + * Date is taken from configuration A_COLUMN_DATE_FORMAT table. + * @example select FILE_MANAGER.GET_DATE_FORMAT( + * pTemplateTableName => 'STANDING_FACILITIES_HEADER', + * pColumnName => 'SNAPSHOT_DATE') + * from dual; + * @ex_rslt DD/MM/YYYY HH24:MI:SS + **/ + FUNCTION GET_DATE_FORMAT( + pTemplateTableName IN VARCHAR2, + pColumnName IN VARCHAR2 + ) RETURN VARCHAR2; + + + + /** + * @name GENERATE_EXTERNAL_TABLE_PARAMS + * @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName. + * @example + * declare + * vColumnList CLOB; + * vFieldList CLOB; + * begin + * FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS ( + * pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER' + * ,pColumnList => vColumnList + * ,pFieldList => vFieldList + * ); + * DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList); + * DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList); + * end; + * / + **/ + PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS ( + pTemplateTableName IN VARCHAR2, + pColumnList OUT CLOB, + pFieldList OUT CLOB + ); + + + + + + /** + * @name ADD_SOURCE + * @desc Insert a new record to A_SOURCE table. + * pSourceKey is a PRIMARY KEY value. + **/ + PROCEDURE ADD_SOURCE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE, + pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE + ); + + + + /** + * @name DELETE_SOURCE_CASCADE + * @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables: + * - A_SOURCE_FILE_CONFIG + * - A_SOURCE_FILE_RECEIVED + * - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems) + * The procedure checks if template tables are shared before deleting date format configurations. + * If a template table is used by multiple source systems, date formats are preserved. + * @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS'); + **/ + PROCEDURE DELETE_SOURCE_CASCADE ( + pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE + ); + + + + /** + * @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY + * @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID) + * it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record. + * @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY( + * pSourceFileId => 'UC_DISSEM') + * from dual; + * @ex_rslt 126 + **/ + FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY ( + pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name GET_SOURCE_FILE_CONFIG_KEY + * @desc For specified input parameters, + * it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY. + * @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY ( + * pSourceFileType => 'INPUT' + * ,pSourceFileId => 'UC_DISSEM' + * ,pTableId => 'UC_NMA_DISSEM') + * from dual; + * @ex_rslt 126 + **/ + FUNCTION GET_SOURCE_FILE_CONFIG_KEY ( + pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT' + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name ADD_SOURCE_FILE_CONFIG + * @desc Insert a new record to A_SOURCE_FILE_CONFIG table. + * MARS-1049: Added pEncoding parameter for CSV character set specification. + * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2') + * If NULL, no CHARACTERSET clause is added to external table definitions + * @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( + * pSourceKey => 'C2D', pSourceFileType => 'INPUT', + * pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS', + * pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS', + * pEncoding => 'UTF8' + * ); + **/ + PROCEDURE ADD_SOURCE_FILE_CONFIG ( + pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE + ,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE + ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE + ,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE + ,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE + ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL + ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL + ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ); + + + + /** + * @name ADD_COLUMN_DATE_FORMAT + * @desc Insert a new record to A_COLUMN_DATE_FORMAT table. + **/ + PROCEDURE ADD_COLUMN_DATE_FORMAT ( + pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE + ,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE + ,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE + ); + + + + /** + * @name GET_BUCKET_URI + * @desc Function used to get string with bucket http url. + * Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE' + * @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual; + * @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/ + **/ + FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2) + RETURN VARCHAR2; + + + + /** + * @name GET_DET_SOURCE_FILE_CONFIG_INFO + * @desc Function returns details about A_SOURCE_FILE_CONFIG record + * for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT) + * @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO ( + * pSourceFileConfigKey => 128 + * ,pIncludeContainerInfo => 1 + * ,pIncludeColumnFormatInfo => 1 + * ) from dual; + * @ex_rslt + * Details about File Configuration: + * -------------------------------- + * A_SOURCE_FILE_CONFIG_KEY = 128 + * A_SOURCE_KEY = C2D + * ... + * -------------------------------- + * + * Details about related Container Config: + * -------------------------------- + * A_SOURCE_FILE_CONFIG_KEY = 126 + * A_SOURCE_KEY = C2D + * ... + * -------------------------------- + * + * Column Date Format config entries: + * -------------------------------- + * TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM + * ... + * -------------------------------- + **/ + FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2; + + + + /** + * @name GET_DET_SOURCE_FILE_RECEIVED_INFO + * @desc Function returns details about A_SOURCE_FILE_RECEIVED record + * for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY). + * If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG) + * If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT) + * @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO ( + * pSourceFileReceivedKey => 377 + * ,pIncludeConfigInfo => 1 + * ,pIncludeContainerInfo => 1 + * ,pIncludeColumnFormatInfo => 1 + * ) from dual; + * + **/ + FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE + ,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 + ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 + ) RETURN VARCHAR2; + + + + /** + * @name GET_DET_USER_LOAD_OPERATIONS + * @desc Function returns details from USER_LOAD_OPERATIONS table + * for specified pOperationId. + * @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual; + * @ex_rslt + * Details about USER_LOAD_OPERATIONS where ID = 3608 + * -------------------------------- + * ID = 3608 + * TYPE = VALIDATE + * SID = 31260 + * SERIAL# = 52915 + * START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE + * UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE + * STATUS = FAILED + * OWNER_NAME = CT_MRDS + * TABLE_NAME = STANDING_FACILITIES_HEADER + * PARTITION_NAME = + * SUBPARTITION_NAME = + * FILE_URI_LIST = + * ROWS_LOADED = + * LOGFILE_TABLE = VALIDATE$3608_LOG + * BADFILE_TABLE = VALIDATE$3608_BAD + * STATUS_TABLE = + * TEMPEXT_TABLE = + * CREDENTIAL_NAME = + * EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE + * -------------------------------- + **/ + FUNCTION GET_DET_USER_LOAD_OPERATIONS ( + pOperationId PLS_INTEGER + ) RETURN VARCHAR2; + + /** + * @name ANALYZE_VALIDATION_ERRORS + * @desc Wrapper function that analyzes validation errors for a source file using its received key. + * Automatically derives template schema, table name, CSV URI and validation log table + * from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS. + * @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL; + * @ex_rslt Detailed validation analysis report with column mismatches and solutions + **/ + FUNCTION ANALYZE_VALIDATION_ERRORS( + pSourceFileReceivedKey IN NUMBER + ) RETURN VARCHAR2; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_MANAGER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL; + * @ex_rslt 3.2.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_MANAGER + * Version: 3.2.0 + * Build Date: 2025-10-22 16:30:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_MANAGER Version History: + * 3.2.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ + +/ diff --git a/MARS_Packages/REL02_POST/MARS-1409/track_package_versions.sql b/MARS_Packages/REL02_POST/MARS-1409/track_package_versions.sql new file mode 100644 index 0000000..276daef --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/track_package_versions.sql @@ -0,0 +1,68 @@ +-- ============================================================================ +-- MARS-1409 Package Version Tracking +-- ============================================================================ +-- Purpose: Record package versions in A_PACKAGE_VERSION_HISTORY table +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED + +PROMPT +PROMPT ============================================================================ +PROMPT Recording Package Versions +PROMPT ============================================================================ + +DECLARE + v_file_manager_version VARCHAR2(50); + v_file_manager_build VARCHAR2(100); + v_env_manager_version VARCHAR2(50); + v_env_manager_build VARCHAR2(100); +BEGIN + -- Get FILE_MANAGER version + BEGIN + v_file_manager_version := CT_MRDS.FILE_MANAGER.GET_VERSION(); + v_file_manager_build := CT_MRDS.FILE_MANAGER.GET_BUILD_INFO(); + + DBMS_OUTPUT.PUT_LINE('FILE_MANAGER Version: ' || v_file_manager_version); + DBMS_OUTPUT.PUT_LINE('FILE_MANAGER Build: ' || v_file_manager_build); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Could not retrieve FILE_MANAGER version'); + END; + + -- Get ENV_MANAGER version + BEGIN + v_env_manager_version := CT_MRDS.ENV_MANAGER.GET_VERSION(); + v_env_manager_build := CT_MRDS.ENV_MANAGER.GET_BUILD_INFO(); + + DBMS_OUTPUT.PUT_LINE('ENV_MANAGER Version: ' || v_env_manager_version); + DBMS_OUTPUT.PUT_LINE('ENV_MANAGER Build: ' || v_env_manager_build); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('WARNING: Could not retrieve ENV_MANAGER version'); + END; + + -- Insert version records (if A_PACKAGE_VERSION_HISTORY table exists) + BEGIN + EXECUTE IMMEDIATE 'INSERT INTO CT_MRDS.A_PACKAGE_VERSION_HISTORY + (PACKAGE_NAME, PACKAGE_VERSION, BUILD_INFO, DEPLOYMENT_DATE, DEPLOYMENT_USER, MARS_TICKET) + VALUES (:1, :2, :3, SYSDATE, USER, :4)' + USING 'FILE_MANAGER', v_file_manager_version, v_file_manager_build, 'MARS-1409'; + + EXECUTE IMMEDIATE 'INSERT INTO CT_MRDS.A_PACKAGE_VERSION_HISTORY + (PACKAGE_NAME, PACKAGE_VERSION, BUILD_INFO, DEPLOYMENT_DATE, DEPLOYMENT_USER, MARS_TICKET) + VALUES (:1, :2, :3, SYSDATE, USER, :4)' + USING 'ENV_MANAGER', v_env_manager_version, v_env_manager_build, 'MARS-1409'; + + COMMIT; + DBMS_OUTPUT.PUT_LINE('Package version history recorded successfully'); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('INFO: A_PACKAGE_VERSION_HISTORY table not available - skipping version tracking'); + END; +END; +/ + +PROMPT +PROMPT ============================================================================ +PROMPT Version Tracking Complete +PROMPT ============================================================================ diff --git a/MARS_Packages/REL02_POST/MARS-1409/verify_packages_version.sql b/MARS_Packages/REL02_POST/MARS-1409/verify_packages_version.sql new file mode 100644 index 0000000..bf66c99 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/verify_packages_version.sql @@ -0,0 +1,49 @@ +-- ============================================================================ +-- MARS-1409 Package Version Verification +-- ============================================================================ +-- Purpose: Verify package versions after installation +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK OFF +SET ECHO OFF + +PROMPT +PROMPT ============================================================================ +PROMPT Package Version Verification +PROMPT ============================================================================ + +-- FILE_MANAGER version +PROMPT +PROMPT CT_MRDS.FILE_MANAGER Package: +SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() AS VERSION FROM DUAL; +SELECT CT_MRDS.FILE_MANAGER.GET_BUILD_INFO() AS BUILD_INFO FROM DUAL; + +-- ENV_MANAGER version +PROMPT +PROMPT CT_MRDS.ENV_MANAGER Package: +SELECT CT_MRDS.ENV_MANAGER.GET_VERSION() AS VERSION FROM DUAL; +SELECT CT_MRDS.ENV_MANAGER.GET_BUILD_INFO() AS BUILD_INFO FROM DUAL; + +-- Package compilation status +PROMPT +PROMPT Package Compilation Status: +SELECT object_name, object_type, status, last_ddl_time +FROM user_objects +WHERE object_name IN ('FILE_MANAGER', 'ENV_MANAGER') + AND object_type IN ('PACKAGE', 'PACKAGE BODY') +ORDER BY object_name, object_type; + +-- Check for compilation errors +PROMPT +PROMPT Compilation Errors (if any): +SELECT name, type, line, position, text +FROM user_errors +WHERE name IN ('FILE_MANAGER', 'ENV_MANAGER') +ORDER BY name, type, line, position; + +PROMPT +PROMPT ============================================================================ +PROMPT Verification Complete +PROMPT ============================================================================