admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add FILE_ARCHIVER package and enhance A_SOURCE_FILE_CONFIG table

Browse Source 
- Created new FILE_ARCHIVER package (v3.2.1) for managing file archival processes, including procedures for archiving, restoring, and purging files.
- Added versioning and build information to the package with detailed version history.
- Enhanced A_SOURCE_FILE_CONFIG table by adding ARCHIVE_ENABLED and KEEP_IN_TRASH columns to control archival participation and TRASH retention policy.
- Implemented constraints and comments for new columns to ensure data integrity and provide clarity on their usage.
This commit is contained in:
Grzegorz Michalski
2026-02-11 18:24:18 +01:00
parent b96becd8ef
commit 26aba08759
13 changed files with 1792 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql
View File

@@ -10,15 +10,23 @@ PROMPT ========================================
-- Add new columns -- Add new columns
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD ( ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD (
ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL, ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL,
MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL,
ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
KEEP_IN_TRASH CHAR(1) DEFAULT 'Y' NOT NULL
); );
-- Add check constraint for valid strategies -- Add check constraints
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
CHK_ARCHIVAL_STRATEGY CHECK ( CHK_ARCHIVAL_STRATEGY CHECK (
ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID') ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID')
); );
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N'));
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N'));
-- Add comments -- Add comments
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
'Archival strategy: THRESHOLD_BASED (days), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; 'Archival strategy: THRESHOLD_BASED (days), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
@@ -26,6 +34,12 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)'; 'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0';
-- Verify columns added -- Verify columns added
SELECT SELECT
column_name, column_name,
@@ -36,7 +50,7 @@ SELECT
FROM all_tab_columns FROM all_tab_columns
WHERE owner = 'CT_MRDS' WHERE owner = 'CT_MRDS'
AND table_name = 'A_SOURCE_FILE_CONFIG' AND table_name = 'A_SOURCE_FILE_CONFIG'
AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS') AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'ARCHIVE_ENABLED', 'KEEP_IN_TRASH')
ORDER BY column_id; ORDER BY column_id;
PROMPT ======================================== PROMPT ========================================

36
MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
View File

@@ -1,17 +1,29 @@
-- =================================================================== --=============================================================================================================================
-- MARS-828: Install FILE_ARCHIVER Package Specification v3.2.0 -- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0
-- =================================================================== --=============================================================================================================================
-- Purpose: Deploy updated package specification with version 3.2.0 -- Purpose: Deploy FILE_ARCHIVER Package Specification with FN_ function names
-- Author: Grzegorz Michalski -- Author: Grzegorz Michalski
-- Date: 2026-02-06 -- Date: 2026-02-11
-- Changes: -- Related: MARS-828 Archival Strategy Implementation
-- - Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA --=============================================================================================================================
-- - TRASH folder retention control for safety and compliance
-- =================================================================== SET SERVEROUTPUT ON
PROMPT ========================================================================
PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0
PROMPT ========================================================================
@@new_version/FILE_ARCHIVER.pkg @@new_version/FILE_ARCHIVER.pkg
-- Verify package compilation (check specific schema when installing as ADMIN)
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
FROM ALL_OBJECTS
WHERE OWNER = 'CT_MRDS'
AND OBJECT_NAME = 'FILE_ARCHIVER'
AND OBJECT_TYPE = 'PACKAGE';
PROMPT ======================================== PROMPT SUCCESS: FILE_ARCHIVER Package Specification v3.3.0 installed
PROMPT FILE_ARCHIVER Specification v3.2.0 installed successfully
PROMPT ======================================== --=============================================================================================================================
-- End of Script
--=============================================================================================================================

46
MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
View File

@@ -1,16 +1,38 @@
-- =================================================================== --=============================================================================================================================
-- MARS-828: Install FILE_ARCHIVER Package Body v3.2.0 -- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Body v3.3.0
-- =================================================================== --=============================================================================================================================
-- Purpose: Deploy updated package body with TRASH folder retention control -- Purpose: Deploy FILE_ARCHIVER Package Body with config-based archival and FN_ function names
-- Author: Grzegorz Michalski -- Author: Grzegorz Michalski
-- Date: 2026-02-06 -- Date: 2026-02-11
-- Changes: -- Related: MARS-828 Archival Strategy Implementation
-- - v3.0.0: Added GET_ARCHIVAL_WHERE_CLAUSE for flexible archival strategies --=============================================================================================================================
-- - v3.1.0: Added function overloads for Python integration
-- - v3.1.1: Fixed ORA-01422 for multiple parquet files (directory prefix storage) SET SERVEROUTPUT ON
-- - v3.1.2: Fixed PARTITION_YEAR/PARTITION_MONTH assignments and circular dependency
-- - v3.2.0: Added pKeepInTrash parameter for TRASH folder retention control PROMPT ========================================================================
-- =================================================================== PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Body v3.3.0
PROMPT ========================================================================
@@new_version/FILE_ARCHIVER.pkb @@new_version/FILE_ARCHIVER.pkb
-- Verify package compilation (check specific schema when installing as ADMIN)
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
FROM ALL_OBJECTS
WHERE OWNER = 'CT_MRDS'
AND OBJECT_NAME = 'FILE_ARCHIVER'
AND OBJECT_TYPE IN ('PACKAGE', 'PACKAGE BODY')
ORDER BY OBJECT_TYPE;
-- Check for any compilation errors
SELECT 'COMPILATION ERRORS FOUND' AS WARNING
FROM ALL_ERRORS
WHERE OWNER = 'CT_MRDS'
AND NAME = 'FILE_ARCHIVER'
AND TYPE = 'PACKAGE BODY'
AND ROWNUM = 1;
PROMPT SUCCESS: FILE_ARCHIVER Package Body v3.3.0 installed
--=============================================================================================================================
-- End of Script
--=============================================================================================================================

30
MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql
View File

@@ -60,7 +60,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10,
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824, -- 1 GB BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824, -- 1 GB
HOURS_TO_EXPIRE_STATISTICS = 24 HOURS_TO_EXPIRE_STATISTICS = 24,
ARCHIVE_ENABLED = 'Y', -- Enable archival for all LM tables
KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND A_SOURCE_KEY = 'LM' AND A_SOURCE_KEY = 'LM'
AND TABLE_ID IN ( AND TABLE_ID IN (
@@ -104,7 +106,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 5, FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 5,
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912, -- 512 MB BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912, -- 512 MB
HOURS_TO_EXPIRE_STATISTICS = 48 HOURS_TO_EXPIRE_STATISTICS = 48,
ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB DEBT tables
KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND A_SOURCE_KEY = 'CSDB' AND A_SOURCE_KEY = 'CSDB'
AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY'); AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY');
@@ -128,7 +132,9 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10,
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456, -- 256 MB BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456, -- 256 MB
HOURS_TO_EXPIRE_STATISTICS = 72 HOURS_TO_EXPIRE_STATISTICS = 72,
ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB rating/description tables
KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND A_SOURCE_KEY = 'CSDB' AND A_SOURCE_KEY = 'CSDB'
AND TABLE_ID IN ( AND TABLE_ID IN (
@@ -168,6 +174,8 @@ SELECT
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR,
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
ARCHIVE_ENABLED,
KEEP_IN_TRASH,
CASE CASE
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
AND MINIMUM_AGE_MONTHS = 0 AND MINIMUM_AGE_MONTHS = 0
@@ -175,6 +183,8 @@ SELECT
AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000 AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000
AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824
AND HOURS_TO_EXPIRE_STATISTICS = 24 AND HOURS_TO_EXPIRE_STATISTICS = 24
AND ARCHIVE_ENABLED = 'Y'
AND KEEP_IN_TRASH = 'N'
THEN 'OK' THEN 'OK'
ELSE 'ERROR' ELSE 'ERROR'
END AS STATUS END AS STATUS
@@ -196,6 +206,8 @@ SELECT
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR,
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
ARCHIVE_ENABLED,
KEEP_IN_TRASH,
CASE CASE
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
AND MINIMUM_AGE_MONTHS = 6 AND MINIMUM_AGE_MONTHS = 6
@@ -203,6 +215,8 @@ SELECT
AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000 AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000
AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912
AND HOURS_TO_EXPIRE_STATISTICS = 48 AND HOURS_TO_EXPIRE_STATISTICS = 48
AND ARCHIVE_ENABLED = 'Y'
AND KEEP_IN_TRASH = 'N'
THEN 'OK' THEN 'OK'
ELSE 'ERROR' ELSE 'ERROR'
END AS STATUS END AS STATUS
@@ -224,6 +238,8 @@ SELECT
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR,
BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR,
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
ARCHIVE_ENABLED,
KEEP_IN_TRASH,
CASE CASE
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
AND MINIMUM_AGE_MONTHS = 0 AND MINIMUM_AGE_MONTHS = 0
@@ -231,6 +247,8 @@ SELECT
AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000 AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000
AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456 AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456
AND HOURS_TO_EXPIRE_STATISTICS = 72 AND HOURS_TO_EXPIRE_STATISTICS = 72
AND ARCHIVE_ENABLED = 'Y'
AND KEEP_IN_TRASH = 'N'
THEN 'OK' THEN 'OK'
ELSE 'ERROR' ELSE 'ERROR'
END AS STATUS END AS STATUS
@@ -252,7 +270,9 @@ SELECT
SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_FILE_THRESHOLD, SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_FILE_THRESHOLD,
SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_ROWS_THRESHOLD, SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_ROWS_THRESHOLD,
SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_BYTES_THRESHOLD, SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_BYTES_THRESHOLD,
SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY,
SUM(CASE WHEN ARCHIVE_ENABLED = 'Y' THEN 1 ELSE 0 END) AS ARCHIVAL_ENABLED,
SUM(CASE WHEN KEEP_IN_TRASH = 'N' THEN 1 ELSE 0 END) AS IMMEDIATE_DELETE
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%') AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%')
@@ -269,6 +289,8 @@ PROMPT - WITH_FILE_THRESHOLD = 25
PROMPT - WITH_ROWS_THRESHOLD = 25 PROMPT - WITH_ROWS_THRESHOLD = 25
PROMPT - WITH_BYTES_THRESHOLD = 25 PROMPT - WITH_BYTES_THRESHOLD = 25
PROMPT - WITH_STATS_EXPIRY = 25 PROMPT - WITH_STATS_EXPIRY = 25
PROMPT - ARCHIVAL_ENABLED = 25 (all tables enabled for archival)
PROMPT - IMMEDIATE_DELETE = 25 (all tables delete files immediately, no TRASH retention)
PROMPT PROMPT
PROMPT ===================================================================== PROMPT =====================================================================

20
MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql
View File

@@ -1,20 +1,28 @@
-- MARS-828: Rollback archival strategy columns -- MARS-828: Rollback archival strategy columns
-- Author: Grzegorz Michalski -- Author: Grzegorz Michalski
-- Date: 2026-01-27 -- Date: 2026-01-27
-- Description: Remove ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns -- Description: Remove ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, ARCHIVE_ENABLED, and KEEP_IN_TRASH columns
PROMPT ======================================== PROMPT ========================================
PROMPT MARS-828: Removing archival strategy columns PROMPT MARS-828: Removing archival strategy and config columns
PROMPT ======================================== PROMPT ========================================
-- Drop check constraint first -- Drop check constraints first
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY; DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY;
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
DROP CONSTRAINT CHK_ARCHIVE_ENABLED;
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
DROP CONSTRAINT CHK_KEEP_IN_TRASH;
-- Drop columns -- Drop columns
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP ( ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
ARCHIVAL_STRATEGY, ARCHIVAL_STRATEGY,
MINIMUM_AGE_MONTHS MINIMUM_AGE_MONTHS,
ARCHIVE_ENABLED,
KEEP_IN_TRASH
); );
-- Verify columns dropped -- Verify columns dropped
@@ -23,8 +31,8 @@ SELECT
FROM all_tab_columns FROM all_tab_columns
WHERE owner = 'CT_MRDS' WHERE owner = 'CT_MRDS'
AND table_name = 'A_SOURCE_FILE_CONFIG' AND table_name = 'A_SOURCE_FILE_CONFIG'
AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS'); AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'ARCHIVE_ENABLED', 'KEEP_IN_TRASH');
PROMPT ======================================== PROMPT ========================================
PROMPT Archival strategy columns removed successfully PROMPT Archival strategy and config columns removed successfully
PROMPT ======================================== PROMPT ========================================

34
MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql
View File

@@ -35,9 +35,9 @@ PROMPT =========================================================================
PROMPT MARS-828 Installation Starting PROMPT MARS-828 Installation Starting
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT Package: CT_MRDS.FILE_ARCHIVER PROMPT Package: CT_MRDS.FILE_ARCHIVER
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention control PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving
PROMPT Purpose: Flexible archival policies per data source with file retention management PROMPT Purpose: Flexible archival policies per data source with file retention and config-based control
PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.2.0, Verify, Track, Configure) PROMPT Steps: 9 (DDL, Trigger, Statuses, Package v3.3.0, Verify, Track, Configure)
PROMPT Timestamp: PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
PROMPT ============================================================================ PROMPT ============================================================================
@@ -55,8 +55,8 @@ WHENEVER SQLERROR CONTINUE
-- Installation steps -- Installation steps
PROMPT PROMPT
PROMPT Step 1/9: Adding archival strategy columns to A_SOURCE_FILE_CONFIG PROMPT Step 1/9: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG
PROMPT =================================================================== PROMPT =============================================================================
@@01_MARS_828_install_add_archival_strategy_columns.sql @@01_MARS_828_install_add_archival_strategy_columns.sql
PROMPT PROMPT
@@ -70,12 +70,12 @@ PROMPT =====================================================================
@@07_MARS_828_install_add_trash_retention_statuses.sql @@07_MARS_828_install_add_trash_retention_statuses.sql
PROMPT PROMPT
PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.2.0 PROMPT Step 4/9: Deploying FILE_ARCHIVER Package Specification v3.3.0
PROMPT ================================================================ PROMPT ================================================================
@@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql @@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
PROMPT PROMPT
PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.2.0 PROMPT Step 5/9: Deploying FILE_ARCHIVER Package Body v3.3.0
PROMPT ====================================================== PROMPT ======================================================
@@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
@@ -108,21 +108,21 @@ SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL;
PROMPT PROMPT
PROMPT Installation Summary: PROMPT Installation Summary:
PROMPT - Package: CT_MRDS.FILE_ARCHIVER PROMPT - Package: CT_MRDS.FILE_ARCHIVER
PROMPT - Version: 3.2.0 (includes TRASH folder retention control) PROMPT - Version: 3.3.0 (includes selective archiving and config-based TRASH policy)
PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID
PROMPT - TRASH Retention: pKeepInTrash parameter (DEFAULT TRUE for safety) PROMPT - Selective Archiving: ARCHIVE_ENABLED column (Y=archive, N=skip)
PROMPT - TRASH Policy: KEEP_IN_TRASH column (Y=keep files, N=delete immediately)
PROMPT * Default: ARCHIVE_ENABLED='Y', KEEP_IN_TRASH='N' (archiving enabled, immediate deletion)
PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME) PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
PROMPT * Files kept in TRASH by default for compliance and rollback capability PROMPT * No more pKeepInTrash parameter - policy from config only
PROMPT - New Procedure: ARCHIVE_ALL_FOR_SOURCE(pSourceKey) for batch processing
PROMPT - TRASH Management: RESTORE_FILE_FROM_TRASH, PURGE_TRASH_FOLDER (3-level granularity)
PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED
PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED + TRASH retention preserved) PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED, existing behavior preserved)
PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB) PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB)
PROMPT - Includes Fixes: PROMPT - Includes All Fixes from v3.0.0 through v3.2.1
PROMPT * v3.1.1: ORA-01422 for multiple parquet files
PROMPT * v3.1.2: PARTITION_YEAR/PARTITION_MONTH assignments
PROMPT * v3.1.2: Export query circular dependency
PROMPT * v3.2.0: TRASH retention control (pKeepInTrash parameter)
PROMPT PROMPT
PROMPT Note: Incremental patches (v3.1.0->v3.1.1->v3.1.2->v3.2.0) available in patches/ PROMPT Note: Incremental patches available in patches/ directory
PROMPT PROMPT
PROMPT Log file: &_filename PROMPT Log file: &_filename
PROMPT ============================================================================ PROMPT ============================================================================

56
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql Normal file
View File

@@ -0,0 +1,56 @@
-- ====================================================================
-- A_SOURCE_FILE_CONFIG Table
-- ====================================================================
-- Purpose: Store source file configuration and processing rules
-- MARS-1049: Added ENCODING column for CSV character set support
-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE,
SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG'
SOURCE_FILE_ID VARCHAR2(200),
SOURCE_FILE_DESC VARCHAR2(2000),
SOURCE_FILE_NAME_PATTERN VARCHAR2(200),
TABLE_ID VARCHAR2(200),
TEMPLATE_TABLE_NAME VARCHAR2(200),
CONTAINER_FILE_KEY NUMBER(38,0),
DAYS_FOR_ARCHIVE_THRESHOLD NUMBER(4,0),
FILES_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
BYTES_SUM_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
ODS_SCHEMA_NAME VARCHAR2(100),
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3),
ARCHIVAL_STRATEGY VARCHAR2(50),
MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID)
) TABLESPACE "DATA";
-- Primary key index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY")
TABLESPACE "DATA";
-- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID")
TABLESPACE "DATA";
-- Column comments
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;

318
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb
View File

@@ -87,8 +87,7 @@ AS
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
PROCEDURE ARCHIVE_TABLE_DATA ( PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) )
IS IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
@@ -103,15 +102,27 @@ AS
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
vProcessControlStatus VARCHAR2(60) := 'OK'; vProcessControlStatus VARCHAR2(60) := 'OK';
vKeepInTrash BOOLEAN; -- Derived from config
BEGIN BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
)); ));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
-- Check if archiving is enabled for this configuration
IF vSourceFileConfig.ARCHIVE_ENABLED = 'N' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archiving disabled for this configuration (ARCHIVE_ENABLED=N). Skipping.', 'WARNING', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN;
END IF;
-- Get TRASH policy from configuration
vKeepInTrash := (vSourceFileConfig.KEEP_IN_TRASH = 'Y');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: KEEP_IN_TRASH=' || vSourceFileConfig.KEEP_IN_TRASH, 'INFO', vParameters);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
@@ -271,7 +282,7 @@ AS
-- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART) -- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART)
-- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME) -- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME)
IF vProcessControlStatus = 'OK' THEN IF vProcessControlStatus = 'OK' THEN
IF NOT pKeepInTrash THEN IF NOT vKeepInTrash THEN
-- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED -- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
@@ -285,10 +296,10 @@ AS
AND r.source_file_name = f.filename AND r.source_file_name = f.filename
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
END LOOP; END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO'); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: KEEP_IN_TRASH=N).','INFO');
ELSE ELSE
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED) -- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO'); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO');
END IF; END IF;
--ROLLBACK PART --ROLLBACK PART
@@ -947,20 +958,18 @@ AS
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
FUNCTION ARCHIVE_TABLE_DATA ( FUNCTION FN_ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER ) RETURN PLS_INTEGER
IS IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
)); ));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
---- ----
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash); ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey);
---- ----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE; RETURN SQLCODE;
@@ -968,11 +977,11 @@ AS
WHEN OTHERS THEN WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE; RETURN SQLCODE;
END ARCHIVE_TABLE_DATA; END FN_ARCHIVE_TABLE_DATA;
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
FUNCTION GATHER_TABLE_STAT ( FUNCTION FN_GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) RETURN PLS_INTEGER ) RETURN PLS_INTEGER
IS IS
@@ -989,7 +998,284 @@ AS
WHEN OTHERS THEN WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE; RETURN SQLCODE;
END GATHER_TABLE_STAT; END FN_GATHER_TABLE_STAT;
----------------------------------------------------------------------------------------------------
-- BATCH ARCHIVAL PROCEDURES
----------------------------------------------------------------------------------------------------
PROCEDURE ARCHIVE_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pArchiveAll IN BOOLEAN DEFAULT FALSE
)
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vTablesArchived PLS_INTEGER := 0;
vTablesSkipped PLS_INTEGER := 0;
vTablesFailed PLS_INTEGER := 0;
vProcessingLevel VARCHAR2(50);
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''',
'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''',
'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||''''
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Determine processing level and validate parameters
IF pSourceFileConfigKey IS NOT NULL THEN
vProcessingLevel := 'LEVEL 1: Single Config Key';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO');
ELSIF pSourceKey IS NOT NULL THEN
vProcessingLevel := 'LEVEL 2: Source Key';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO');
ELSIF pArchiveAll THEN
vProcessingLevel := 'LEVEL 3: Archive All';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Archive All Enabled Tables', 'INFO');
ELSE
RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pArchiveAll=TRUE');
END IF;
FOR config_rec IN (
SELECT
A_SOURCE_FILE_CONFIG_KEY,
TABLE_ID,
ARCHIVE_ENABLED,
KEEP_IN_TRASH,
A_SOURCE_KEY
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT'
AND (
-- Level 1: Specific config key
(pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey)
OR
-- Level 2: All configs for source key
(pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey)
OR
-- Level 3: All configs when pArchiveAll = TRUE
(pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pArchiveAll = TRUE)
)
ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY
) LOOP
IF config_rec.ARCHIVE_ENABLED = 'N' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Skipping table ' || config_rec.TABLE_ID || ' (ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']',
'INFO'
);
vTablesSkipped := vTablesSkipped + 1;
ELSE
BEGIN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', KEEP_IN_TRASH=' || config_rec.KEEP_IN_TRASH || ']',
'INFO'
);
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY);
vTablesArchived := vTablesArchived + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Successfully archived table ' || config_rec.TABLE_ID,
'INFO'
);
EXCEPTION
WHEN OTHERS THEN
vTablesFailed := vTablesFailed + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Failed to archive table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM,
'ERROR'
);
-- Continue with next table
END;
END IF;
END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
vProcessingLevel || ' - Batch archival summary: Archived=' || vTablesArchived || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed,
'INFO'
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END ARCHIVE_ALL;
----------------------------------------------------------------------------------------------------
FUNCTION FN_ARCHIVE_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pArchiveAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''',
'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''',
'pArchiveAll => '''||CASE WHEN pArchiveAll THEN 'TRUE' ELSE 'FALSE' END||''''
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
ARCHIVE_ALL(
pSourceFileConfigKey => pSourceFileConfigKey,
pSourceKey => pSourceKey,
pArchiveAll => pArchiveAll
);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END FN_ARCHIVE_ALL;
----------------------------------------------------------------------------------------------------
-- BATCH STATISTICS GATHERING PROCEDURES
----------------------------------------------------------------------------------------------------
PROCEDURE GATHER_TABLE_STAT_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pGatherAll IN BOOLEAN DEFAULT FALSE,
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
)
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vTablesProcessed PLS_INTEGER := 0;
vTablesSkipped PLS_INTEGER := 0;
vTablesFailed PLS_INTEGER := 0;
vProcessingLevel VARCHAR2(50);
vEnabledFilter VARCHAR2(50);
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''',
'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''',
'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''',
'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||''''
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Determine processing level and validate parameters
IF pSourceFileConfigKey IS NOT NULL THEN
vProcessingLevel := 'LEVEL 1: Single Config Key';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 1: pSourceFileConfigKey=' || pSourceFileConfigKey, 'INFO');
ELSIF pSourceKey IS NOT NULL THEN
vProcessingLevel := 'LEVEL 2: Source Key';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 2: pSourceKey=' || pSourceKey, 'INFO');
ELSIF pGatherAll THEN
vProcessingLevel := 'LEVEL 3: Gather All Statistics';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing Level 3: Gather All Enabled Statistics', 'INFO');
ELSE
RAISE_APPLICATION_ERROR(-20003, 'No processing level specified. Provide pSourceFileConfigKey, pSourceKey, or set pGatherAll=TRUE');
END IF;
-- Set enabled filter info
vEnabledFilter := CASE WHEN pOnlyEnabled THEN 'ARCHIVE_ENABLED=Y only' ELSE 'All tables' END;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Filter mode: ' || vEnabledFilter, 'INFO');
FOR config_rec IN (
SELECT
A_SOURCE_FILE_CONFIG_KEY,
TABLE_ID,
ARCHIVE_ENABLED,
A_SOURCE_KEY
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT'
AND (
-- Level 1: Specific config key
(pSourceFileConfigKey IS NOT NULL AND A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey)
OR
-- Level 2: All configs for source key
(pSourceFileConfigKey IS NULL AND pSourceKey IS NOT NULL AND A_SOURCE_KEY = pSourceKey)
OR
-- Level 3: All configs when pGatherAll = TRUE
(pSourceFileConfigKey IS NULL AND pSourceKey IS NULL AND pGatherAll = TRUE)
)
-- Apply ARCHIVE_ENABLED filter if pOnlyEnabled = TRUE
AND (pOnlyEnabled = FALSE OR ARCHIVE_ENABLED = 'Y')
ORDER BY A_SOURCE_KEY, A_SOURCE_FILE_CONFIG_KEY
) LOOP
IF pOnlyEnabled AND config_rec.ARCHIVE_ENABLED = 'N' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Skipping table ' || config_rec.TABLE_ID || ' (ARCHIVE_ENABLED=N) [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']',
'INFO'
);
vTablesSkipped := vTablesSkipped + 1;
ELSE
BEGIN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Gathering statistics for table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', ARCHIVE_ENABLED=' || config_rec.ARCHIVE_ENABLED || ']',
'INFO'
);
GATHER_TABLE_STAT(pSourceFileConfigKey => config_rec.A_SOURCE_FILE_CONFIG_KEY);
vTablesProcessed := vTablesProcessed + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Successfully gathered statistics for table ' || config_rec.TABLE_ID,
'INFO'
);
EXCEPTION
WHEN OTHERS THEN
vTablesFailed := vTablesFailed + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Failed to gather statistics for table ' || config_rec.TABLE_ID || ' [Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ']: ' || SQLERRM,
'ERROR'
);
-- Continue with next table
END;
END IF;
END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
vProcessingLevel || ' - Batch statistics gathering summary: Processed=' || vTablesProcessed || ', Skipped=' || vTablesSkipped || ', Failed=' || vTablesFailed || ' [Filter: ' || vEnabledFilter || ']',
'INFO'
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GATHER_TABLE_STAT_ALL;
----------------------------------------------------------------------------------------------------
FUNCTION FN_GATHER_TABLE_STAT_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pGatherAll IN BOOLEAN DEFAULT FALSE,
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '''||nvl(to_char(pSourceFileConfigKey),'NULL')||'''',
'pSourceKey => '''||nvl(pSourceKey,'NULL')||'''',
'pGatherAll => '''||CASE WHEN pGatherAll THEN 'TRUE' ELSE 'FALSE' END||'''',
'pOnlyEnabled => '''||CASE WHEN pOnlyEnabled THEN 'TRUE' ELSE 'FALSE' END||''''
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
GATHER_TABLE_STAT_ALL(
pSourceFileConfigKey => pSourceFileConfigKey,
pSourceKey => pSourceKey,
pGatherAll => pGatherAll,
pOnlyEnabled => pOnlyEnabled
);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END FN_GATHER_TABLE_STAT_ALL;
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------

116
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg
View File

@@ -17,12 +17,13 @@ AS
**/ **/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.1'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-10 09:00:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.3.0 (2026-02-11): Added ARCHIVE_ENABLED and KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) || '3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) || '3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
@@ -35,30 +36,38 @@ AS
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
/**
* @name GET_TABLE_STAT
* @desc Private function to retrieve table statistics for archival processing.
* Returns A_TABLE_STAT record with table metadata and row counts.
* @param pSourceFileConfigKey - Configuration key for source file
* @return CT_MRDS.A_TABLE_STAT%ROWTYPE - Table statistics record
* @private Internal function for archival operations
**/
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE;
/** /**
* @name ARCHIVE_TABLE_DATA * @name ARCHIVE_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning). * Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
**/ **/
PROCEDURE ARCHIVE_TABLE_DATA ( PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pKeepInTrash IN BOOLEAN DEFAULT TRUE
); );
/** /**
* @name ARCHIVE_TABLE_DATA * @name FN_ARCHIVE_TABLE_DATA
* @desc Function overload for ARCHIVE_TABLE_DATA procedure. * @desc Function wrapper for ARCHIVE_TABLE_DATA procedure.
* Returns SQLCODE for Python library integration. * Returns SQLCODE for Python library integration.
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive. * TRASH policy is controlled by A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
* @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; * @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code * @ex_rslt 0 (success) or error code
**/ **/
FUNCTION ARCHIVE_TABLE_DATA ( FUNCTION FN_ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE, pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER; ) RETURN PLS_INTEGER;
@@ -73,17 +82,92 @@ AS
); );
/** /**
* @name GATHER_TABLE_STAT * @name FN_GATHER_TABLE_STAT
* @desc Function overload for GATHER_TABLE_STAT procedure. * @desc Function wrapper for GATHER_TABLE_STAT procedure.
* Returns SQLCODE for Python library integration. * Returns SQLCODE for Python library integration.
* Calls the main GATHER_TABLE_STAT procedure and captures execution result. * Calls the main GATHER_TABLE_STAT procedure and captures execution result.
* @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; * @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code * @ex_rslt 0 (success) or error code
**/ **/
FUNCTION GATHER_TABLE_STAT ( FUNCTION FN_GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) RETURN PLS_INTEGER; ) RETURN PLS_INTEGER;
/**
* @name GATHER_TABLE_STAT_ALL
* @desc Multi-level batch statistics gathering procedure with three granularity levels.
* Processes configurations based on ARCHIVE_ENABLED setting (when pOnlyEnabled=TRUE).
* Gathers statistics for external tables and inserts data into A_TABLE_STAT and A_TABLE_STAT_HIST.
* @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority)
* @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (e.g., 'LM', 'C2D') (medium priority)
* @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority)
* @param pOnlyEnabled - When TRUE (default), only process tables with ARCHIVE_ENABLED='Y'
* @example -- Level 1: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceFileConfigKey => 123);
* @example -- Level 2: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceKey => 'LM');
* @example -- Level 3: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE);
* @example -- All tables regardless of ARCHIVE_ENABLED: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE, pOnlyEnabled => FALSE);
**/
PROCEDURE GATHER_TABLE_STAT_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pGatherAll IN BOOLEAN DEFAULT FALSE,
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
);
/**
* @name FN_GATHER_TABLE_STAT_ALL
* @desc Function wrapper for GATHER_TABLE_STAT_ALL procedure.
* Returns SQLCODE for Python library integration.
* Calls the main GATHER_TABLE_STAT_ALL procedure and captures execution result.
* @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority)
* @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (medium priority)
* @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority)
* @param pOnlyEnabled - When TRUE (default), only process tables with ARCHIVE_ENABLED='Y'
* @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT_ALL(pSourceKey => 'LM') FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION FN_GATHER_TABLE_STAT_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pGatherAll IN BOOLEAN DEFAULT FALSE,
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER;
/**
* @name ARCHIVE_ALL
* @desc Multi-level batch archival procedure with three granularity levels.
* Only processes configurations where ARCHIVE_ENABLED='Y'.
* TRASH policy for each table is controlled by individual KEEP_IN_TRASH column.
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority)
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)
* @example -- Level 1: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceFileConfigKey => 123);
* @example -- Level 2: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceKey => 'LM');
* @example -- Level 3: CALL FILE_ARCHIVER.ARCHIVE_ALL(pArchiveAll => TRUE);
**/
PROCEDURE ARCHIVE_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pArchiveAll IN BOOLEAN DEFAULT FALSE
);
/**
* @name FN_ARCHIVE_ALL
* @desc Function wrapper for ARCHIVE_ALL procedure.
* Returns SQLCODE for Python library integration.
* Calls the main ARCHIVE_ALL procedure and captures execution result.
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (medium priority)
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)
* @example SELECT FILE_ARCHIVER.FN_ARCHIVE_ALL(pSourceKey => 'LM') FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION FN_ARCHIVE_ALL (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
pArchiveAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER;
/** /**
* @name RESTORE_FILE_FROM_TRASH * @name RESTORE_FILE_FROM_TRASH
* @desc Restores files from TRASH folder back to ODS at three different granularity levels. * @desc Restores files from TRASH folder back to ODS at three different granularity levels.

6
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql
View File

@@ -34,7 +34,7 @@ PROMPT
PROMPT Rollback steps: PROMPT Rollback steps:
PROMPT 1. Rollback TRASH retention statuses PROMPT 1. Rollback TRASH retention statuses
PROMPT 2. Remove validation trigger PROMPT 2. Remove validation trigger
PROMPT 3. Drop ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns PROMPT 3. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, ARCHIVE_ENABLED, KEEP_IN_TRASH)
PROMPT 4. Restore FILE_ARCHIVER package to v2.0.0 PROMPT 4. Restore FILE_ARCHIVER package to v2.0.0
PROMPT 5. Revert all archival strategies to THRESHOLD_BASED PROMPT 5. Revert all archival strategies to THRESHOLD_BASED
PROMPT PROMPT
@@ -65,8 +65,8 @@ PROMPT ======================================
@@93_MARS_828_rollback_trigger.sql @@93_MARS_828_rollback_trigger.sql
PROMPT PROMPT
PROMPT Step 3/6: Dropping archival strategy columns PROMPT Step 3/6: Dropping all archival configuration columns
PROMPT ============================================= PROMPT ======================================================
@@94_MARS_828_rollback_columns.sql @@94_MARS_828_rollback_columns.sql
PROMPT PROMPT

998
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkb Normal file
View File

@@ -0,0 +1,998 @@
create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER
AS
----------------------------------------------------------------------------------------------------
-- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE
----------------------------------------------------------------------------------------------------
/**
* @name GET_ARCHIVAL_WHERE_CLAUSE
* @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration.
* Supports three strategies: THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID.
* @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY
* @return VARCHAR2 - WHERE clause for filtering archival candidates
**/
FUNCTION GET_ARCHIVAL_WHERE_CLAUSE(
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
) RETURN VARCHAR2
IS
vWhereClause VARCHAR2(4000);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
BEGIN
CASE pSourceFileConfig.ARCHIVAL_STRATEGY
-- Legacy threshold-based strategy (backward compatible)
WHEN 'THRESHOLD_BASED' THEN
vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD;
-- Archive data older than X months (0 = current month only)
WHEN 'MINIMUM_AGE_MONTHS' THEN
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy');
END IF;
vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
-- Hybrid: Current month exclusion AND minimum age requirement
WHEN 'HYBRID' THEN
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy');
END IF;
vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' ||
'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
ELSE
RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY);
END CASE;
RETURN vWhereClause;
END GET_ARCHIVAL_WHERE_CLAUSE;
----------------------------------------------------------------------------------------------------
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
IS
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vCount PLS_INTEGER;
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT count(*) , min(SOURCE_FILE_TYPE)
INTO vCount, vSourceFileType
FROM CT_MRDS.A_TABLE_STAT s
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
IF vCount=0 and vSourceFileType='INPUT' THEN
GATHER_TABLE_STAT(pSourceFileConfigKey);
END IF;
BEGIN
SELECT *
INTO vTableStat
FROM CT_MRDS.A_TABLE_STAT
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- EXCEPTION
-- WHEN NO_DATA_FOUND THEN
--
END;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vTableStat;
END GET_TABLE_STAT;
----------------------------------------------------------------------------------------------------
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vQuery VARCHAR2(4000);
vTableName VARCHAR2(200);
vUri VARCHAR2(1000);
vfiles T_FILENAMES;
vFilename VARCHAR2(300);
vOperationId NUMBER := -1;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
vProcessControlStatus VARCHAR2(60) := 'OK';
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
end if;
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
end if;
-- Strategy-based trigger logic (MARS-828)
IF vSourceFileConfig.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN
-- MINIMUM_AGE_MONTHS: Archive based on age only, ignore thresholds
vArchivalTriggeredBy := 'AGE_BASED';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival strategy: MINIMUM_AGE_MONTHS (threshold-independent)','INFO');
ELSE
-- THRESHOLD_BASED and HYBRID: Check thresholds
if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT';
elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT';
elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM';
else CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO');
end if;
END IF;
if LENGTH(vArchivalTriggeredBy)>0 THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO');
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
-- Use strategy-based WHERE clause (MARS-828)
-- Using GROUP BY instead of DISTINCT to avoid ORA-22950 (object type ordering issue)
vQuery := '
select CT_MRDS.t_filename(
file$name
,file$path
, to_char(h.workflow_start,''yyyy'')
, to_char(h.workflow_start,''mm'')
)
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || '
group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')'
;
-- Get all files that will be archived into "vfiles" collection ("regular data files")
execute immediate vQuery bulk collect into vfiles;
-- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
vQuery:=
'select
s.*
from '|| vTableName ||' s
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
on s.file$name = r.source_file_name
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
and r.PROCESSING_STATUS = ''INGESTED''
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||'''
and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||'''
'
;
vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
BEGIN
DBMS_CLOUD.EXPORT_DATA(
credential_name => ENV_MANAGER.gvCredentialName,
file_uri_list => vUri||'d' ,
format => json_object('type' value 'parquet'),
query => vQuery,
operation_id => vOperationId
);
EXCEPTION
WHEN OTHERS THEN
vProcessControlStatus :='EXPORT_FAILURE';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED);
END;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
-- Get USER_LOAD_OPERATIONS info
select *
into vUserLoadOperations
from USER_LOAD_OPERATIONS
where id = vOperationId;
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS);
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.');
ELSE
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month,'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
END IF;
-- Note: DBMS_CLOUD.EXPORT_DATA may create multiple parquet files (parallel execution)
-- Instead of tracking individual files, we store the archive directory prefix
-- ARCH_FILE_NAME will contain the directory URI where all parquet files are located
vFilename := vUri; -- Store directory prefix instead of individual filename
-- Try to drop EXPORTED FILES ("regular data files")
BEGIN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
-- first change of status to ARCHIVED_AND_TRASHED (file will be moved to TRASH folder)
BEGIN
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -- Status reflects file is archived and kept in TRASH
,ARCH_FILE_NAME = vFilename -- Now contains directory prefix, not individual file
,PARTITION_YEAR = ym_loop.year -- Record which partition year the data was archived to
,PARTITION_MONTH = ym_loop.month -- Record which partition month the data was archived to
WHERE r.a_source_file_config_key= pSourceFileConfigKey
AND r.source_file_name = f.filename
AND r.processing_status = 'INGESTED'
;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE';
-- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => f.pathname||'/'||f.filename,
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
rollback;
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
commit;
END LOOP;
--------------------------------------------------------------------
-- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART)
-- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME)
IF vProcessControlStatus = 'OK' THEN
IF NOT pKeepInTrash THEN
-- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
-- Update status to ARCHIVED_AND_PURGED
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
WHERE r.a_source_file_config_key = pSourceFileConfigKey
AND r.source_file_name = f.filename
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO');
ELSE
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO');
END IF;
--ROLLBACK PART
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket)
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
FOR f in ( SELECT vf.filename, vf.pathname
FROM TABLE(vfiles) vf
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
ON r.source_file_name = vf.filename
AND r.a_source_file_config_key = pSourceFileConfigKey
AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')
AND vf.year = ym_loop.year
AND vf.month = ym_loop.month
) LOOP
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_object_uri => f.pathname||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'INGESTED'
,ARCH_FILE_NAME = NULL
WHERE r.a_source_file_config_key = pSourceFileConfigKey
AND r.source_file_name = f.filename
;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
END;
END LOOP;
-- ROLLBACK: Delete all parquet files from archive directory
FOR arch_file IN (
SELECT object_name
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
location_uri => vFilename -- vFilename now contains directory prefix
)
) LOOP
DBMS_CLOUD.DELETE_OBJECT(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => vFilename || arch_file.object_name
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name);
END LOOP;
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters);
-- ROLLBACK: Delete all parquet files from archive directory
FOR arch_file IN (
SELECT object_name
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
location_uri => vFilename -- vFilename now contains directory prefix
)
) LOOP
DBMS_CLOUD.DELETE_OBJECT(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => vFilename || arch_file.object_name
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name);
END LOOP;
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
END IF;
EXCEPTION
WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
WHEN CT_MRDS.ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
END;
-- END of "Try to drop EXPORTED FILES"
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
END LOOP; --ym_loop end (YEAR_MONTH)
COMMIT;
ELSE
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO');
END IF;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN CT_MRDS.ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN CT_MRDS.ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END ARCHIVE_TABLE_DATA;
----------------------------------------------------------------------------------------------------
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableName VARCHAR2(200);
vQuery VARCHAR2(32000);
vWhereClause VARCHAR2(4000);
vOdsBucketUri VARCHAR2(1000);
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
-- Get WHERE clause based on archival strategy (MARS-828)
vWhereClause := GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vWhereClause','DEBUG',vWhereClause);
-- Get ODS bucket URI before building query
vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
-- Use strategy-based WHERE clause for statistics (MARS-828)
vQuery :=
'with tmp as (
select
s.*
,file$name as filename
,h.workflow_start
, to_char(h.workflow_start,''yyyy'') as year
, to_char(h.workflow_start,''mm'') as month
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
)
, tmp_gr as (
select
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
from tmp
group by filename
)
select
NULL as A_TABLE_STAT_KEY
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
,'''||vTableName||''' as TABLE_NAME
,count(*) as FILE_COUNT
,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT
,sum (row_count_per_file) as ROW_COUNT
,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT
,sum(r.bytes) as BYTES
,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES
,'||COALESCE(TO_CHAR(vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD), 'NULL')||' as DAYS_FOR_ARCHIVE_THRESHOLD
,systimestamp as CREATED
from tmp_gr t
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''',
location_uri => '''||vOdsBucketUri||'''
)
) r
on t.filename = r.object_name'
;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
execute immediate vQuery into vStats;
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
insert into CT_MRDS.A_TABLE_STAT_HIST values vStats;
delete from CT_MRDS.A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
insert into CT_MRDS.A_TABLE_STAT values vStats;
COMMIT;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GATHER_TABLE_STAT;
----------------------------------------------------------------------------------------------------
-- TRASH FOLDER MANAGEMENT PROCEDURES
----------------------------------------------------------------------------------------------------
PROCEDURE RESTORE_FILE_FROM_TRASH (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pRestoreAll IN BOOLEAN DEFAULT FALSE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vTrashPath VARCHAR2(1000);
vOdsPath VARCHAR2(1000);
vFilesRestored PLS_INTEGER := 0;
vFilesUpdated PLS_INTEGER := 0;
vRestoreLevel VARCHAR2(50);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
IF pSourceFileReceivedKey IS NOT NULL THEN
vRestoreLevel := 'LEVEL_3_SINGLE_FILE';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
-- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY
BEGIN
SELECT *
INTO vSourceFileReceived
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY);
EXCEPTION
WHEN NO_DATA_FOUND THEN
RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
END;
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath);
BEGIN
DBMS_CLOUD.MOVE_OBJECT(
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
source_object_uri => vTrashPath,
target_object_uri => vOdsPath,
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
);
vFilesRestored := 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM);
END;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'INGESTED',
ARCH_FILE_NAME = NULL,
PARTITION_YEAR = NULL,
PARTITION_MONTH = NULL
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSIF pSourceFileConfigKey IS NOT NULL THEN
vRestoreLevel := 'LEVEL_2_CONFIG_FILES';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
-- LEVEL 2: Restore all files for specific pSourceFileConfigKey
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
FOR file_rec IN (
SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
) LOOP
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
BEGIN
DBMS_CLOUD.MOVE_OBJECT(
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
source_object_uri => vTrashPath,
target_object_uri => vOdsPath,
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
);
vFilesRestored := vFilesRestored + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
END;
END LOOP;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'INGESTED',
ARCH_FILE_NAME = NULL,
PARTITION_YEAR = NULL,
PARTITION_MONTH = NULL
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSIF pRestoreAll THEN
vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status');
-- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations
FOR file_rec IN (
SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME,
c.A_SOURCE_KEY, c.TABLE_ID
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
) LOOP
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
BEGIN
DBMS_CLOUD.MOVE_OBJECT(
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
source_object_uri => vTrashPath,
target_object_uri => vOdsPath,
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
);
vFilesRestored := vFilesRestored + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
END;
END LOOP;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'INGESTED',
ARCH_FILE_NAME = NULL,
PARTITION_YEAR = NULL,
PARTITION_MONTH = NULL
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSE
RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE');
END IF;
COMMIT;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END RESTORE_FILE_FROM_TRASH;
----------------------------------------------------------------------------------------------------
PROCEDURE PURGE_TRASH_FOLDER (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pPurgeAll IN BOOLEAN DEFAULT FALSE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vTrashLocationUri VARCHAR2(1000);
vFilesDeleted PLS_INTEGER := 0;
vFilesUpdated PLS_INTEGER := 0;
vPurgeLevel VARCHAR2(50);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
IF pSourceFileReceivedKey IS NOT NULL THEN
vPurgeLevel := 'LEVEL_3_SINGLE_FILE';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
-- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY
BEGIN
SELECT *
INTO vSourceFileReceived
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY);
EXCEPTION
WHEN NO_DATA_FOUND THEN
RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
END;
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => vTrashLocationUri
);
vFilesDeleted := 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM);
END;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSIF pSourceFileConfigKey IS NOT NULL THEN
vPurgeLevel := 'LEVEL_2_CONFIG_FILES';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
-- LEVEL 2: Delete all files for specific pSourceFileConfigKey
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri);
BEGIN
FOR trash_file IN (
SELECT object_name
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
location_uri => vTrashLocationUri
)
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => vTrashLocationUri || trash_file.object_name
);
vFilesDeleted := vFilesDeleted + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
END;
END LOOP;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM);
END;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSIF pPurgeAll THEN
vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status');
-- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations
FOR config_rec IN (
SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID
FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
) LOOP
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/';
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri);
BEGIN
FOR trash_file IN (
SELECT object_name
FROM DBMS_CLOUD.LIST_OBJECTS(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
location_uri => vTrashLocationUri
)
) LOOP
BEGIN
DBMS_CLOUD.DELETE_OBJECT(
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
object_uri => vTrashLocationUri || trash_file.object_name
);
vFilesDeleted := vFilesDeleted + 1;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
END;
END LOOP;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM);
END;
END LOOP;
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
vFilesUpdated := SQL%ROWCOUNT;
ELSE
RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE');
END IF;
COMMIT;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END PURGE_TRASH_FOLDER;
----------------------------------------------------------------------------------------------------
FUNCTION PURGE_TRASH_FOLDER (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pPurgeAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER
IS
PRAGMA AUTONOMOUS_TRANSACTION;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
PURGE_TRASH_FOLDER(
pSourceFileReceivedKey => pSourceFileReceivedKey,
pSourceFileConfigKey => pSourceFileConfigKey,
pPurgeAll => pPurgeAll
);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END PURGE_TRASH_FOLDER;
----------------------------------------------------------------------------------------------------
FUNCTION RESTORE_FILE_FROM_TRASH (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pRestoreAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER
IS
PRAGMA AUTONOMOUS_TRANSACTION;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
RESTORE_FILE_FROM_TRASH(
pSourceFileReceivedKey => pSourceFileReceivedKey,
pSourceFileConfigKey => pSourceFileConfigKey,
pRestoreAll => pRestoreAll
);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END RESTORE_FILE_FROM_TRASH;
----------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION
RETURN VARCHAR2
IS
BEGIN
RETURN PACKAGE_VERSION;
END GET_VERSION;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUILD_INFO
RETURN VARCHAR2
IS
BEGIN
RETURN CT_MRDS.ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
pPackageName => 'FILE_ARCHIVER',
pVersion => PACKAGE_VERSION,
pBuildDate => PACKAGE_BUILD_DATE,
pAuthor => PACKAGE_AUTHOR
);
END GET_BUILD_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION_HISTORY
RETURN VARCHAR2
IS
BEGIN
RETURN CT_MRDS.ENV_MANAGER.FORMAT_VERSION_HISTORY(
pPackageName => 'FILE_ARCHIVER',
pVersionHistory => VERSION_HISTORY
);
END GET_VERSION_HISTORY;
----------------------------------------------------------------------------------------------------
FUNCTION ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END ARCHIVE_TABLE_DATA;
----------------------------------------------------------------------------------------------------
FUNCTION GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
----
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END GATHER_TABLE_STAT;
----------------------------------------------------------------------------------------------------
END;
/

195
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_version/v3.2.1/FILE_ARCHIVER.pkg Normal file
View File

@@ -0,0 +1,195 @@
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.1';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-10 09:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
'3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) ||
'3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) ||
'3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) ||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
/**
* @name ARCHIVE_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
**/
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
);
/**
* @name ARCHIVE_TABLE_DATA
* @desc Function overload for ARCHIVE_TABLE_DATA procedure.
* Returns SQLCODE for Python library integration.
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
* @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
pKeepInTrash IN BOOLEAN DEFAULT TRUE
) RETURN PLS_INTEGER;
/**
* @name GATHER_TABLE_STAT
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
**/
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
);
/**
* @name GATHER_TABLE_STAT
* @desc Function overload for GATHER_TABLE_STAT procedure.
* Returns SQLCODE for Python library integration.
* Calls the main GATHER_TABLE_STAT procedure and captures execution result.
* @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) RETURN PLS_INTEGER;
/**
* @name RESTORE_FILE_FROM_TRASH
* @desc Restores files from TRASH folder back to ODS at three different granularity levels.
* Moves files from TRASH subfolder back to ODS subfolder in DATA bucket.
* Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata.
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
* @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345);
* @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341);
* @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE);
**/
PROCEDURE RESTORE_FILE_FROM_TRASH (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pRestoreAll IN BOOLEAN DEFAULT FALSE
);
/**
* @name RESTORE_FILE_FROM_TRASH
* @desc Function overload for RESTORE_FILE_FROM_TRASH procedure.
* Returns SQLCODE for Python library integration.
* Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result.
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
* @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION RESTORE_FILE_FROM_TRASH (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pRestoreAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER;
/**
* @name PURGE_TRASH_FOLDER
* @desc Deletes files from TRASH folder at three different granularity levels.
* Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files.
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
* @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345);
* @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341);
* @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE);
**/
PROCEDURE PURGE_TRASH_FOLDER (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pPurgeAll IN BOOLEAN DEFAULT FALSE
);
/**
* @name PURGE_TRASH_FOLDER
* @desc Function overload for PURGE_TRASH_FOLDER procedure.
* Returns SQLCODE for Python library integration.
* Calls the main PURGE_TRASH_FOLDER procedure and captures execution result.
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
* @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
FUNCTION PURGE_TRASH_FOLDER (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
pPurgeAll IN BOOLEAN DEFAULT FALSE
) RETURN PLS_INTEGER;
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_VERSION
* @desc Returns the current version number of the FILE_ARCHIVER package.
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
* @ex_rslt 2.0.0
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* @name GET_BUILD_INFO
* @desc Returns comprehensive build information including version, build date, and author.
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: FILE_ARCHIVER
* Version: 2.0.0
* Build Date: 2025-10-22 16:45:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* @name GET_VERSION_HISTORY
* @desc Returns complete version history with all releases and changes.
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
* @ex_rslt FILE_ARCHIVER Version History:
* 2.0.0 (2025-10-22): Added package versioning system...
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END;
/

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql
View File

@@ -25,7 +25,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
ARCHIVAL_STRATEGY VARCHAR2(50), ARCHIVAL_STRATEGY VARCHAR2(50),
MINIMUM_AGE_MONTHS NUMBER(3,0), MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8', ENCODING VARCHAR2(50) DEFAULT 'UTF8',
ARCHIVE_ENABLED CHAR(1) DEFAULT 'Y' NOT NULL,
KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
@@ -46,5 +50,7 @@ TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;