diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/.gitignore b/MARS_Packages/REL01_ADDITIONS/MARS-828/.gitignore new file mode 100644 index 0000000..d8c41ba --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/.gitignore @@ -0,0 +1,25 @@ +# MARS-828 Package - Git Ignore Rules +# Standard exclusions for MARS deployment packages + +# Confluence documentation (generated, not source) +confluence/ + +# Log files from SPOOL operations +log/ +*.log + +# Test directories and files +test/ +*_test.sql + +# Mock data scripts (development only) +mock_data/ + +# Temporary files +*.tmp +*.bak +*~ + +# OS-specific files +.DS_Store +Thumbs.db diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql new file mode 100644 index 0000000..7e64f56 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql @@ -0,0 +1,44 @@ +-- MARS-828: Add archival strategy columns to A_SOURCE_FILE_CONFIG +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Description: Adds ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns to support flexible archival strategies + +PROMPT ======================================== +PROMPT MARS-828: Adding archival strategy columns +PROMPT ======================================== + +-- Add new columns +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD ( + ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL, + MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL +); + +-- Add check constraint for valid strategies +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT + CHK_ARCHIVAL_STRATEGY CHECK ( + ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID') + ); + +-- Add comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS + 'Archival strategy: THRESHOLD_BASED (days), CURRENT_MONTH_ONLY (exclude current month), MINIMUM_AGE_MONTHS (minimum age), HYBRID (combination)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS + 'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)'; + +-- Verify columns added +SELECT + column_name, + data_type, + data_length, + nullable, + data_default +FROM all_tab_columns +WHERE owner = 'CT_MRDS' + AND table_name = 'A_SOURCE_FILE_CONFIG' + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS') +ORDER BY column_id; + +PROMPT ======================================== +PROMPT Archival strategy columns added successfully +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/02_MARS_828_install_archival_strategy_trigger.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/02_MARS_828_install_archival_strategy_trigger.sql new file mode 100644 index 0000000..401ca46 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/02_MARS_828_install_archival_strategy_trigger.sql @@ -0,0 +1,24 @@ +-- MARS-828: Create validation trigger for archival strategy +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Description: Validates archival strategy configuration consistency + +PROMPT ======================================== +PROMPT MARS-828: Creating archival strategy validation trigger +PROMPT ======================================== + +@@new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql + +-- Verify trigger created +SELECT + trigger_name, + status, + trigger_type, + triggering_event +FROM all_triggers +WHERE owner = 'CT_MRDS' + AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL'; + +PROMPT ======================================== +PROMPT Archival strategy validation trigger created successfully +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql new file mode 100644 index 0000000..9ff3c4c --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql @@ -0,0 +1,14 @@ +-- =================================================================== +-- MARS-828: Install FILE_ARCHIVER Package Specification v3.0.0 +-- =================================================================== +-- Purpose: Deploy updated package specification with version 3.0.0 +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- =================================================================== + +@@new_version/FILE_ARCHIVER.pkg + + +PROMPT ======================================== +PROMPT FILE_ARCHIVER Specification v3.0.0 ready for installation +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql new file mode 100644 index 0000000..d7d8792 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@ -0,0 +1,14 @@ +-- =================================================================== +-- MARS-828: Install FILE_ARCHIVER Package Body v3.0.0 +-- =================================================================== +-- Purpose: Deploy updated package body with GET_ARCHIVAL_WHERE_CLAUSE function +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Changes: +-- - Added GET_ARCHIVAL_WHERE_CLAUSE private function +-- - Updated ARCHIVE_TABLE_DATA to use strategy-based filtering +-- - Updated GATHER_TABLE_STAT to use strategy-based statistics +-- =================================================================== + +@@new_version/FILE_ARCHIVER.pkb + diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/05_MARS_828_verify_installation.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/05_MARS_828_verify_installation.sql new file mode 100644 index 0000000..c4a4f6e --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/05_MARS_828_verify_installation.sql @@ -0,0 +1,122 @@ +-- MARS-828: Verify installation +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 + +PROMPT ======================================== +PROMPT MARS-828: Verification Script +PROMPT ======================================== + +SET SERVEROUTPUT ON SIZE UNLIMITED + +-- 1. Verify columns added +PROMPT +PROMPT 1. Verifying A_SOURCE_FILE_CONFIG columns... +SELECT + column_name, + data_type, + nullable, + data_default +FROM all_tab_columns +WHERE owner = 'CT_MRDS' + AND table_name = 'A_SOURCE_FILE_CONFIG' + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS') +ORDER BY column_id; + +-- 2. Verify constraint +PROMPT +PROMPT 2. Verifying check constraint... +SELECT + constraint_name, + constraint_type, + search_condition +FROM all_constraints +WHERE owner = 'CT_MRDS' + AND table_name = 'A_SOURCE_FILE_CONFIG' + AND constraint_name = 'CHK_ARCHIVAL_STRATEGY'; + +-- 3. Verify trigger +PROMPT +PROMPT 3. Verifying validation trigger... +SELECT + trigger_name, + status, + trigger_type +FROM all_triggers +WHERE owner = 'CT_MRDS' + AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL'; + +-- 4. Check package compilation status +PROMPT +PROMPT 4. Checking FILE_ARCHIVER package status... +SELECT + object_name, + object_type, + status, + TO_CHAR(last_ddl_time, 'YYYY-MM-DD HH24:MI:SS') as last_ddl_time +FROM all_objects +WHERE owner = 'CT_MRDS' + AND object_name = 'FILE_ARCHIVER' + AND object_type IN ('PACKAGE', 'PACKAGE BODY') +ORDER BY object_type; + +-- 5. Check for compilation errors +PROMPT +PROMPT 5. Checking for compilation errors... +SELECT + name, + type, + line, + position, + text +FROM all_errors +WHERE owner = 'CT_MRDS' + AND name = 'FILE_ARCHIVER' +ORDER BY type, sequence; + +-- 6. Verify package version +PROMPT +PROMPT 6. Verifying FILE_ARCHIVER version... +SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() as package_version FROM DUAL; + +-- 7. Test trigger validation +PROMPT +PROMPT 7. Testing trigger validation (should fail)... +DECLARE + vTestPassed BOOLEAN := FALSE; +BEGIN + -- This should fail - MINIMUM_AGE_MONTHS strategy without value + INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG ( + A_SOURCE_FILE_CONFIG_KEY, + A_SOURCE_KEY, + SOURCE_FILE_TYPE, + SOURCE_FILE_ID, + ARCHIVAL_STRATEGY, + MINIMUM_AGE_MONTHS + ) VALUES ( + -999, + 'TEST', + 'INPUT', + 'TEST', + 'MINIMUM_AGE_MONTHS', + NULL -- Should trigger error + ); + + ROLLBACK; + DBMS_OUTPUT.PUT_LINE('ERROR: Trigger validation did not fire!'); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20999 THEN + DBMS_OUTPUT.PUT_LINE('SUCCESS: Trigger validation working correctly'); + DBMS_OUTPUT.PUT_LINE('Expected error: ' || SQLERRM); + vTestPassed := TRUE; + ELSE + DBMS_OUTPUT.PUT_LINE('ERROR: Unexpected error: ' || SQLERRM); + END IF; + ROLLBACK; +END; +/ + +PROMPT +PROMPT ======================================== +PROMPT MARS-828: Verification Complete +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql new file mode 100644 index 0000000..204e834 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql @@ -0,0 +1,11 @@ +-- =================================================================== +-- MARS-828: Rollback FILE_ARCHIVER Package Specification to v2.0.0 +-- =================================================================== +-- Purpose: Restore previous package specification version (pre-MARS-828) +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- WARNING: This removes all MARS-828 version information updates +-- =================================================================== + +@@rollback/FILE_ARCHIVER.pkg + diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql new file mode 100644 index 0000000..9004a2b --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql @@ -0,0 +1,11 @@ +-- =================================================================== +-- MARS-828: Rollback FILE_ARCHIVER Package Body to v2.0.0 +-- =================================================================== +-- Purpose: Restore previous package body version (pre-MARS-828) +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- WARNING: This removes all MARS-828 archival strategy enhancements +-- =================================================================== + +@@rollback/FILE_ARCHIVER.pkb + diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/93_MARS_828_rollback_trigger.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/93_MARS_828_rollback_trigger.sql new file mode 100644 index 0000000..0289cd4 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/93_MARS_828_rollback_trigger.sql @@ -0,0 +1,20 @@ +-- MARS-828: Rollback validation trigger +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Description: Drop archival strategy validation trigger + +PROMPT ======================================== +PROMPT MARS-828: Dropping archival strategy validation trigger +PROMPT ======================================== + +DROP TRIGGER CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL; + +-- Verify trigger dropped +SELECT COUNT(*) as trigger_count +FROM all_triggers +WHERE owner = 'CT_MRDS' + AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL'; + +PROMPT ======================================== +PROMPT Validation trigger dropped successfully +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql new file mode 100644 index 0000000..fc481eb --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql @@ -0,0 +1,30 @@ +-- MARS-828: Rollback archival strategy columns +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Description: Remove ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns + +PROMPT ======================================== +PROMPT MARS-828: Removing archival strategy columns +PROMPT ======================================== + +-- Drop check constraint first +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG +DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY; + +-- Drop columns +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP ( + ARCHIVAL_STRATEGY, + MINIMUM_AGE_MONTHS +); + +-- Verify columns dropped +SELECT + column_name +FROM all_tab_columns +WHERE owner = 'CT_MRDS' + AND table_name = 'A_SOURCE_FILE_CONFIG' + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS'); + +PROMPT ======================================== +PROMPT Archival strategy columns removed successfully +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md b/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md new file mode 100644 index 0000000..8d4aa72 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md @@ -0,0 +1,269 @@ +# MARS-828: Enhanced Archival Strategies + +## Overview + +Implementation of flexible archival strategies for FILE_ARCHIVER package to support business requirements for different data retention policies across source systems. + +### Background + +**Issue**: Current FILE_ARCHIVER v2.0.0 uses fixed threshold-based archival (DAYS_FOR_ARCHIVE_THRESHOLD) which cannot accommodate: +1. **LM/TOP sources**: Archive all data except current month +2. **CSDB sources**: Archive only data older than 6 months + +**Root Cause**: Hardcoded WHERE clause in ARCHIVE_TABLE_DATA procedure: +```sql +WHERE extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRESHOLD +``` + +**Solution**: Introduce ARCHIVAL_STRATEGY configuration column with four strategies: +- `THRESHOLD_BASED` - Days-based threshold (backward compatible) +- `CURRENT_MONTH_ONLY` - Keep only current month data +- `MINIMUM_AGE_MONTHS` - Archive data older than X months +- `HYBRID` - Combination of current month and minimum age + +## Changes Made + +### Database Schema Changes +**Table**: CT_MRDS.A_SOURCE_FILE_CONFIG + +**Before**: +```sql +-- Only threshold-based configuration available +DAYS_FOR_ARCHIVE_THRESHOLD NUMBER DEFAULT 30 +``` + +**After**: +```sql +-- Flexible strategy-based configuration +ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL, +MINIMUM_AGE_MONTHS NUMBER(3), +DAYS_FOR_ARCHIVE_THRESHOLD NUMBER DEFAULT 30, +CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK (ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID')) +``` + +**New Trigger**: TRG_BI_ARCHIVAL_STRATEGY_VAL +- Validates MINIMUM_AGE_MONTHS required for strategies that need it +- Ensures data integrity before insert/update + +### Package Changes +**Package**: CT_MRDS.FILE_ARCHIVER + +**Version**: 2.0.0 → 3.0.0 (MAJOR - breaking changes to internal logic) + +**Specification Changes**: +- Updated PACKAGE_VERSION: '2.0.0' → '3.0.0' +- Updated VERSION_HISTORY with MARS-828 entry +- Updated PACKAGE_BUILD_DATE to deployment date + +**Body Changes**: +1. **New Function**: GET_ARCHIVAL_WHERE_CLAUSE(pSourceFileConfig) - Returns strategy-specific WHERE clause +2. **Updated**: ARCHIVE_TABLE_DATA - Uses GET_ARCHIVAL_WHERE_CLAUSE for flexible filtering +3. **Updated**: GATHER_TABLE_STAT - Uses GET_ARCHIVAL_WHERE_CLAUSE for statistics calculation + +**File Structure**: +- **rollback/** - Backup of FILE_ARCHIVER v2.0.0 (for rollback) +- **new_version/** - Updated FILE_ARCHIVER v3.0.0 (with strategy support) + +## Archival Strategies + +| Strategy | WHERE Clause Logic | Configuration Required | Use Case | +|----------|-------------------|----------------------|----------| +| `THRESHOLD_BASED` | `extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRESHOLD` | DAYS_FOR_ARCHIVE_THRESHOLD | Legacy compatibility | +| `CURRENT_MONTH_ONLY` | `TRUNC(workflow_start, 'MM') < TRUNC(SYSDATE, 'MM')` | None | General sources (LM, TOP) | +| `MINIMUM_AGE_MONTHS` | `workflow_start < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -X)` | MINIMUM_AGE_MONTHS | CSDB (6 months retention) | +| `HYBRID` | Both CURRENT_MONTH_ONLY AND MINIMUM_AGE_MONTHS | MINIMUM_AGE_MONTHS | Advanced scenarios | + +## Configuration Examples + +```sql +-- LM/TOP sources: Archive everything except current month +UPDATE A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', + MINIMUM_AGE_MONTHS = NULL +WHERE A_SOURCE_KEY IN ('LM', 'TOP'); + +-- CSDB: Archive only data older than 6 months +UPDATE A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 6 +WHERE A_SOURCE_KEY = 'CSDB' + AND TABLE_ID IN ('DEBT', 'DEBT_DAILY'); + +-- Legacy sources: Keep existing threshold behavior +UPDATE A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = 'THRESHOLD_BASED', + DAYS_FOR_ARCHIVE_THRESHOLD = 30 +WHERE A_SOURCE_KEY = 'C2D'; + +-- Advanced hybrid: Current month + 3 months minimum +UPDATE A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = 'HYBRID', + MINIMUM_AGE_MONTHS = 3 +WHERE A_SOURCE_KEY = 'SPECIAL'; +``` + +## Deployment + +### Prerequisites +- **User**: ADMIN with full privileges +- **Database**: Oracle 23ai +- **ENV_MANAGER**: v3.x or higher +- **FILE_MANAGER**: v3.x or higher +- **FILE_ARCHIVER**: v2.0.0 (will upgrade to v3.0.0) +- **Table**: A_SOURCE_FILE_CONFIG must exist + +### Installation Steps + +**Option 1: Master Script (Recommended)** +```powershell +# Navigate to MARS-828 directory +cd .\MARS_Packages\REL01_ADDITIONS\MARS-828 + +# Execute installation (requires ADMIN user) +sql "ADMIN/Cloudpass#34@ggmichalski_high" "@install_mars828.sql" + +# Log file created: log/INSTALL_MARS_828__.log +``` + +**Installation Workflow**: +1. **Add Columns** - ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS to A_SOURCE_FILE_CONFIG +2. **Create Trigger** - Validation trigger TRG_BI_ARCHIVAL_STRATEGY_VAL +3. **Deploy Package Spec** - FILE_ARCHIVER v3.0.0 specification +4. **Deploy Package Body** - FILE_ARCHIVER v3.0.0 body with GET_ARCHIVAL_WHERE_CLAUSE +5. **Verify Installation** - Check package compilation and structure +6. **Track Version** - Record v3.0.0 in ENV_MANAGER +7. **Verify Packages** - Check for untracked changes using ENV_MANAGER hash verification + +**Option 2: Using Get-Content** +```powershell +Get-Content "MARS_Packages\REL01_ADDITIONS\MARS-828\install_mars828.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high" +``` + +### Verification +```sql +-- Check package compilation status +SELECT object_name, object_type, status +FROM all_objects +WHERE owner = 'CT_MRDS' + AND object_name = 'FILE_ARCHIVER' + AND object_type IN ('PACKAGE', 'PACKAGE BODY'); + +-- Verify package version +SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() FROM DUAL; +-- Expected: 3.0.0 + +-- Check new columns exist +SELECT column_name, data_type, data_default +FROM all_tab_columns +WHERE owner = 'CT_MRDS' + AND table_name = 'A_SOURCE_FILE_CONFIG' + AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS'); + +-- Test strategy configuration +SELECT A_SOURCE_KEY, TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS +FROM CT_MRDS.A_SOURCE_FILE_CONFIG +ORDER BY A_SOURCE_KEY, TABLE_ID; +``` + +### Rollback +```powershell +# Execute rollback script (requires ADMIN user) +cd .\MARS_Packages\REL01_ADDITIONS\MARS-828 +sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars828.sql" + +# Log file created: log/ROLLBACK_MARS_828__.log +``` + +**Rollback Workflow**: +1. **Restore Package Body** - FILE_ARCHIVER v2.0.0 body +2. **Restore Package Spec** - FILE_ARCHIVER v2.0.0 specification +3. **Drop Trigger** - Remove TRG_BI_ARCHIVAL_STRATEGY_VAL +4. **Drop Columns** - Remove ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS +5. **Track Rollback** - Record v2.0.0 restoration in ENV_MANAGER +6. **Verify Packages** - Check package hash consistency + +## Testing + +**Test Scripts**: Located in `test/` folder + +**Main Test Script**: test/test_archival_strategies.sql + +```sql +-- Test 1: CURRENT_MONTH_ONLY strategy +-- Expected: Archives data from previous months only +SELECT COUNT(*) FROM table WHERE TRUNC(workflow_start, 'MM') < TRUNC(SYSDATE, 'MM'); + +-- Test 2: MINIMUM_AGE_MONTHS strategy (6 months) +-- Expected: Archives data older than 6 months +SELECT COUNT(*) FROM table WHERE workflow_start < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -6); + +-- Test 3: HYBRID strategy +-- Expected: Archives data from previous months AND older than X months +``` + +## Dependencies + +### Required Packages +- **CT_MRDS.ENV_MANAGER** v3.x - Error handling, logging, version tracking +- **CT_MRDS.FILE_MANAGER** v3.x - Bucket URI resolution +- **MRDS_LOADER.cloud_wrapper** - DBMS_CLOUD operations + +### Database Objects +- **Table**: CT_MRDS.A_SOURCE_FILE_CONFIG - Configuration storage +- **Table**: CT_ODS.A_LOAD_HISTORY - Workflow tracking +- **Credential**: DEF_CRED_ARN - OCI bucket access + +## Files Included + +1. **README.md** - This documentation file +2. **.gitignore** - Git exclusions (confluence/, log/, test/, mock_data/) +3. **install_mars828.sql** - Master installation script with SPOOL logging (7 steps) +4. **rollback_mars828.sql** - Master rollback script (6 steps) +5. **01_MARS_828_install_add_archival_strategy_columns.sql** - ALTER TABLE DDL +6. **02_MARS_828_install_archival_strategy_trigger.sql** - CREATE TRIGGER DDL +7. **03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql** - Deploy package specification v3.0.0 +8. **04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql** - Deploy package body v3.0.0 +9. **05_MARS_828_verify_installation.sql** - Verification queries +10. **91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql** - Restore package specification v2.0.0 +11. **92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql** - Restore package body v2.0.0 +12. **93_MARS_828_rollback_trigger.sql** - DROP TRIGGER +13. **94_MARS_828_rollback_columns.sql** - ALTER TABLE DROP COLUMN +14. **track_package_versions.sql** - Universal version tracking script (Standard) +15. **verify_packages_version.sql** - Universal package verification script (Standard) +16. **test/** - Test scripts and validation scenarios +17. **rollback/** - Backup of FILE_ARCHIVER v2.0.0 (for rollback) +18. **new_version/** - Updated FILE_ARCHIVER v3.0.0 (deployment source) + +## Impact Analysis + +### Backward Compatibility +✅ **FULLY BACKWARD COMPATIBLE**: Default ARCHIVAL_STRATEGY = 'THRESHOLD_BASED' maintains existing behavior for all sources without configuration changes. + +### Affected Procedures +1. **ARCHIVE_TABLE_DATA** - Uses GET_ARCHIVAL_WHERE_CLAUSE for WHERE clause generation +2. **GATHER_TABLE_STAT** - Uses GET_ARCHIVAL_WHERE_CLAUSE for statistics calculation + +### Configuration Migration +No automatic migration required. New columns have sensible defaults: +- ARCHIVAL_STRATEGY = 'THRESHOLD_BASED' (maintains current behavior) +- MINIMUM_AGE_MONTHS = NULL (not required for THRESHOLD_BASED) + +## Version History +- **v3.0.0** (2026-01-27): Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration +- **v2.0.0** (2025-10-01): Initial FILE_ARCHIVER package with threshold-based archival + +## Related JIRA Issues +- **MARS-828**: Enhanced Archival Strategies implementation + +## Author +Created by: Grzegorz Michalski +Date: 2026-01-27 +Schema: CT_MRDS +Package: FILE_ARCHIVER + + +Grzegorz Michalski + +## Date + +2026-01-27 diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql new file mode 100644 index 0000000..319bf02 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql @@ -0,0 +1,116 @@ +-- ============================================================================ +-- MARS-828 Master Installation Script +-- ============================================================================ +-- Purpose: Deploy enhanced archival strategies for FILE_ARCHIVER package +-- Target Schema: CT_MRDS +-- Estimated Time: 2-3 minutes +-- Prerequisites: FILE_ARCHIVER v2.0.0, ENV_MANAGER v3.x, ADMIN privileges +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK ON +SET ECHO OFF + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET VERIFY OFF +SET FEEDBACK ON +SET ECHO OFF + +-- Create log directory if it doesn't exist +host mkdir log 2>nul + +-- Generate dynamic SPOOL filename with timestamp +var filename VARCHAR2(100) +BEGIN + :filename := 'log/INSTALL_MARS_828_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-828 Installation Starting +PROMPT ============================================================================ +PROMPT Package: CT_MRDS.FILE_ARCHIVER +PROMPT Change: Enhanced archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) +PROMPT Purpose: Flexible archival policies per data source +PROMPT Steps: 7 (DDL, Trigger, Packages, Verify, Track, Verify) +PROMPT Timestamp: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; +PROMPT ============================================================================ + +-- Confirm installation with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20001, 'Installation aborted by user'); + END IF; +END; +/ +WHENEVER SQLERROR CONTINUE + +-- Installation steps +PROMPT7: Adding archival strategy columns to A_SOURCE_FILE_CONFIG +PROMPT =================================================================== +@@01_MARS_828_install_add_archival_strategy_columns.sql + +PROMPT +PROMPT Step 2/7: Creating validation trigger +PROMPT ====================================== +@@02_MARS_828_install_archival_strategy_trigger.sql + +PROMPT +PROMPT Step 3/7: Deploying FILE_ARCHIVER Package Specification v3.0.0 +PROMPT =============================================================== +@@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql + +PROMPT +PROMPT Step 4/7: Deploying FILE_ARCHIVER Package Body v3.0.0 +PROMPT ====================================================== +@@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql + +PROMPT +PROMPT Step 5/7: Verifying installation +PROMPT ================================= +@@05_MARS_828_verify_installation.sql + +PROMPT +PROMPT Step 6/7: Tracking package versions +PROMPT ==================================== +@@track_package_versions.sql + +PROMPT +PROMPT Step 7/7: Verifying tracked packages +PROMPT ===================================== +@@verify_packages_version.sql + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-828 Installation Completed +PROMPT ============================================================================ +PROMPT Completion Time: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL; +PROMPT +PROMPT Installation Summary: +PROMPT - Package: CT_MRDS.FILE_ARCHIVER +PROMPT - Version: 2.0.0 -> 3.0.0 (MAJOR) +PROMPT - New Strategies: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID +PROMPT - Backward Compatible: THRESHOLD_BASED (default) +PROMPT +PROMPT Next Steps: +PROMPT 1. Configure archival strategies per source: +PROMPT UPDATE A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY' WHERE A_SOURCE_KEY = 'LM'; +PROMPT UPDATE A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', MINIMUM_AGE_MONTHS = 6 WHERE A_SOURCE_KEY = 'CSDB'; +PROMPT 2. Test strategies using test_archival_strategies.sql +PROMPT 3. Monitor first archival run +PROMPT +PROMPT Log file: &_filename +PROMPT ============================================================================ + +spool off + +quit; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..e9f83f7 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb @@ -0,0 +1,537 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE + ---------------------------------------------------------------------------------------------------- + /** + * @name GET_ARCHIVAL_WHERE_CLAUSE + * @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration. + * Supports four strategies: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. + * @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY + * @return VARCHAR2 - WHERE clause for filtering archival candidates + **/ + FUNCTION GET_ARCHIVAL_WHERE_CLAUSE( + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) RETURN VARCHAR2 + IS + vWhereClause VARCHAR2(4000); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + CASE pSourceFileConfig.ARCHIVAL_STRATEGY + -- Legacy threshold-based strategy (backward compatible) + WHEN 'THRESHOLD_BASED' THEN + vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD; + + -- Archive all data except current month + WHEN 'CURRENT_MONTH_ONLY' THEN + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'')'; + + -- Archive only data older than X months + WHEN 'MINIMUM_AGE_MONTHS' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); + END IF; + vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + -- Hybrid: Current month exclusion AND minimum age requirement + WHEN 'HYBRID' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy'); + END IF; + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' || + 'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + ELSE + RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY); + END CASE; + + RETURN vWhereClause; + END GET_ARCHIVAL_WHERE_CLAUSE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + + -- Use strategy-based WHERE clause (MARS-828) + vQuery := ' + select t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* +-- ,r.partition_year +-- ,r.partition_month + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + and r.partition_year='''||ym_loop.year||''' + and r.partition_month='''||ym_loop.month||''' + and r.PROCESSING_STATUS = ''INGESTED'' + ' + ; + vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + SELECT + object_name + into vFilename + from DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vUri) + where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7')) + between vUserLoadOperations.START_TIME + and vUserLoadOperations.UPDATE_TIME + ; + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED' + ,ARCH_FILE_NAME = vUri||vFilename + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + + -- move file to trash before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART) + IF vProcessControlStatus = 'OK' THEN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + --Drop file from TRASH + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename); + END LOOP; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS = 'ARCHIVED' + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_FILE_NAME = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + + ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + + -- Use strategy-based WHERE clause for statistics (MARS-828) + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then r.bytes else 0 end) as OLD_BYTES + ,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||ENV_MANAGER.gvCredentialName||''', + location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/'' + ) + ) r + on t.filename = r.object_name' + ; + ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into A_TABLE_STAT_HIST values vStats; + delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into A_TABLE_STAT values vStats; + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + ---- + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..2ff8ff2 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg @@ -0,0 +1,116 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.1.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-01-29 21:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Function overload for ARCHIVE_TABLE_DATA procedure. + * Returns SQLCODE for Python library integration. + * Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result. + * @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + /** + * @name GATHER_TABLE_STAT + * @desc Function overload for GATHER_TABLE_STAT procedure. + * Returns SQLCODE for Python library integration. + * Calls the main GATHER_TABLE_STAT procedure and captures execution result. + * @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) RETURN PLS_INTEGER; + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql new file mode 100644 index 0000000..d25647a --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql @@ -0,0 +1,40 @@ +-- =================================================================== +-- MARS-828: Archival Strategy Validation Trigger +-- =================================================================== +-- Purpose: Validates archival strategy configuration consistency +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Version: 1.0.0 +-- =================================================================== + +CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL +BEFORE INSERT OR UPDATE ON CT_MRDS.A_SOURCE_FILE_CONFIG +FOR EACH ROW +DECLARE + vErrorMsg VARCHAR2(1000); +BEGIN + -- Validate MINIMUM_AGE_MONTHS required for specific strategies + IF :NEW.ARCHIVAL_STRATEGY IN ('MINIMUM_AGE_MONTHS', 'HYBRID') + AND :NEW.MINIMUM_AGE_MONTHS IS NULL THEN + vErrorMsg := 'MINIMUM_AGE_MONTHS is required for ' || :NEW.ARCHIVAL_STRATEGY || ' strategy'; + RAISE_APPLICATION_ERROR(-20999, vErrorMsg); + END IF; + + -- Validate MINIMUM_AGE_MONTHS is positive + IF :NEW.MINIMUM_AGE_MONTHS IS NOT NULL + AND :NEW.MINIMUM_AGE_MONTHS < 1 THEN + RAISE_APPLICATION_ERROR(-20998, 'MINIMUM_AGE_MONTHS must be greater than 0'); + END IF; + + -- Warn if MINIMUM_AGE_MONTHS set but strategy doesn't use it + IF :NEW.MINIMUM_AGE_MONTHS IS NOT NULL + AND :NEW.ARCHIVAL_STRATEGY NOT IN ('MINIMUM_AGE_MONTHS', 'HYBRID') THEN + -- This is just a warning scenario - allow but could log + NULL; + END IF; + +EXCEPTION + WHEN OTHERS THEN + RAISE; +END; +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..83edcf3 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkb @@ -0,0 +1,443 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + vQuery := ' + select t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* +-- ,r.partition_year +-- ,r.partition_month + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + and r.partition_year='''||ym_loop.year||''' + and r.partition_month='''||ym_loop.month||''' + and r.PROCESSING_STATUS = ''INGESTED'' + ' + ; + vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + SELECT + object_name + into vFilename + from DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vUri) + where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7')) + between vUserLoadOperations.START_TIME + and vUserLoadOperations.UPDATE_TIME + ; + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED' + ,ARCH_FILE_NAME = vUri||vFilename + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + + -- move file to trash before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART) + IF vProcessControlStatus = 'OK' THEN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + --Drop file from TRASH + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename); + END LOOP; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS = 'ARCHIVED' + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_FILE_NAME = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + + ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then r.bytes else 0 end) as OLD_BYTES + ,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||ENV_MANAGER.gvCredentialName||''', + location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/'' + ) + ) r + on t.filename = r.object_name' + ; + ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into A_TABLE_STAT_HIST values vStats; + delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into A_TABLE_STAT values vStats; + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..5185456 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v2.0.0/FILE_ARCHIVER.pkg @@ -0,0 +1,90 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.0.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-10-22 16:45:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkb new file mode 100644 index 0000000..66cd24b --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkb @@ -0,0 +1,495 @@ +create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER +AS + + ---------------------------------------------------------------------------------------------------- + -- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE + ---------------------------------------------------------------------------------------------------- + /** + * @name GET_ARCHIVAL_WHERE_CLAUSE + * @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration. + * Supports four strategies: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. + * @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY + * @return VARCHAR2 - WHERE clause for filtering archival candidates + **/ + FUNCTION GET_ARCHIVAL_WHERE_CLAUSE( + pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE + ) RETURN VARCHAR2 + IS + vWhereClause VARCHAR2(4000); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + CASE pSourceFileConfig.ARCHIVAL_STRATEGY + -- Legacy threshold-based strategy (backward compatible) + WHEN 'THRESHOLD_BASED' THEN + vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD; + + -- Archive all data except current month + WHEN 'CURRENT_MONTH_ONLY' THEN + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'')'; + + -- Archive only data older than X months + WHEN 'MINIMUM_AGE_MONTHS' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); + END IF; + vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + -- Hybrid: Current month exclusion AND minimum age requirement + WHEN 'HYBRID' THEN + IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN + RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy'); + END IF; + vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' || + 'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')'; + + ELSE + RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY); + END CASE; + + RETURN vWhereClause; + END GET_ARCHIVAL_WHERE_CLAUSE; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) + RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE + IS + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vCount PLS_INTEGER; + vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); + SELECT count(*) , min(SOURCE_FILE_TYPE) + INTO vCount, vSourceFileType + FROM CT_MRDS.A_TABLE_STAT s + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c + ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + + IF vCount=0 and vSourceFileType='INPUT' THEN + GATHER_TABLE_STAT(pSourceFileConfigKey); + END IF; + + BEGIN + SELECT * + INTO vTableStat + FROM CT_MRDS.A_TABLE_STAT + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; +-- EXCEPTION +-- WHEN NO_DATA_FOUND THEN +-- + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); + RETURN vTableStat; + + END GET_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE; + vQuery VARCHAR2(4000); + vTableName VARCHAR2(200); + vUri VARCHAR2(1000); + vfiles T_FILENAMES; + vFilename VARCHAR2(300); + vOperationId NUMBER := -1; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM + vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; + vProcessControlStatus VARCHAR2(60) := 'OK'; + + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + + if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE); + end if; + + if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then + GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey); + end if; + + if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT'; + elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM'; + else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO'); + end if; + + if LENGTH(vArchivalTriggeredBy)>0 THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO'); + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + + -- Use strategy-based WHERE clause (MARS-828) + vQuery := ' + select t_filename( + file$name + ,file$path + , to_char(h.workflow_start,''yyyy'') + , to_char(h.workflow_start,''mm'') + ) + + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) + ; + + -- Get all files that will be archived into "vfiles" collection ("regular data files") + execute immediate vQuery bulk collect into vfiles; + + -- Start EXPORT "regular data files" to parquet and DROP "csv" + FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP + dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month); + vQuery:= + 'select + s.* +-- ,r.partition_year +-- ,r.partition_month + from '|| vTableName ||' s + join CT_MRDS.A_SOURCE_FILE_RECEIVED r + on s.file$name = r.source_file_name + and r.a_source_file_config_key = '||pSourceFileConfigKey||' + and r.partition_year='''||ym_loop.year||''' + and r.partition_month='''||ym_loop.month||''' + and r.PROCESSING_STATUS = ''INGESTED'' + ' + ; + vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; + + ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri); + ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery); + + + + BEGIN + DBMS_CLOUD.EXPORT_DATA( + credential_name => ENV_MANAGER.gvCredentialName, + file_uri_list => vUri||'d' , + format => json_object('type' value 'parquet'), + query => vQuery, + operation_id => vOperationId + ); + EXCEPTION + WHEN OTHERS THEN + vProcessControlStatus :='EXPORT_FAILURE'; + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED); + + END; + + ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG'); + + -- Get USER_LOAD_OPERATIONS info + select * + into vUserLoadOperations + from USER_LOAD_OPERATIONS + where id = vOperationId; + + IF vUserLoadOperations.STATUS <>'COMPLETED' THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS); + ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, + ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.'); + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters); + END IF; + + SELECT + object_name + into vFilename + from DBMS_CLOUD.LIST_OBJECTS( + credential_name => 'OCI$RESOURCE_PRINCIPAL', + location_uri => vUri) + where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7')) + between vUserLoadOperations.START_TIME + and vUserLoadOperations.UPDATE_TIME + ; + + -- Try to drop EXPORTED FILES ("regular data files") + BEGIN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop + + -- first change of status + BEGIN + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'ARCHIVED' + ,ARCH_FILE_NAME = vUri||vFilename + WHERE r.a_source_file_config_key= pSourceFileConfigKey + AND r.source_file_name = f.filename + AND r.processing_status = 'INGESTED' + ; + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE'; + + -- move file to trash before dropping + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => f.pathname||'/'||f.filename, + target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + rollback; + vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE'; + END; + EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE'; + commit; + END LOOP; + + -------------------------------------------------------------------- + -- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART) + IF vProcessControlStatus = 'OK' THEN + FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP + --Drop file from TRASH + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename); + END LOOP; + + --ROLLBACK PART + --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH) + ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN + FOR f in ( SELECT vf.filename, vf.pathname + FROM TABLE(vfiles) vf + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r + ON r.source_file_name = vf.filename + AND r.a_source_file_config_key = pSourceFileConfigKey + AND r.PROCESSING_STATUS = 'ARCHIVED' + AND vf.year = ym_loop.year + AND vf.month = ym_loop.month + ) LOOP + BEGIN + DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, + source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename, + target_object_uri => f.pathname||'/'||f.filename, + target_credential_name => ENV_MANAGER.gvCredentialName + ); + ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename); + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r + SET PROCESSING_STATUS = 'INGESTED' + ,ARCH_FILE_NAME = NULL + WHERE r.a_source_file_config_key = pSourceFileConfigKey + AND r.source_file_name = f.filename + ; + + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE'; + END; + END LOOP; + + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters); + DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName, + object_uri => vUri||vFilename); + ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR'); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + END IF; + + EXCEPTION + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED); + + WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR'); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED); + END; + -- END of "Try to drop EXPORTED FILES" + + ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO'); + ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO'); + END LOOP; --ym_loop end (YEAR_MONTH) + + COMMIT; + + ELSE + ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO'); + END IF; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + + END ARCHIVE_TABLE_DATA; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ) IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vStats CT_MRDS.A_TABLE_STAT%ROWTYPE; + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vTableName VARCHAR2(200); + vQuery VARCHAR2(32000); + BEGIN + vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); + ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS'; + ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName); + + -- Use strategy-based WHERE clause for statistics (MARS-828) + vQuery := + 'with tmp as ( + select + s.* + ,file$name as filename + ,h.workflow_start + , to_char(h.workflow_start,''yyyy'') as year + , to_char(h.workflow_start,''mm'') as month + from '||vTableName||' s + join CT_MRDS.a_workflow_history h + on s.a_workflow_history_key = h.a_workflow_history_key + ) + , tmp_gr as ( + select + filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + from tmp + group by filename + ) + select + NULL as A_TABLE_STAT_KEY + ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY + ,'''||vTableName||''' as TABLE_NAME + ,count(*) as FILE_COUNT + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then 1 else 0 end) as OLD_FILE_COUNT + ,sum (row_count_per_file) as ROW_COUNT + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT + ,sum(r.bytes) as BYTES + ,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then r.bytes else 0 end) as OLD_BYTES + ,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD + ,systimestamp as CREATED + from tmp_gr t + join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( + credential_name => '''||ENV_MANAGER.gvCredentialName||''', + location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/'' + ) + ) r + on t.filename = r.object_name' + ; + ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery); + execute immediate vQuery into vStats; + + vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL; + insert into A_TABLE_STAT_HIST values vStats; + delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; + insert into A_TABLE_STAT values vStats; + COMMIT; + + ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + EXCEPTION + WHEN OTHERS THEN + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END GATHER_TABLE_STAT; + + ---------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION + RETURN VARCHAR2 + IS + BEGIN + RETURN PACKAGE_VERSION; + END GET_VERSION; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_BUILD_INFO + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( + pPackageName => 'FILE_ARCHIVER', + pVersion => PACKAGE_VERSION, + pBuildDate => PACKAGE_BUILD_DATE, + pAuthor => PACKAGE_AUTHOR + ); + END GET_BUILD_INFO; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION GET_VERSION_HISTORY + RETURN VARCHAR2 + IS + BEGIN + RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( + pPackageName => 'FILE_ARCHIVER', + pVersionHistory => VERSION_HISTORY + ); + END GET_VERSION_HISTORY; + + ---------------------------------------------------------------------------------------------------- + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkg new file mode 100644 index 0000000..e7267d9 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback/v3.0.0/FILE_ARCHIVER.pkg @@ -0,0 +1,91 @@ +create or replace PACKAGE CT_MRDS.FILE_ARCHIVER +AUTHID CURRENT_USER +AS + /** + * General comment for package: Please put comments for functions and procedures as shown in below example. + * It is a standard. + * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function + * which returns documentation text for confluence page (to Copy-Paste it). + **/ + + -- Example comment: + /** + * @name EX_PROCEDURE_NAME + * @desc Procedure description + * @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual; + * @ex_rslt Example Result + **/ + + -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.0.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-01-27 12:00:00'; + PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; + + -- Version History (Latest changes first) + VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || + '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || + '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; + + cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL; + + /** + * @name ARCHIVE_TABLE_DATA + * @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA. + * Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure. + * Each YEAR_MONTH pair goes to seperate file (implicit partitioning). + **/ + PROCEDURE ARCHIVE_TABLE_DATA ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + + + /** + * @name GATHER_TABLE_STAT + * @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY). + * Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST. + **/ + PROCEDURE GATHER_TABLE_STAT ( + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE + ); + + --------------------------------------------------------------------------------------------------------------------------- + -- PACKAGE VERSION MANAGEMENT FUNCTIONS + --------------------------------------------------------------------------------------------------------------------------- + + /** + * @name GET_VERSION + * @desc Returns the current version number of the FILE_ARCHIVER package. + * Uses semantic versioning format (MAJOR.MINOR.PATCH). + * @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL; + * @ex_rslt 2.0.0 + **/ + FUNCTION GET_VERSION RETURN VARCHAR2; + + /** + * @name GET_BUILD_INFO + * @desc Returns comprehensive build information including version, build date, and author. + * Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function. + * @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL; + * @ex_rslt Package: FILE_ARCHIVER + * Version: 2.0.0 + * Build Date: 2025-10-22 16:45:00 + * Author: Grzegorz Michalski + **/ + FUNCTION GET_BUILD_INFO RETURN VARCHAR2; + + /** + * @name GET_VERSION_HISTORY + * @desc Returns complete version history with all releases and changes. + * Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function. + * @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL; + * @ex_rslt FILE_ARCHIVER Version History: + * 2.0.0 (2025-10-22): Added package versioning system... + **/ + FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2; + +END; + +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql new file mode 100644 index 0000000..86bd778 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql @@ -0,0 +1,114 @@ +-- =================================================================== +-- MARS-828: Enhanced Archival Strategies Rollback +-- =================================================================== +-- Purpose: Rollback all changes from MARS-828 installation +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 + +-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required) +-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir) +host mkdir log 2>nul + +var filename VARCHAR2(100) +BEGIN + :filename := 'log/ROLLBACK_MARS_828_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log'; +END; +/ +column filename new_value _filename +select :filename filename from dual; +spool &_filename + +SET ECHO OFF +SET TIMING ON +SET SERVEROUTPUT ON SIZE UNLIMITED +SET LINESIZE 200 +SET PAGESIZE 1000 +SET PAUSE OFF + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-828 Rollback Starting +PROMPT ============================================================================ +PROMPT WARNING: This will restore FILE_ARCHIVER to v2.0.0 +PROMPT +PROMPT CRITICAL IMPACT: +PROMPT 1. All archival strategies revert to THRESHOLD_BASED +PROMPT 2. ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns will be dropped +PROMPT 3. Validation trigger will be removed +PROMPT 4. Reconfigure archival thresholds after rollback +PROMPT +PROMPT Timestamp: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; +PROMPT ============================================================================ + +-- Confirm rollback with user +ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: ' +WHENEVER SQLERROR EXIT SQL.SQLCODE +BEGIN + IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN + RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user'); + END IF; +END; +/ +WHENEVER SQLERROR CONTINUE + +-- Rollback steps (in reverse order) +PROMPT +PROMPT Step 1/6: Restoring FILE_ARCHIVER Package Specification v2.0.0 +PROMPT =============================================================== +@@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql + +PROMPT +PROMPT Step 2/6: Restoring FILE_ARCHIVER Package Body v2.0.0 +PROMPT ====================================================== +@@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql + +PROMPT +PROMPT Step 3/6: Dropping validation trigger +PROMPT ====================================== +@@93_MARS_828_rollback_trigger.sql + +PROMPT +PROMPT Step 4/6: Dropping archival strategy columns +PROMPT ============================================= +@@94_MARS_828_rollback_columns.sql + +PROMPT +PROMPT Step 5/6: Tracking rollback version +PROMPT ==================================== +@@track_package_versions.sql + +PROMPT +PROMPT Step 6/6: Verifying tracked packages +PROMPT ===================================== +@@verify_packages_version.sql + +-- Verify rollback +PROMPT +PROMPT Verification: Package Compilation Status +PROMPT ========================================= +SELECT object_name, object_type, status, last_ddl_time +FROM all_objects +WHERE owner = 'CT_MRDS' + AND object_name = 'FILE_ARCHIVER' + AND object_type IN ('PACKAGE', 'PACKAGE BODY') +ORDER BY object_type; + +PROMPT +PROMPT ============================================================================ +PROMPT MARS-828 Rollback Completed +PROMPT ============================================================================ +PROMPT Completion Time: +SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_end FROM DUAL; +PROMPT +PROMPT Rollback Summary: +PROMPT - Package: CT_MRDS.FILE_ARCHIVER +PROMPT - Restored Version: 2.0.0 (THRESHOLD_BASED archival only) +PROMPT - Removed Features: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID strategies +PROMPT +PROMPT Log file: &_filename +PROMPT ============================================================================ + +spool off + +quit; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/track_package_versions.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/track_package_versions.sql new file mode 100644 index 0000000..b497dfa --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/track_package_versions.sql @@ -0,0 +1,96 @@ +-- =================================================================== +-- Simple Package Version Tracking Script +-- =================================================================== +-- Purpose: Track specified Oracle package versions +-- Author: Grzegorz Michalski +-- Date: 2026-01-27 +-- Version: 3.1.0 - List-Based Edition +-- +-- USAGE: +-- 1. Edit package list below (add/remove packages as needed) +-- 2. Include in your install/rollback script: @@track_package_versions.sql +-- =================================================================== + +SET SERVEROUTPUT ON; + +DECLARE + TYPE t_package_rec IS RECORD ( + owner VARCHAR2(50), + package_name VARCHAR2(50), + version VARCHAR2(50) + ); + TYPE t_packages IS TABLE OF t_package_rec; + TYPE t_string_array IS TABLE OF VARCHAR2(100); + + -- =================================================================== + -- PACKAGE LIST - Edit this array to specify packages to track + -- =================================================================== + -- Add or remove entries as needed for your MARS issue + -- Format: 'SCHEMA.PACKAGE_NAME' + -- =================================================================== + vPackageList t_string_array := t_string_array( + 'CT_MRDS.FILE_ARCHIVER' + ); + -- =================================================================== + + vPackages t_packages := t_packages(); + vVersion VARCHAR2(50); + vCount NUMBER := 0; + vOwner VARCHAR2(50); + vPackageName VARCHAR2(50); + vDotPos NUMBER; +BEGIN + DBMS_OUTPUT.PUT_LINE('========================================'); + DBMS_OUTPUT.PUT_LINE('Package Version Tracking'); + DBMS_OUTPUT.PUT_LINE('========================================'); + + -- Process each package in the list + FOR i IN 1..vPackageList.COUNT LOOP + vDotPos := INSTR(vPackageList(i), '.'); + IF vDotPos > 0 THEN + vOwner := SUBSTR(vPackageList(i), 1, vDotPos - 1); + vPackageName := SUBSTR(vPackageList(i), vDotPos + 1); + + -- Get package version + BEGIN + EXECUTE IMMEDIATE 'SELECT ' || vOwner || '.' || vPackageName || '.GET_VERSION() FROM DUAL' INTO vVersion; + vPackages.EXTEND; + vPackages(vPackages.COUNT).owner := vOwner; + vPackages(vPackages.COUNT).package_name := vPackageName; + vPackages(vPackages.COUNT).version := vVersion; + + -- Track in ENV_MANAGER + BEGIN + CT_MRDS.ENV_MANAGER.TRACK_PACKAGE_VERSION( + pPackageOwner => vOwner, + pPackageName => vPackageName, + pPackageVersion => vVersion, + pPackageBuildDate => TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'), + pPackageAuthor => 'Grzegorz Michalski' + ); + vCount := vCount + 1; + EXCEPTION + WHEN OTHERS THEN NULL; -- Continue even if tracking fails + END; + EXCEPTION + WHEN OTHERS THEN NULL; -- Skip packages that fail + END; + END IF; + END LOOP; + + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('Summary:'); + DBMS_OUTPUT.PUT_LINE('--------'); + DBMS_OUTPUT.PUT_LINE('Packages tracked: ' || vCount || '/' || vPackageList.COUNT); + + IF vPackages.COUNT > 0 THEN + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('Tracked Packages:'); + FOR i IN 1..vPackages.COUNT LOOP + DBMS_OUTPUT.PUT_LINE(' ' || vPackages(i).owner || '.' || vPackages(i).package_name || ' v' || vPackages(i).version); + END LOOP; + END IF; + + DBMS_OUTPUT.PUT_LINE('========================================'); +END; +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/verify_packages_version.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/verify_packages_version.sql new file mode 100644 index 0000000..f8ec009 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/verify_packages_version.sql @@ -0,0 +1,62 @@ +-- =================================================================== +-- Universal Package Version Verification Script +-- =================================================================== +-- Purpose: Verify all tracked Oracle packages for code changes +-- Author: Grzegorz Michalski +-- Date: 2025-12-04 +-- Version: 1.0.0 +-- +-- USAGE: +-- Include at the end of install/rollback scripts: @@verify_packages_version.sql +-- +-- OUTPUT: +-- - List of all tracked packages with their current status +-- - OK: Package has not changed since last tracking +-- - WARNING: Package code changed without version update +-- =================================================================== + +SET LINESIZE 200 +SET PAGESIZE 1000 +SET FEEDBACK OFF + +PROMPT +PROMPT ======================================== +PROMPT Package Version Verification +PROMPT ======================================== +PROMPT + +COLUMN PACKAGE_OWNER FORMAT A15 +COLUMN PACKAGE_NAME FORMAT A20 +COLUMN VERSION FORMAT A10 +COLUMN STATUS FORMAT A80 + +SELECT + PACKAGE_OWNER, + PACKAGE_NAME, + PACKAGE_VERSION AS VERSION, + CT_MRDS.ENV_MANAGER.CHECK_PACKAGE_CHANGES(PACKAGE_OWNER, PACKAGE_NAME) AS STATUS +FROM ( + SELECT + PACKAGE_OWNER, + PACKAGE_NAME, + PACKAGE_VERSION, + ROW_NUMBER() OVER (PARTITION BY PACKAGE_OWNER, PACKAGE_NAME ORDER BY TRACKING_DATE DESC) AS RN + FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING +) +WHERE RN = 1 +ORDER BY PACKAGE_OWNER, PACKAGE_NAME; + +PROMPT +PROMPT ======================================== +PROMPT Verification Complete +PROMPT ======================================== +PROMPT +PROMPT Legend: +PROMPT OK - Package has not changed since last tracking +PROMPT WARNING - Package code changed without version update +PROMPT +PROMPT For detailed hash information, use: +PROMPT SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL; +PROMPT ======================================== + +SET FEEDBACK ON