diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql index 7e64f56..9e057d3 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/01_MARS_828_install_add_archival_strategy_columns.sql @@ -16,12 +16,12 @@ ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD ( -- Add check constraint for valid strategies ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK ( - ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID') + ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID') ); -- Add comments COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS - 'Archival strategy: THRESHOLD_BASED (days), CURRENT_MONTH_ONLY (exclude current month), MINIMUM_AGE_MONTHS (minimum age), HYBRID (combination)'; + 'Archival strategy: THRESHOLD_BASED (days), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)'; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql index aecdbcd..987a502 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/06_MARS_828_configure_release01_tables.sql @@ -1,255 +1,142 @@ -- ===================================================================== -- Script: 06_MARS_828_configure_release01_tables.sql -- MARS Issue: MARS-828 --- Purpose: Configure archival strategies for Release 01 tables +-- Purpose: Configure COMPLETE archival parameters for Release 01 tables -- Author: Grzegorz Michalski --- Date: 2026-02-03 +-- Date: 2026-02-05 -- -- Description: --- Configures ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for 25 tables: --- - 19 LM tables: CURRENT_MONTH_ONLY strategy --- - 2 CSDB tables (DEBT, DEBT_DAILY): MINIMUM_AGE_MONTHS=6 strategy --- - 4 CSDB tables (ratings/descriptions): CURRENT_MONTH_ONLY strategy +-- Configures ALL archival parameters for 25 tables: +-- - ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS +-- - Archival triggers (FILES/ROWS/BYTES thresholds) +-- - Statistics expiration (HOURS_TO_EXPIRE_STATISTICS) +-- +-- Configuration by group: +-- - 19 LM tables: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 100K rows OR 1GB, 24h stats +-- - 2 CSDB DEBT: MINIMUM_AGE_MONTHS=6, 5 files OR 50K rows OR 512MB, 48h stats +-- - 4 CSDB ratings: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 20K rows OR 256MB, 72h stats -- -- Dependencies: --- - A_SOURCE_FILE_CONFIG table with ARCHIVAL_STRATEGY column +-- - A_SOURCE_FILE_CONFIG table with all archival columns -- - TRG_BI_A_SRC_FILE_CFG_ARCH_VAL trigger -- ===================================================================== PROMPT ===================================================================== -PROMPT MARS-828: Configuring Release 01 Tables - Archival Strategies +PROMPT MARS-828: Enhanced Configuration - Complete Archival Parameters PROMPT ===================================================================== PROMPT -PROMPT This script will configure archival strategies for 25 Release 01 tables: -PROMPT - 19 LM tables: CURRENT_MONTH_ONLY -PROMPT - 2 CSDB DEBT tables: MINIMUM_AGE_MONTHS (6 months) -PROMPT - 4 CSDB rating/description tables: CURRENT_MONTH_ONLY +PROMPT This script configures COMPLETE archival parameters for 25 tables +PROMPT +PROMPT LM Tables (19): +PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only) +PROMPT - Triggers: 10 files OR 100,000 rows OR 1 GB +PROMPT - Stats Expiration: 24 hours +PROMPT +PROMPT CSDB DEBT Tables (2): +PROMPT - Strategy: MINIMUM_AGE_MONTHS = 6 +PROMPT - Triggers: 5 files OR 50,000 rows OR 512 MB +PROMPT - Stats Expiration: 48 hours +PROMPT +PROMPT CSDB Rating/Description Tables (4): +PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only) +PROMPT - Triggers: 10 files OR 20,000 rows OR 256 MB +PROMPT - Stats Expiration: 72 hours PROMPT PROMPT Current timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS CURRENT_TIME FROM DUAL; PROMPT PROMPT ===================================================================== -PROMPT SECTION 1: LM Tables Configuration (CURRENT_MONTH_ONLY) +PROMPT SECTION 1: LM Tables Configuration (MINIMUM_AGE_MONTHS = 0) +PROMPT ===================================================================== +PROMPT Thresholds: 10 files OR 100K rows OR 1GB +PROMPT Stats expire: 24 hours PROMPT ===================================================================== --- LM_STANDING_FACILITIES -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeStandingFacilities' - AND TABLE_ID = 'LM_STANDING_FACILITIES'; - --- LM_STANDING_FACILITIES_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeStandingFacilities' - AND TABLE_ID = 'LM_STANDING_FACILITIES_HEADER'; - --- LM_TTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeTwoTierSystem' - AND TABLE_ID = 'LM_TTS_HEADER'; - --- LM_TTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeTwoTierSystem' - AND TABLE_ID = 'LM_TTS_ITEM'; - --- LM_ADHOC_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_HEADER'; - --- LM_ADHOC_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_ITEM'; - --- LM_ADHOC_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER'; - --- LM_BALANCESHEET_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AggregatedDailyFinancialStatementOfTheEurosystem' - AND TABLE_ID = 'LM_BALANCESHEET_HEADER'; - --- LM_BALANCESHEET_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AggregatedDailyFinancialStatementOfTheEurosystem' - AND TABLE_ID = 'LM_BALANCESHEET_ITEM'; - --- LM_CSM_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_HEADER'; - --- LM_CSM_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_ITEM'; - --- LM_CSM_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_ITEM_HEADER'; - --- LM_CURRENT_ACCOUNTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MinimumReservesRequirements' - AND TABLE_ID = 'LM_CURRENT_ACCOUNTS_HEADER'; - --- LM_CURRENT_ACCOUNTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MinimumReservesRequirements' - AND TABLE_ID = 'LM_CURRENT_ACCOUNTS_ITEM'; - --- LM_FORECAST_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'EurosystemAutonomousFactorForecast' - AND TABLE_ID = 'LM_FORECAST_HEADER'; - --- LM_FORECAST_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'EurosystemAutonomousFactorForecast' - AND TABLE_ID = 'LM_FORECAST_ITEM'; - --- LM_QRE_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_HEADER'; - --- LM_QRE_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_ITEM'; - --- LM_QRE_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_ITEM_HEADER'; - -PROMPT -PROMPT LM tables configured: 19 rows updated -PROMPT - -PROMPT ===================================================================== -PROMPT SECTION 2: CSDB DEBT Tables Configuration (MINIMUM_AGE_MONTHS) -PROMPT ===================================================================== - --- CSDB_DEBT (6 months retention) +-- Update all 19 LM tables in a single statement UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', - MINIMUM_AGE_MONTHS = 6 + MINIMUM_AGE_MONTHS = 0, -- 0 = current month only + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824, -- 1 GB + HOURS_TO_EXPIRE_STATISTICS = 24 WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'CentralizedSecuritiesDissemination' - AND TABLE_ID = 'CSDB_DEBT'; + AND A_SOURCE_KEY = 'LM' + AND TABLE_ID IN ( + 'LM_STANDING_FACILITIES', + 'LM_STANDING_FACILITIES_HEADER', + 'LM_TTS_HEADER', + 'LM_TTS_ITEM', + 'LM_ADHOC_ADJUSTMENTS_HEADER', + 'LM_ADHOC_ADJUSTMENTS_ITEM', + 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', + 'LM_BALANCESHEET_HEADER', + 'LM_BALANCESHEET_ITEM', + 'LM_CSM_ADJUSTMENTS_HEADER', + 'LM_CSM_ADJUSTMENTS_ITEM', + 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', + 'LM_CURRENT_ACCOUNTS_HEADER', + 'LM_CURRENT_ACCOUNTS_ITEM', + 'LM_FORECAST_HEADER', + 'LM_FORECAST_ITEM', + 'LM_QRE_ADJUSTMENTS_HEADER', + 'LM_QRE_ADJUSTMENTS_ITEM', + 'LM_QRE_ADJUSTMENTS_ITEM_HEADER' + ); --- CSDB_DEBT_DAILY (6 months retention) +PROMPT +PROMPT LM tables configuration completed +PROMPT + +PROMPT ===================================================================== +PROMPT SECTION 2: CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 6) +PROMPT ===================================================================== +PROMPT Thresholds: 5 files OR 50K rows OR 512MB +PROMPT Stats expire: 48 hours +PROMPT ===================================================================== + +-- Update CSDB DEBT tables (6-month retention) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', - MINIMUM_AGE_MONTHS = 6 + MINIMUM_AGE_MONTHS = 6, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 5, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912, -- 512 MB + HOURS_TO_EXPIRE_STATISTICS = 48 WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'CentralizedSecuritiesDailyReferenceDataDissemination' - AND TABLE_ID = 'CSDB_DEBT_DAILY'; + AND A_SOURCE_KEY = 'CSDB' + AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY'); PROMPT -PROMPT CSDB DEBT tables configured: 2 rows updated +PROMPT CSDB DEBT tables configuration completed PROMPT PROMPT ===================================================================== -PROMPT SECTION 3: CSDB Rating/Description Tables (CURRENT_MONTH_ONLY) +PROMPT SECTION 3: CSDB Rating/Description Tables (MINIMUM_AGE_MONTHS = 0) +PROMPT ===================================================================== +PROMPT Thresholds: 10 files OR 20K rows OR 256MB +PROMPT Stats expire: 72 hours PROMPT ===================================================================== --- CSDB_INSTR_RAT_FULL +-- Update CSDB rating/description tables UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0, -- 0 = current month only + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456, -- 256 MB + HOURS_TO_EXPIRE_STATISTICS = 72 WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_INSTR_RAT_FULL'; - --- CSDB_INSTR_DESC_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_INSTR_DESC_FULL'; - --- CSDB_ISSUER_RAT_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_ISSUER_RAT_FULL'; - --- CSDB_ISSUER_DESC_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_ISSUER_DESC_FULL'; + AND A_SOURCE_KEY = 'CSDB' + AND TABLE_ID IN ( + 'CSDB_INSTR_RAT_FULL', + 'CSDB_INSTR_DESC_FULL', + 'CSDB_ISSUER_RAT_FULL', + 'CSDB_ISSUER_DESC_FULL' + ); PROMPT -PROMPT CSDB rating/description tables configured: 4 rows updated +PROMPT CSDB rating/description tables configuration completed PROMPT PROMPT ===================================================================== @@ -263,36 +150,35 @@ PROMPT Configuration committed successfully. PROMPT PROMPT ===================================================================== -PROMPT VERIFICATION: Archival Strategy Configuration Summary +PROMPT VERIFICATION: Complete Archival Configuration PROMPT ===================================================================== PROMPT -PROMPT LM Tables (CURRENT_MONTH_ONLY): +PROMPT LM Tables (MINIMUM_AGE_MONTHS = 0): PROMPT SELECT - A_SOURCE_KEY, TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, CASE - WHEN ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY' AND MINIMUM_AGE_MONTHS IS NULL + WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' + AND MINIMUM_AGE_MONTHS = 0 + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10 + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000 + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 1073741824 + AND HOURS_TO_EXPIRE_STATISTICS = 24 THEN 'OK' ELSE 'ERROR' END AS STATUS FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_KEY = 'LM' AND SOURCE_FILE_TYPE = 'INPUT' - AND TABLE_ID IN ( - 'LM_ADHOC_ADJUSTMENTS_HEADER', 'LM_ADHOC_ADJUSTMENTS_ITEM', 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', - 'LM_BALANCESHEET_HEADER', 'LM_BALANCESHEET_ITEM', - 'LM_CSM_ADJUSTMENTS_HEADER', 'LM_CSM_ADJUSTMENTS_ITEM', 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', - 'LM_CURRENT_ACCOUNTS_HEADER', 'LM_CURRENT_ACCOUNTS_ITEM', - 'LM_FORECAST_HEADER', 'LM_FORECAST_ITEM', - 'LM_QRE_ADJUSTMENTS_HEADER', 'LM_QRE_ADJUSTMENTS_ITEM', 'LM_QRE_ADJUSTMENTS_ITEM_HEADER', - 'LM_STANDING_FACILITIES', 'LM_STANDING_FACILITIES_HEADER', - 'LM_TTS_HEADER', 'LM_TTS_ITEM' - ) + AND TABLE_ID LIKE 'LM_%' ORDER BY TABLE_ID; PROMPT @@ -300,12 +186,20 @@ PROMPT CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 6): PROMPT SELECT - A_SOURCE_KEY, TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, CASE - WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' AND MINIMUM_AGE_MONTHS = 6 + WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' + AND MINIMUM_AGE_MONTHS = 6 + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 5 + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 50000 + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 536870912 + AND HOURS_TO_EXPIRE_STATISTICS = 48 THEN 'OK' ELSE 'ERROR' END AS STATUS @@ -316,16 +210,24 @@ WHERE A_SOURCE_KEY = 'CSDB' ORDER BY TABLE_ID; PROMPT -PROMPT CSDB Rating/Description Tables (CURRENT_MONTH_ONLY): +PROMPT CSDB Rating/Description Tables (MINIMUM_AGE_MONTHS = 0): PROMPT SELECT - A_SOURCE_KEY, TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, CASE - WHEN ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY' AND MINIMUM_AGE_MONTHS IS NULL + WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' + AND MINIMUM_AGE_MONTHS = 0 + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10 + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 20000 + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 268435456 + AND HOURS_TO_EXPIRE_STATISTICS = 72 THEN 'OK' ELSE 'ERROR' END AS STATUS @@ -337,36 +239,69 @@ ORDER BY TABLE_ID; PROMPT PROMPT ===================================================================== -PROMPT Summary: Total Configured Tables +PROMPT Summary: Total Configured Tables with Full Parameters PROMPT ===================================================================== SELECT COUNT(*) AS TOTAL_CONFIGURED, - SUM(CASE WHEN ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY' THEN 1 ELSE 0 END) AS CURRENT_MONTH_ONLY, - SUM(CASE WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN 1 ELSE 0 END) AS MINIMUM_AGE_MONTHS, - SUM(CASE WHEN ARCHIVAL_STRATEGY IS NULL THEN 1 ELSE 0 END) AS NOT_CONFIGURED + SUM(CASE WHEN MINIMUM_AGE_MONTHS = 0 THEN 1 ELSE 0 END) AS CURRENT_MONTH_ONLY, + SUM(CASE WHEN MINIMUM_AGE_MONTHS > 0 THEN 1 ELSE 0 END) AS MULTI_MONTH_RETENTION, + SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_FILE_THRESHOLD, + SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_ROWS_THRESHOLD, + SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NOT NULL THEN 1 ELSE 0 END) AS WITH_BYTES_THRESHOLD, + SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' - AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID IN ( - 'LM_ADHOC_ADJUSTMENTS_HEADER', 'LM_ADHOC_ADJUSTMENTS_ITEM', 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', - 'LM_BALANCESHEET_HEADER', 'LM_BALANCESHEET_ITEM', - 'LM_CSM_ADJUSTMENTS_HEADER', 'LM_CSM_ADJUSTMENTS_ITEM', 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', - 'LM_CURRENT_ACCOUNTS_HEADER', 'LM_CURRENT_ACCOUNTS_ITEM', - 'LM_FORECAST_HEADER', 'LM_FORECAST_ITEM', - 'LM_QRE_ADJUSTMENTS_HEADER', 'LM_QRE_ADJUSTMENTS_ITEM', 'LM_QRE_ADJUSTMENTS_ITEM_HEADER', - 'LM_STANDING_FACILITIES', 'LM_STANDING_FACILITIES_HEADER', - 'LM_TTS_HEADER', 'LM_TTS_ITEM' - )) - OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( + AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%') + OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( 'CSDB_DEBT', 'CSDB_DEBT_DAILY', 'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL' ))); PROMPT -PROMPT Expected: TOTAL_CONFIGURED = 25, NOT_CONFIGURED = 0 +PROMPT Expected: +PROMPT - TOTAL_CONFIGURED = 25 +PROMPT - WITH_FILE_THRESHOLD = 25 +PROMPT - WITH_ROWS_THRESHOLD = 25 +PROMPT - WITH_BYTES_THRESHOLD = 25 +PROMPT - WITH_STATS_EXPIRY = 25 PROMPT +PROMPT ===================================================================== +PROMPT Parameter Configuration Summary by Group +PROMPT ===================================================================== + +SELECT + CASE + WHEN A_SOURCE_KEY = 'LM' THEN 'LM Tables' + WHEN TABLE_ID LIKE 'CSDB_DEBT%' THEN 'CSDB DEBT' + ELSE 'CSDB Ratings' + END AS GROUP_NAME, + COUNT(*) AS TABLE_COUNT, + MAX(ARCHIVAL_STRATEGY) AS STRATEGY, + MAX(MINIMUM_AGE_MONTHS) AS MIN_AGE, + MAX(FILES_COUNT_OVER_ARCHIVE_THRESHOLD) AS FILES_THRESHOLD, + MAX(ROWS_COUNT_OVER_ARCHIVE_THRESHOLD) AS ROWS_THRESHOLD, + ROUND(MAX(BYTES_SUM_OVER_ARCHIVE_THRESHOLD)/1048576, 0) || ' MB' AS BYTES_THRESHOLD, + MAX(HOURS_TO_EXPIRE_STATISTICS) AS STATS_HOURS +FROM CT_MRDS.A_SOURCE_FILE_CONFIG +WHERE SOURCE_FILE_TYPE = 'INPUT' + AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%') + OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( + 'CSDB_DEBT', 'CSDB_DEBT_DAILY', + 'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', + 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL' + ))) +GROUP BY + CASE + WHEN A_SOURCE_KEY = 'LM' THEN 'LM Tables' + WHEN TABLE_ID LIKE 'CSDB_DEBT%' THEN 'CSDB DEBT' + ELSE 'CSDB Ratings' + END +ORDER BY GROUP_NAME; + +PROMPT PROMPT ===================================================================== PROMPT Configuration Complete PROMPT ===================================================================== @@ -374,8 +309,17 @@ PROMPT PROMPT Next Steps: PROMPT 1. Review verification results above PROMPT 2. All 25 tables should show STATUS = 'OK' -PROMPT 3. If errors exist, review and re-run specific UPDATE statements +PROMPT 3. Verify parameter configuration summary matches expectations +PROMPT 4. Monitor archival process with new thresholds +PROMPT +PROMPT Archival Trigger Logic (OR condition): +PROMPT - Archival starts when ANY threshold is exceeded +PROMPT - Example: 10 files OR 100K rows OR 1GB triggers archival +PROMPT +PROMPT Statistics Refresh: +PROMPT - LM: Every 24 hours (daily updates for frequent changes) +PROMPT - CSDB DEBT: Every 48 hours (bi-daily for larger datasets) +PROMPT - CSDB Ratings: Every 72 hours (every 3 days for stable data) PROMPT PROMPT Log file: 06_MARS_828_configure_release01_tables.log PROMPT ===================================================================== - diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/96_MARS_828_rollback_release01_configuration.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/96_MARS_828_rollback_release01_configuration.sql index 823bd8d..78b5ceb 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/96_MARS_828_rollback_release01_configuration.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/96_MARS_828_rollback_release01_configuration.sql @@ -1,252 +1,126 @@ -- ===================================================================== -- Script: 96_MARS_828_rollback_release01_configuration.sql -- MARS Issue: MARS-828 --- Purpose: Rollback archival strategy configuration for Release 01 tables +-- Purpose: ROLLBACK archival configuration for Release 01 tables -- Author: Grzegorz Michalski --- Date: 2026-02-03 +-- Date: 2026-02-05 -- -- Description: --- Removes archival strategy configuration from 25 Release 01 tables --- by setting ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS to NULL. --- This is the rollback for 06_MARS_828_configure_release01_tables.sql +-- Rolls back archival configuration for 25 tables by setting all +-- archival parameters back to NULL (unconfigured state): +-- - ARCHIVAL_STRATEGY +-- - MINIMUM_AGE_MONTHS +-- - FILES_COUNT_OVER_ARCHIVE_THRESHOLD +-- - ROWS_COUNT_OVER_ARCHIVE_THRESHOLD +-- - BYTES_SUM_OVER_ARCHIVE_THRESHOLD +-- - HOURS_TO_EXPIRE_STATISTICS +-- +-- This script reverts changes made by: +-- - 06_MARS_828_configure_release01_tables.sql -- -- Dependencies: --- - A_SOURCE_FILE_CONFIG table with ARCHIVAL_STRATEGY column +-- - A_SOURCE_FILE_CONFIG table with archival columns -- ===================================================================== PROMPT ===================================================================== -PROMPT MARS-828: Rollback Release 01 Tables Configuration +PROMPT MARS-828: ROLLBACK - Remove Archival Configuration PROMPT ===================================================================== PROMPT -PROMPT This script will remove archival strategy configuration from 25 tables: -PROMPT - 19 LM tables -PROMPT - 6 CSDB tables (DEBT, DEBT_DAILY, and 4 rating/description tables) +PROMPT This script will REMOVE archival configuration from 25 tables PROMPT -PROMPT ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS will be set to NULL +PROMPT WARNING: This will reset ALL archival parameters to NULL +PROMPT +PROMPT Tables affected: +PROMPT - 19 LM tables +PROMPT - 2 CSDB DEBT tables +PROMPT - 4 CSDB rating/description tables PROMPT PROMPT Current timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS CURRENT_TIME FROM DUAL; PROMPT - -PROMPT ===================================================================== -PROMPT This will disable archival automation for all 25 tables -PROMPT ===================================================================== -PROMPT - PROMPT ===================================================================== PROMPT SECTION 1: Rollback LM Tables Configuration PROMPT ===================================================================== --- LM_STANDING_FACILITIES +-- Rollback all 19 LM tables in a single statement UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL + MINIMUM_AGE_MONTHS = NULL, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = NULL, + HOURS_TO_EXPIRE_STATISTICS = NULL WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeStandingFacilities' - AND TABLE_ID = 'LM_STANDING_FACILITIES'; - --- LM_STANDING_FACILITIES_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeStandingFacilities' - AND TABLE_ID = 'LM_STANDING_FACILITIES_HEADER'; - --- LM_TTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeTwoTierSystem' - AND TABLE_ID = 'LM_TTS_HEADER'; - --- LM_TTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeTwoTierSystem' - AND TABLE_ID = 'LM_TTS_ITEM'; - --- LM_ADHOC_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_HEADER'; - --- LM_ADHOC_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_ITEM'; - --- LM_ADHOC_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AdhocAdjustment' - AND TABLE_ID = 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER'; - --- LM_BALANCESHEET_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AggregatedDailyFinancialStatementOfTheEurosystem' - AND TABLE_ID = 'LM_BALANCESHEET_HEADER'; - --- LM_BALANCESHEET_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'AggregatedDailyFinancialStatementOfTheEurosystem' - AND TABLE_ID = 'LM_BALANCESHEET_ITEM'; - --- LM_CSM_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_HEADER'; - --- LM_CSM_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_ITEM'; - --- LM_CSM_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MonthlyCSMAdjustment' - AND TABLE_ID = 'LM_CSM_ADJUSTMENTS_ITEM_HEADER'; - --- LM_CURRENT_ACCOUNTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MinimumReservesRequirements' - AND TABLE_ID = 'LM_CURRENT_ACCOUNTS_HEADER'; - --- LM_CURRENT_ACCOUNTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'MinimumReservesRequirements' - AND TABLE_ID = 'LM_CURRENT_ACCOUNTS_ITEM'; - --- LM_FORECAST_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'EurosystemAutonomousFactorForecast' - AND TABLE_ID = 'LM_FORECAST_HEADER'; - --- LM_FORECAST_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'EurosystemAutonomousFactorForecast' - AND TABLE_ID = 'LM_FORECAST_ITEM'; - --- LM_QRE_ADJUSTMENTS_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_HEADER'; - --- LM_QRE_ADJUSTMENTS_ITEM -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_ITEM'; - --- LM_QRE_ADJUSTMENTS_ITEM_HEADER -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'QuarterlyRevaluationAdjustment' - AND TABLE_ID = 'LM_QRE_ADJUSTMENTS_ITEM_HEADER'; + AND A_SOURCE_KEY = 'LM' + AND TABLE_ID IN ( + 'LM_STANDING_FACILITIES', + 'LM_STANDING_FACILITIES_HEADER', + 'LM_TTS_HEADER', + 'LM_TTS_ITEM', + 'LM_ADHOC_ADJUSTMENTS_HEADER', + 'LM_ADHOC_ADJUSTMENTS_ITEM', + 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', + 'LM_BALANCESHEET_HEADER', + 'LM_BALANCESHEET_ITEM', + 'LM_CSM_ADJUSTMENTS_HEADER', + 'LM_CSM_ADJUSTMENTS_ITEM', + 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', + 'LM_CURRENT_ACCOUNTS_HEADER', + 'LM_CURRENT_ACCOUNTS_ITEM', + 'LM_FORECAST_HEADER', + 'LM_FORECAST_ITEM', + 'LM_QRE_ADJUSTMENTS_HEADER', + 'LM_QRE_ADJUSTMENTS_ITEM', + 'LM_QRE_ADJUSTMENTS_ITEM_HEADER' + ); PROMPT -PROMPT LM tables configuration removed: 19 rows updated +PROMPT LM tables rollback completed PROMPT PROMPT ===================================================================== -PROMPT SECTION 2: Rollback CSDB Tables Configuration +PROMPT SECTION 2: Rollback CSDB DEBT Tables Configuration PROMPT ===================================================================== --- CSDB_DEBT +-- Rollback CSDB DEBT tables UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL + MINIMUM_AGE_MONTHS = NULL, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = NULL, + HOURS_TO_EXPIRE_STATISTICS = NULL WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'CentralizedSecuritiesDissemination' - AND TABLE_ID = 'CSDB_DEBT'; - --- CSDB_DEBT_DAILY -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'CentralizedSecuritiesDailyReferenceDataDissemination' - AND TABLE_ID = 'CSDB_DEBT_DAILY'; - --- CSDB_INSTR_RAT_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_INSTR_RAT_FULL'; - --- CSDB_INSTR_DESC_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_INSTR_DESC_FULL'; - --- CSDB_ISSUER_RAT_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_ISSUER_RAT_FULL'; - --- CSDB_ISSUER_DESC_FULL -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = NULL, - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'FullRatingsDissemination' - AND TABLE_ID = 'CSDB_ISSUER_DESC_FULL'; + AND A_SOURCE_KEY = 'CSDB' + AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY'); PROMPT -PROMPT CSDB tables configuration removed: 6 rows updated +PROMPT CSDB DEBT tables rollback completed +PROMPT + +PROMPT ===================================================================== +PROMPT SECTION 3: Rollback CSDB Rating/Description Tables Configuration +PROMPT ===================================================================== + +-- Rollback CSDB rating/description tables +UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = NULL, + MINIMUM_AGE_MONTHS = NULL, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = NULL, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD = NULL, + HOURS_TO_EXPIRE_STATISTICS = NULL +WHERE SOURCE_FILE_TYPE = 'INPUT' + AND A_SOURCE_KEY = 'CSDB' + AND TABLE_ID IN ( + 'CSDB_INSTR_RAT_FULL', + 'CSDB_INSTR_DESC_FULL', + 'CSDB_ISSUER_RAT_FULL', + 'CSDB_ISSUER_DESC_FULL' + ); + +PROMPT +PROMPT CSDB rating/description tables rollback completed PROMPT PROMPT ===================================================================== @@ -260,82 +134,138 @@ PROMPT Rollback committed successfully. PROMPT PROMPT ===================================================================== -PROMPT VERIFICATION: Configuration Removal Confirmation +PROMPT VERIFICATION: Archival Configuration Removed PROMPT ===================================================================== PROMPT -PROMPT All 25 Tables - Configuration Status: +PROMPT LM Tables (should all be NULL): PROMPT SELECT - A_SOURCE_KEY, TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, CASE - WHEN ARCHIVAL_STRATEGY IS NULL AND MINIMUM_AGE_MONTHS IS NULL - THEN 'REMOVED' - ELSE 'ERROR - STILL CONFIGURED' + WHEN ARCHIVAL_STRATEGY IS NULL + AND MINIMUM_AGE_MONTHS IS NULL + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NULL + AND HOURS_TO_EXPIRE_STATISTICS IS NULL + THEN 'OK' + ELSE 'ERROR - Still configured' END AS STATUS FROM CT_MRDS.A_SOURCE_FILE_CONFIG -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID IN ( - 'LM_ADHOC_ADJUSTMENTS_HEADER', 'LM_ADHOC_ADJUSTMENTS_ITEM', 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', - 'LM_BALANCESHEET_HEADER', 'LM_BALANCESHEET_ITEM', - 'LM_CSM_ADJUSTMENTS_HEADER', 'LM_CSM_ADJUSTMENTS_ITEM', 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', - 'LM_CURRENT_ACCOUNTS_HEADER', 'LM_CURRENT_ACCOUNTS_ITEM', - 'LM_FORECAST_HEADER', 'LM_FORECAST_ITEM', - 'LM_QRE_ADJUSTMENTS_HEADER', 'LM_QRE_ADJUSTMENTS_ITEM', 'LM_QRE_ADJUSTMENTS_ITEM_HEADER', - 'LM_STANDING_FACILITIES', 'LM_STANDING_FACILITIES_HEADER', - 'LM_TTS_HEADER', 'LM_TTS_ITEM' - )) - OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( - 'CSDB_DEBT', 'CSDB_DEBT_DAILY', - 'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', - 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL' - ))) -ORDER BY A_SOURCE_KEY, TABLE_ID; +WHERE A_SOURCE_KEY = 'LM' + AND SOURCE_FILE_TYPE = 'INPUT' + AND TABLE_ID LIKE 'LM_%' +ORDER BY TABLE_ID; + +PROMPT +PROMPT CSDB DEBT Tables (should all be NULL): +PROMPT + +SELECT + TABLE_ID, + ARCHIVAL_STRATEGY, + MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, + CASE + WHEN ARCHIVAL_STRATEGY IS NULL + AND MINIMUM_AGE_MONTHS IS NULL + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NULL + AND HOURS_TO_EXPIRE_STATISTICS IS NULL + THEN 'OK' + ELSE 'ERROR - Still configured' + END AS STATUS +FROM CT_MRDS.A_SOURCE_FILE_CONFIG +WHERE A_SOURCE_KEY = 'CSDB' + AND SOURCE_FILE_TYPE = 'INPUT' + AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY') +ORDER BY TABLE_ID; + +PROMPT +PROMPT CSDB Rating/Description Tables (should all be NULL): +PROMPT + +SELECT + TABLE_ID, + ARCHIVAL_STRATEGY, + MINIMUM_AGE_MONTHS, + FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THR, + ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THR, + BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THR, + HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS, + CASE + WHEN ARCHIVAL_STRATEGY IS NULL + AND MINIMUM_AGE_MONTHS IS NULL + AND FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL + AND BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NULL + AND HOURS_TO_EXPIRE_STATISTICS IS NULL + THEN 'OK' + ELSE 'ERROR - Still configured' + END AS STATUS +FROM CT_MRDS.A_SOURCE_FILE_CONFIG +WHERE A_SOURCE_KEY = 'CSDB' + AND SOURCE_FILE_TYPE = 'INPUT' + AND TABLE_ID IN ('CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL') +ORDER BY TABLE_ID; PROMPT PROMPT ===================================================================== -PROMPT Summary: Configuration Removal Status +PROMPT Summary: Total Tables with Rollback Applied PROMPT ===================================================================== SELECT COUNT(*) AS TOTAL_TABLES, - SUM(CASE WHEN ARCHIVAL_STRATEGY IS NULL AND MINIMUM_AGE_MONTHS IS NULL THEN 1 ELSE 0 END) AS REMOVED, - SUM(CASE WHEN ARCHIVAL_STRATEGY IS NOT NULL OR MINIMUM_AGE_MONTHS IS NOT NULL THEN 1 ELSE 0 END) AS STILL_CONFIGURED + SUM(CASE WHEN ARCHIVAL_STRATEGY IS NULL THEN 1 ELSE 0 END) AS STRATEGY_NULL, + SUM(CASE WHEN MINIMUM_AGE_MONTHS IS NULL THEN 1 ELSE 0 END) AS MIN_AGE_NULL, + SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS FILE_THR_NULL, + SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS ROWS_THR_NULL, + SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS BYTES_THR_NULL, + SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NULL THEN 1 ELSE 0 END) AS STATS_NULL FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' - AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID IN ( - 'LM_ADHOC_ADJUSTMENTS_HEADER', 'LM_ADHOC_ADJUSTMENTS_ITEM', 'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER', - 'LM_BALANCESHEET_HEADER', 'LM_BALANCESHEET_ITEM', - 'LM_CSM_ADJUSTMENTS_HEADER', 'LM_CSM_ADJUSTMENTS_ITEM', 'LM_CSM_ADJUSTMENTS_ITEM_HEADER', - 'LM_CURRENT_ACCOUNTS_HEADER', 'LM_CURRENT_ACCOUNTS_ITEM', - 'LM_FORECAST_HEADER', 'LM_FORECAST_ITEM', - 'LM_QRE_ADJUSTMENTS_HEADER', 'LM_QRE_ADJUSTMENTS_ITEM', 'LM_QRE_ADJUSTMENTS_ITEM_HEADER', - 'LM_STANDING_FACILITIES', 'LM_STANDING_FACILITIES_HEADER', - 'LM_TTS_HEADER', 'LM_TTS_ITEM' - )) - OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( + AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%') + OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN ( 'CSDB_DEBT', 'CSDB_DEBT_DAILY', 'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL' ))); PROMPT -PROMPT Expected: TOTAL_TABLES = 25, REMOVED = 25, STILL_CONFIGURED = 0 +PROMPT Expected: All counts should be 25 (all parameters NULL for all tables) +PROMPT - TOTAL_TABLES = 25 +PROMPT - STRATEGY_NULL = 25 +PROMPT - MIN_AGE_NULL = 25 +PROMPT - FILE_THR_NULL = 25 +PROMPT - ROWS_THR_NULL = 25 +PROMPT - BYTES_THR_NULL = 25 +PROMPT - STATS_NULL = 25 PROMPT PROMPT ===================================================================== PROMPT Rollback Complete PROMPT ===================================================================== PROMPT -PROMPT Configuration has been removed from all 25 Release 01 tables. -PROMPT Archival automation is now disabled for these tables. +PROMPT Next Steps: +PROMPT 1. Review verification results above +PROMPT 2. All 25 tables should show STATUS = 'OK' +PROMPT 3. All archival parameters should be NULL +PROMPT 4. Tables are now in unconfigured state (no archival strategy) PROMPT -PROMPT To re-enable configuration, run: -PROMPT 06_MARS_828_configure_release01_tables.sql +PROMPT To restore configuration, re-run: +PROMPT @06_MARS_828_configure_release01_tables.sql PROMPT PROMPT Log file: 96_MARS_828_rollback_release01_configuration.log PROMPT ===================================================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md b/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md index 8d4aa72..6c22875 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/README.md @@ -17,7 +17,7 @@ WHERE extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRES **Solution**: Introduce ARCHIVAL_STRATEGY configuration column with four strategies: - `THRESHOLD_BASED` - Days-based threshold (backward compatible) -- `CURRENT_MONTH_ONLY` - Keep only current month data +- `MINIMUM_AGE_MONTHS` - Retain data for specified months (0 = current month only) - `MINIMUM_AGE_MONTHS` - Archive data older than X months - `HYBRID` - Combination of current month and minimum age @@ -38,7 +38,7 @@ DAYS_FOR_ARCHIVE_THRESHOLD NUMBER DEFAULT 30 ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL, MINIMUM_AGE_MONTHS NUMBER(3), DAYS_FOR_ARCHIVE_THRESHOLD NUMBER DEFAULT 30, -CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK (ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID')) +CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK (ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID')) ``` **New Trigger**: TRG_BI_ARCHIVAL_STRATEGY_VAL @@ -69,17 +69,17 @@ CONSTRAINT CHK_ARCHIVAL_STRATEGY CHECK (ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', | Strategy | WHERE Clause Logic | Configuration Required | Use Case | |----------|-------------------|----------------------|----------| | `THRESHOLD_BASED` | `extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRESHOLD` | DAYS_FOR_ARCHIVE_THRESHOLD | Legacy compatibility | -| `CURRENT_MONTH_ONLY` | `TRUNC(workflow_start, 'MM') < TRUNC(SYSDATE, 'MM')` | None | General sources (LM, TOP) | +| `MINIMUM_AGE_MONTHS` | `workflow_start < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -N)` | MINIMUM_AGE_MONTHS (0=current month) | All sources | | `MINIMUM_AGE_MONTHS` | `workflow_start < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -X)` | MINIMUM_AGE_MONTHS | CSDB (6 months retention) | -| `HYBRID` | Both CURRENT_MONTH_ONLY AND MINIMUM_AGE_MONTHS | MINIMUM_AGE_MONTHS | Advanced scenarios | +| `HYBRID` | Combines month truncation AND minimum age | MINIMUM_AGE_MONTHS | Advanced scenarios | ## Configuration Examples ```sql -- LM/TOP sources: Archive everything except current month UPDATE A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0, -- 0 = current month only WHERE A_SOURCE_KEY IN ('LM', 'TOP'); -- CSDB: Archive only data older than 6 months @@ -189,7 +189,7 @@ sql "ADMIN/Cloudpass#34@ggmichalski_high" "@rollback_mars828.sql" **Main Test Script**: test/test_archival_strategies.sql ```sql --- Test 1: CURRENT_MONTH_ONLY strategy +-- Test 1: MINIMUM_AGE_MONTHS=0 strategy (current month only) -- Expected: Archives data from previous months only SELECT COUNT(*) FROM table WHERE TRUNC(workflow_start, 'MM') < TRUNC(SYSDATE, 'MM'); @@ -249,7 +249,7 @@ No automatic migration required. New columns have sensible defaults: - MINIMUM_AGE_MONTHS = NULL (not required for THRESHOLD_BASED) ## Version History -- **v3.0.0** (2026-01-27): Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration +- **v3.0.0** (2026-01-27): Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration - **v2.0.0** (2025-10-01): Initial FILE_ARCHIVER package with threshold-based archival ## Related JIRA Issues diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql index 1c5754a..cb4dcd3 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql @@ -35,7 +35,7 @@ PROMPT ========================================================================= PROMPT MARS-828 Installation Starting PROMPT ============================================================================ PROMPT Package: CT_MRDS.FILE_ARCHIVER -PROMPT Change: Enhanced archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) +PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) PROMPT Purpose: Flexible archival policies per data source PROMPT Steps: 8 (DDL, Trigger, Packages, Verify, Track, Verify, Configure) PROMPT Timestamp: @@ -60,27 +60,27 @@ PROMPT =================================================================== @@01_MARS_828_install_add_archival_strategy_columns.sql PROMPT -PROMPT Step 2/7: Creating validation trigger +PROMPT Step 2/8: Creating validation trigger PROMPT ====================================== @@02_MARS_828_install_archival_strategy_trigger.sql PROMPT -PROMPT Step 3/7: Deploying FILE_ARCHIVER Package Specification v3.0.0 +PROMPT Step 3/8: Deploying FILE_ARCHIVER Package Specification v3.1.0 PROMPT =============================================================== @@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 4/7: Deploying FILE_ARCHIVER Package Body v3.0.0 +PROMPT Step 4/8: Deploying FILE_ARCHIVER Package Body v3.1.0 PROMPT ====================================================== @@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql PROMPT -PROMPT Step 5/7: Verifying installation +PROMPT Step 5/8: Verifying installation PROMPT ================================= @@05_MARS_828_verify_installation.sql PROMPT -PROMPT Step 6/7: Tracking package versions +PROMPT Step 6/8: Tracking package versions PROMPT ==================================== @@track_package_versions.sql @@ -103,9 +103,9 @@ SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL; PROMPT PROMPT Installation Summary: PROMPT - Package: CT_MRDS.FILE_ARCHIVER -PROMPT - Version: 2.0.0 -> 3.0.0 (MAJOR) -PROMPT - New Strategies: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID -PROMPT - Backward Compatible: THRESHOLD_BASED (default) +PROMPT - Version: 3.1.0 (BREAKING CHANGE - CURRENT_MONTH_ONLY removed) +PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID +PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED preserved) PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB) PROMPT PROMPT Log file: &_filename diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb index 2f977e5..bd59fa2 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb @@ -23,11 +23,7 @@ AS WHEN 'THRESHOLD_BASED' THEN vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD; - -- Archive all data except current month - WHEN 'CURRENT_MONTH_ONLY' THEN - vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'')'; - - -- Archive only data older than X months + -- Archive data older than X months (0 = current month only) WHEN 'MINIMUM_AGE_MONTHS' THEN IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg index 2ff8ff2..a78fdc6 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg @@ -24,7 +24,7 @@ AS -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := '3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) || - '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || + '3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) || '2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) || '1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) || '1.0.0 (2025-09-15): Initial release with table archival and statistics gathering'; diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql index d25647a..001b89e 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql @@ -20,10 +20,10 @@ BEGIN RAISE_APPLICATION_ERROR(-20999, vErrorMsg); END IF; - -- Validate MINIMUM_AGE_MONTHS is positive + -- Validate MINIMUM_AGE_MONTHS is non-negative (0 = current month only) IF :NEW.MINIMUM_AGE_MONTHS IS NOT NULL - AND :NEW.MINIMUM_AGE_MONTHS < 1 THEN - RAISE_APPLICATION_ERROR(-20998, 'MINIMUM_AGE_MONTHS must be greater than 0'); + AND :NEW.MINIMUM_AGE_MONTHS < 0 THEN + RAISE_APPLICATION_ERROR(-20998, 'MINIMUM_AGE_MONTHS must be greater than or equal to 0'); END IF; -- Warn if MINIMUM_AGE_MONTHS set but strategy doesn't use it diff --git a/confluence/FILE_ARCHIVER_Guide.md b/confluence/FILE_ARCHIVER_Guide.md index b184ef0..b233978 100644 --- a/confluence/FILE_ARCHIVER_Guide.md +++ b/confluence/FILE_ARCHIVER_Guide.md @@ -8,7 +8,7 @@ The FILE_ARCHIVER package provides flexible archival strategies that accommodate ### Key Features -- **Four Archival Strategies**: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID +- **Three Archival Strategies**: THRESHOLD_BASED, MINIMUM_AGE_MONTHS (with 0=current month only), HYBRID - **Flexible Configuration**: Per-table archival strategy configuration via A_SOURCE_FILE_CONFIG - **Backward Compatible**: Default THRESHOLD_BASED strategy maintains existing behavior - **Validation**: Automatic validation of strategy-specific configuration requirements @@ -39,9 +39,8 @@ The FILE_ARCHIVER package provides flexible archival strategies that accommodate | Strategy | WHERE Clause Logic | Configuration Required | Primary Use Case | |----------|-------------------|----------------------|------------------| | `THRESHOLD_BASED` | Days since workflow start > threshold | DAYS_FOR_ARCHIVE_THRESHOLD | Legacy compatibility, simple time-based archival | -| `CURRENT_MONTH_ONLY` | Archive all data from previous months | None | LM/TOP sources - keep only current month active | -| `MINIMUM_AGE_MONTHS` | Archive data older than X months | MINIMUM_AGE_MONTHS | CSDB sources - retain data for specific period | -| `HYBRID` | Combines CURRENT_MONTH_ONLY + MINIMUM_AGE_MONTHS | MINIMUM_AGE_MONTHS | Advanced retention scenarios | +| `MINIMUM_AGE_MONTHS` | Archive data older than X months (0=current month only) | MINIMUM_AGE_MONTHS (≥0) | All sources - flexible retention (0 for LM, 6 for CSDB) | +| `HYBRID` | Combines month boundary + minimum age | MINIMUM_AGE_MONTHS | Advanced retention scenarios | ### 1. THRESHOLD_BASED (Default) @@ -65,48 +64,27 @@ WHERE SOURCE_FILE_TYPE = 'INPUT' **Use Case**: Simple time-based archival, backward compatible with FILE_ARCHIVER v2.0.0 behavior. -### 2. CURRENT_MONTH_ONLY +### 2. MINIMUM_AGE_MONTHS -Archives all data from previous months, keeping only current month data in ODS bucket. - -**WHERE Clause**: -```sql -TRUNC(workflow_start, 'MM') < TRUNC(SYSDATE, 'MM') -``` - -**Configuration**: -```sql --- LM/TOP sources: Archive everything except current month -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -WHERE SOURCE_FILE_TYPE = 'INPUT' - AND SOURCE_FILE_ID = 'DistributeStandingFacilities' - AND TABLE_ID = 'LM_STANDING_FACILITIES'; -``` - -**Use Case**: -- LM dissemination feeds (daily/intraday updates) -- TOP operational data requiring current month access -- Sources where historical data is rarely accessed - -**Behavior**: -- January data: Archived on February 1st -- February data: Remains in ODS bucket during February -- March 1st: February data archived, March data active - -### 3. MINIMUM_AGE_MONTHS - -Archives only data older than specified number of months, allowing multi-month retention in ODS bucket. +Archives data older than specified number of months. **Special case**: MINIMUM_AGE_MONTHS = 0 archives all data before current month (replaces deprecated CURRENT_MONTH_ONLY strategy). **WHERE Clause**: ```sql workflow_start < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -MINIMUM_AGE_MONTHS) +-- When MINIMUM_AGE_MONTHS = 0: workflow_start < TRUNC(SYSDATE, 'MM') ``` -**Configuration**: +**Configuration Examples**: ```sql --- CSDB: Archive only data older than 6 months +-- LM: Keep only current month data (MINIMUM_AGE_MONTHS = 0) +UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0 +WHERE SOURCE_FILE_TYPE = 'INPUT' + AND SOURCE_FILE_ID = 'DistributeStandingFacilities' + AND TABLE_ID = 'LM_STANDING_FACILITIES'; + +-- CSDB: Retain 6 months of data (MINIMUM_AGE_MONTHS = 6) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', MINIMUM_AGE_MONTHS = 6 @@ -115,19 +93,25 @@ WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY'); ``` -**Use Case**: -- CSDB securities/ratings data requiring 6-month retention -- Regulatory compliance with specific retention periods -- Analytical workloads needing multi-month historical access +**Use Cases**: +- **MINIMUM_AGE_MONTHS = 0**: LM dissemination feeds requiring current month only (daily/intraday updates) +- **MINIMUM_AGE_MONTHS = 6**: CSDB securities/ratings data requiring 6-month retention +- **MINIMUM_AGE_MONTHS = N**: Regulatory compliance with specific N-month retention periods -**Behavior** (with MINIMUM_AGE_MONTHS = 6): -- February 2026: Archives data from July 2025 and earlier -- March 2026: Archives data from August 2025 and earlier -- Keeps 6 months of data always available in ODS bucket +**Behavior Examples**: +- **With MINIMUM_AGE_MONTHS = 0**: + - January data: Archived on February 1st + - February data: Remains in ODS bucket during February + - March 1st: February data archived, March data active + +- **With MINIMUM_AGE_MONTHS = 6**: + - February 2026: Archives data from July 2025 and earlier + - March 2026: Archives data from August 2025 and earlier + - Keeps current month + 6 previous months (7 months total) in ODS bucket -### 4. HYBRID +### 3. HYBRID -Combines CURRENT_MONTH_ONLY and MINIMUM_AGE_MONTHS logic - archives data from previous months AND older than minimum age. +Combines month boundary check with minimum age threshold - archives data from previous months AND older than minimum age. **WHERE Clause**: ```sql @@ -157,13 +141,10 @@ WHERE SOURCE_FILE_TYPE = 'INPUT' Automatically validates archival configuration on INSERT/UPDATE to A_SOURCE_FILE_CONFIG: **Validation Rules**: -1. **CURRENT_MONTH_ONLY**: Requires `MINIMUM_AGE_MONTHS IS NULL` - - Error: "Strategy CURRENT_MONTH_ONLY requires MINIMUM_AGE_MONTHS to be NULL" +1. **MINIMUM_AGE_MONTHS**: Requires `MINIMUM_AGE_MONTHS IS NOT NULL AND MINIMUM_AGE_MONTHS >= 0` + - Error: "Strategy MINIMUM_AGE_MONTHS requires MINIMUM_AGE_MONTHS to be set (≥0)" -2. **MINIMUM_AGE_MONTHS**: Requires `MINIMUM_AGE_MONTHS IS NOT NULL` - - Error: "Strategy MINIMUM_AGE_MONTHS requires MINIMUM_AGE_MONTHS to be set" - -3. **HYBRID**: Requires `MINIMUM_AGE_MONTHS IS NOT NULL` +2. **HYBRID**: Requires `MINIMUM_AGE_MONTHS IS NOT NULL` - Error: "Strategy HYBRID requires MINIMUM_AGE_MONTHS to be set" **Example Validation Error**: @@ -233,13 +214,13 @@ END; ## Configuration Examples -### Example 1: Configure LM Standing Facilities (CURRENT_MONTH_ONLY) +### Example 1: Configure LM Standing Facilities (Current Month Only) ```sql --- Keep only current month data in ODS bucket +-- Keep only current month data in ODS bucket (MINIMUM_AGE_MONTHS = 0) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0 -- 0 = archives all data before current month WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'DistributeStandingFacilities' AND TABLE_ID = 'LM_STANDING_FACILITIES'; @@ -280,10 +261,10 @@ WHERE TABLE_ID = 'CSDB_DEBT'; ### Example 3: Bulk Configuration for LM Source ```sql --- Configure all 19 LM tables with CURRENT_MONTH_ONLY strategy +-- Configure all 19 LM tables with MINIMUM_AGE_MONTHS = 0 (current month only) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL +SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0 -- 0 = keep only current month WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID IN ( 'DistributeStandingFacilities', @@ -342,7 +323,7 @@ ORDER BY ARCHIVAL_STRATEGY; The following 25 Release 01 tables were configured with archival strategies: -**LM Tables (19 total) - CURRENT_MONTH_ONLY**: +**LM Tables (19 total) - MINIMUM_AGE_MONTHS = 0 (current month only)**: - LM_STANDING_FACILITIES - LM_STANDING_FACILITIES_HEADER - LM_TTS_HEADER @@ -365,11 +346,11 @@ The following 25 Release 01 tables were configured with archival strategies: **CSDB Tables (6 total)**: -*MINIMUM_AGE_MONTHS = 6*: +*MINIMUM_AGE_MONTHS = 6 (6-month retention)*: - CSDB_DEBT - CSDB_DEBT_DAILY -*CURRENT_MONTH_ONLY*: +*MINIMUM_AGE_MONTHS = 0 (current month only)*: - CSDB_INSTR_RAT_FULL - CSDB_INSTR_DESC_FULL - CSDB_ISSUER_RAT_FULL @@ -426,25 +407,7 @@ SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', WHERE ...; ``` -#### Issue 2: Validation Error on CURRENT_MONTH_ONLY - -**Error**: -``` -ORA-20001: Strategy CURRENT_MONTH_ONLY requires MINIMUM_AGE_MONTHS to be NULL -``` - -**Cause**: MINIMUM_AGE_MONTHS has value but strategy doesn't use it - -**Solution**: -```sql --- Set MINIMUM_AGE_MONTHS to NULL for CURRENT_MONTH_ONLY -UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL -- Required to be NULL -WHERE ...; -``` - -#### Issue 3: Archival Not Working as Expected +#### Issue 2: Archival Not Working as Expected **Symptoms**: Data not being archived according to strategy @@ -492,7 +455,7 @@ END; / ``` -#### Issue 4: Package Compilation Errors After Upgrade +#### Issue 3: Package Compilation Errors After Upgrade **Symptoms**: FILE_ARCHIVER package shows INVALID status @@ -516,16 +479,19 @@ WHERE object_name = 'FILE_ARCHIVER'; ## Version History -### v3.1.0 (Current) -- Enhanced archival strategies -- Improved error handling -- Additional validation features +### v3.1.0 (Current - 2026-02-05) +- **BREAKING CHANGE**: Removed CURRENT_MONTH_ONLY strategy (replaced by MINIMUM_AGE_MONTHS = 0) +- Mathematical equivalence: CURRENT_MONTH_ONLY ≡ MINIMUM_AGE_MONTHS = 0 +- Updated trigger validation to allow MINIMUM_AGE_MONTHS >= 0 (previously >= 1) +- Simplified architecture from 4 strategies to 3 +- Enhanced error handling +- All 25 Release 01 tables migrated to MINIMUM_AGE_MONTHS (23 with value 0, 2 with value 6) -### v3.0.0 (MARS-828) +### v3.0.0 (MARS-828 - 2026-02-04) - Added ARCHIVAL_STRATEGY configuration column -- Implemented four archival strategies: +- Implemented four archival strategies (later reduced to three in v3.1.0): - THRESHOLD_BASED (backward compatible) - - CURRENT_MONTH_ONLY + - CURRENT_MONTH_ONLY (deprecated in v3.1.0, use MINIMUM_AGE_MONTHS = 0) - MINIMUM_AGE_MONTHS - HYBRID - Added GET_ARCHIVAL_WHERE_CLAUSE function @@ -567,26 +533,26 @@ WHERE object_name = 'FILE_ARCHIVER'; ### Strategy Selection Guidelines -1. **Use CURRENT_MONTH_ONLY when**: - - Data is updated frequently (daily/intraday) - - Historical data access is rare - - ODS bucket space is limited - - Example: LM dissemination feeds +1. **Use MINIMUM_AGE_MONTHS when**: + - **MINIMUM_AGE_MONTHS = 0**: Current month only retention + - Data updated frequently (daily/intraday) + - Historical data access is rare + - ODS bucket space is limited + - Example: LM dissemination feeds + - **MINIMUM_AGE_MONTHS = N (N > 0)**: Multi-month retention + - Regulatory compliance requires specific retention period + - Analytical workloads need N-month access + - Data updates are infrequent + - Example: CSDB securities data (MINIMUM_AGE_MONTHS = 6) -2. **Use MINIMUM_AGE_MONTHS when**: - - Regulatory compliance requires specific retention period - - Analytical workloads need multi-month access - - Data updates are infrequent - - Example: CSDB securities data (6-month retention) - -3. **Use THRESHOLD_BASED when**: +2. **Use THRESHOLD_BASED when**: - Maintaining backward compatibility with legacy behavior - Simple time-based archival is sufficient - Migration from FILE_ARCHIVER v2.0.0 -4. **Use HYBRID when**: +3. **Use HYBRID when**: - Complex retention requirements - - Combining current month access with minimum age threshold + - Combining month boundary check with minimum age threshold - Advanced scenarios not covered by other strategies ### Configuration Best Practices @@ -595,8 +561,8 @@ WHERE object_name = 'FILE_ARCHIVER'; ```sql -- Test on single table first UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG - SET ARCHIVAL_STRATEGY = 'CURRENT_MONTH_ONLY', - MINIMUM_AGE_MONTHS = NULL + SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', + MINIMUM_AGE_MONTHS = 0 -- 0 = current month only WHERE SOURCE_FILE_ID = 'TEST_FILE' AND TABLE_ID = 'TEST_TABLE'; @@ -610,8 +576,8 @@ WHERE object_name = 'FILE_ARCHIVER'; SELECT SOURCE_FILE_ID, TABLE_ID, - 'CURRENT_MONTH_ONLY' AS NEW_STRATEGY, - NULL AS NEW_MIN_AGE, + 'MINIMUM_AGE_MONTHS' AS NEW_STRATEGY, + 0 AS NEW_MIN_AGE, -- 0 = current month only ARCHIVAL_STRATEGY AS OLD_STRATEGY, MINIMUM_AGE_MONTHS AS OLD_MIN_AGE FROM CT_MRDS.A_SOURCE_FILE_CONFIG