diff --git a/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_add_archival_strategy_column.sql b/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_add_archival_strategy_column.sql new file mode 100644 index 0000000..f4511d1 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_add_archival_strategy_column.sql @@ -0,0 +1,348 @@ +-- ============================================================================ +-- MARS-1409 Step 10: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns +-- to A_TABLE_STAT and A_TABLE_STAT_HIST tables; +-- add IS_WORKFLOW_SUCCESS_REQUIRED to A_SOURCE_FILE_CONFIG +-- ============================================================================ +-- Purpose: Make statistics records fully self-documenting: +-- ARCHIVAL_STRATEGY - strategy in effect when stats were gathered +-- (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID) +-- ARCH_MINIMUM_AGE_MONTHS - minimum age threshold in months; complements +-- ARCH_THRESHOLD_DAYS: +-- THRESHOLD_BASED -> ARCH_THRESHOLD_DAYS set, ARCH_MINIMUM_AGE_MONTHS NULL +-- MINIMUM_AGE_MONTHS -> ARCH_THRESHOLD_DAYS NULL, ARCH_MINIMUM_AGE_MONTHS set +-- HYBRID -> both columns set +-- Prerequisites: A_TABLE_STAT and A_TABLE_STAT_HIST tables exist +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +WHENEVER SQLERROR EXIT SQL.SQLCODE + +-- ---------------------------------------------------------------------------- +-- ARCHIVAL_STRATEGY +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( + ARCHIVAL_STRATEGY VARCHAR2(30) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS + ''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; + DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( + ARCHIVAL_STRATEGY VARCHAR2(30) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS + ''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; + DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- ARCH_MINIMUM_AGE_MONTHS +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( + ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS + ''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; + DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( + ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS + ''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; + DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- IS_WORKFLOW_SUCCESS_REQUIRED (A_TABLE_STAT, A_TABLE_STAT_HIST) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS + ''Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows regardless. Added MARS-1409'''; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS + ''Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows regardless. Added MARS-1409'''; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- IS_WORKFLOW_SUCCESS_REQUIRED (A_SOURCE_FILE_CONFIG) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_SOURCE_FILE_CONFIG... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_SOURCE_FILE_CONFIG' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_SOURCE_FILE_CONFIG.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD ( + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT ''Y'' NOT NULL + )'; + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT + CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN (''Y'', ''N''))'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS + ''Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409'''; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_SOURCE_FILE_CONFIG successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_SOURCE_FILE_CONFIG: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE +-- (A_TABLE_STAT) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE columns to A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT already exists in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( + WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_SIZE NUMBER(38,0) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_FILE_COUNT IS + ''Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_ROW_COUNT IS + ''Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_SIZE IS + ''Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding WORKFLOW_SUCCESS_* columns to A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE +-- (A_TABLE_STAT_HIST) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Adding WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE columns to A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT'; + + IF vColumnExists > 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT already exists in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE ' + ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( + WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_SIZE NUMBER(38,0) + )'; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_FILE_COUNT IS + ''Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_ROW_COUNT IS + ''Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + EXECUTE IMMEDIATE ' + COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_SIZE IS + ''Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'''; + DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR adding WORKFLOW_SUCCESS_* columns to A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Step 10 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns added to both statistics tables. +PROMPT IS_WORKFLOW_SUCCESS_REQUIRED added to A_SOURCE_FILE_CONFIG, A_TABLE_STAT and A_TABLE_STAT_HIST. +PROMPT WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT and A_TABLE_STAT_HIST. +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql b/MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql rename to MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql b/MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql rename to MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql b/MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql rename to MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql b/MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql rename to MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql b/MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql rename to MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql b/MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql rename to MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql b/MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql rename to MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/100_MARS_1409_rollback_archival_strategy_column.sql b/MARS_Packages/REL02_POST/MARS-1409/100_MARS_1409_rollback_archival_strategy_column.sql deleted file mode 100644 index d63f59d..0000000 --- a/MARS_Packages/REL02_POST/MARS-1409/100_MARS_1409_rollback_archival_strategy_column.sql +++ /dev/null @@ -1,122 +0,0 @@ --- ============================================================================ --- MARS-1409 Rollback Step 100: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS --- columns from A_TABLE_STAT and A_TABLE_STAT_HIST --- ============================================================================ --- Purpose: Rollback of step 10 - remove both stat self-documentation columns --- Prerequisites: Step 10 was applied --- ============================================================================ - -SET SERVEROUTPUT ON SIZE UNLIMITED -WHENEVER SQLERROR EXIT SQL.SQLCODE - --- ---------------------------------------------------------------------------- --- ARCH_MINIMUM_AGE_MONTHS (drop first - added second) --- ---------------------------------------------------------------------------- - -PROMPT -PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT' - AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; - - IF vColumnExists = 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT.'); - ELSE - EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCH_MINIMUM_AGE_MONTHS'; - DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT_HIST... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT_HIST' - AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; - - IF vColumnExists = 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT_HIST.'); - ELSE - EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCH_MINIMUM_AGE_MONTHS'; - DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT_HIST successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT_HIST: ' || SQLERRM); - RAISE; -END; -/ - --- ---------------------------------------------------------------------------- --- ARCHIVAL_STRATEGY --- ---------------------------------------------------------------------------- - -PROMPT -PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT' - AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; - - IF vColumnExists = 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT.'); - ELSE - EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCHIVAL_STRATEGY'; - DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT_HIST... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT_HIST' - AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; - - IF vColumnExists = 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT_HIST.'); - ELSE - EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCHIVAL_STRATEGY'; - DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT_HIST successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT_HIST: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Rollback step 100 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns removed from both statistics tables. -PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_add_archival_strategy_column.sql b/MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_add_archival_strategy_column.sql deleted file mode 100644 index 46e0b67..0000000 --- a/MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_add_archival_strategy_column.sql +++ /dev/null @@ -1,153 +0,0 @@ --- ============================================================================ --- MARS-1409 Step 10: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns --- to A_TABLE_STAT and A_TABLE_STAT_HIST tables --- ============================================================================ --- Purpose: Make statistics records fully self-documenting: --- ARCHIVAL_STRATEGY - strategy in effect when stats were gathered --- (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID) --- ARCH_MINIMUM_AGE_MONTHS - minimum age threshold in months; complements --- ARCH_THRESHOLD_DAYS: --- THRESHOLD_BASED -> ARCH_THRESHOLD_DAYS set, ARCH_MINIMUM_AGE_MONTHS NULL --- MINIMUM_AGE_MONTHS -> ARCH_THRESHOLD_DAYS NULL, ARCH_MINIMUM_AGE_MONTHS set --- HYBRID -> both columns set --- Prerequisites: A_TABLE_STAT and A_TABLE_STAT_HIST tables exist --- ============================================================================ - -SET SERVEROUTPUT ON SIZE UNLIMITED -WHENEVER SQLERROR EXIT SQL.SQLCODE - --- ---------------------------------------------------------------------------- --- ARCHIVAL_STRATEGY --- ---------------------------------------------------------------------------- - -PROMPT -PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT' - AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; - - IF vColumnExists > 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT.'); - ELSE - EXECUTE IMMEDIATE ' - ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( - ARCHIVAL_STRATEGY VARCHAR2(30) - )'; - EXECUTE IMMEDIATE ' - COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS - ''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; - DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT_HIST... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT_HIST' - AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; - - IF vColumnExists > 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT_HIST.'); - ELSE - EXECUTE IMMEDIATE ' - ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( - ARCHIVAL_STRATEGY VARCHAR2(30) - )'; - EXECUTE IMMEDIATE ' - COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS - ''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; - DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT_HIST successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT_HIST: ' || SQLERRM); - RAISE; -END; -/ - --- ---------------------------------------------------------------------------- --- ARCH_MINIMUM_AGE_MONTHS --- ---------------------------------------------------------------------------- - -PROMPT -PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT' - AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; - - IF vColumnExists > 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT.'); - ELSE - EXECUTE IMMEDIATE ' - ALTER TABLE CT_MRDS.A_TABLE_STAT ADD ( - ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) - )'; - EXECUTE IMMEDIATE ' - COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS - ''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; - DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT_HIST... - -DECLARE - vColumnExists NUMBER := 0; -BEGIN - SELECT COUNT(*) INTO vColumnExists - FROM ALL_TAB_COLUMNS - WHERE OWNER = 'CT_MRDS' - AND TABLE_NAME = 'A_TABLE_STAT_HIST' - AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; - - IF vColumnExists > 0 THEN - DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT_HIST.'); - ELSE - EXECUTE IMMEDIATE ' - ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD ( - ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) - )'; - EXECUTE IMMEDIATE ' - COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS - ''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'''; - DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT_HIST successfully.'); - END IF; -EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT_HIST: ' || SQLERRM); - RAISE; -END; -/ - -PROMPT -PROMPT Step 10 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns added to both statistics tables. -PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_verify_installation.sql b/MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_verify_installation.sql similarity index 100% rename from MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_verify_installation.sql rename to MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_verify_installation.sql diff --git a/MARS_Packages/REL02_POST/MARS-1409/98_MARS_1409_rollback_archival_strategy_column.sql b/MARS_Packages/REL02_POST/MARS-1409/98_MARS_1409_rollback_archival_strategy_column.sql new file mode 100644 index 0000000..f7f3487 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/98_MARS_1409_rollback_archival_strategy_column.sql @@ -0,0 +1,264 @@ +-- ============================================================================ +-- MARS-1409 Rollback Step 100: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS +-- columns from A_TABLE_STAT and A_TABLE_STAT_HIST; +-- drop IS_WORKFLOW_SUCCESS_REQUIRED from A_SOURCE_FILE_CONFIG +-- ============================================================================ +-- Purpose: Rollback of step 10 - remove both stat self-documentation columns +-- Prerequisites: Step 10 was applied +-- ============================================================================ + +SET SERVEROUTPUT ON SIZE UNLIMITED +WHENEVER SQLERROR EXIT SQL.SQLCODE + +-- ---------------------------------------------------------------------------- +-- ARCH_MINIMUM_AGE_MONTHS (drop first - added second) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCH_MINIMUM_AGE_MONTHS'; + DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCH_MINIMUM_AGE_MONTHS'; + DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- ARCHIVAL_STRATEGY +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCHIVAL_STRATEGY'; + DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'ARCHIVAL_STRATEGY'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCHIVAL_STRATEGY'; + DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- IS_WORKFLOW_SUCCESS_REQUIRED (A_TABLE_STAT, A_TABLE_STAT_HIST) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED'; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED'; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- IS_WORKFLOW_SUCCESS_REQUIRED (A_SOURCE_FILE_CONFIG) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_SOURCE_FILE_CONFIG... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_SOURCE_FILE_CONFIG' + AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_SOURCE_FILE_CONFIG.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED'; + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED'; + DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_SOURCE_FILE_CONFIG successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_SOURCE_FILE_CONFIG: ' || SQLERRM); + RAISE; +END; +/ + +-- ---------------------------------------------------------------------------- +-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE +-- (drop from A_TABLE_STAT and A_TABLE_STAT_HIST) +-- ---------------------------------------------------------------------------- + +PROMPT +PROMPT Dropping WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE from A_TABLE_STAT... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT' + AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT does not exist in A_TABLE_STAT.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP (WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE)'; + DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE dropped from A_TABLE_STAT successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping WORKFLOW_SUCCESS_* columns from A_TABLE_STAT: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Dropping WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE from A_TABLE_STAT_HIST... + +DECLARE + vColumnExists NUMBER := 0; +BEGIN + SELECT COUNT(*) INTO vColumnExists + FROM ALL_TAB_COLUMNS + WHERE OWNER = 'CT_MRDS' + AND TABLE_NAME = 'A_TABLE_STAT_HIST' + AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT'; + + IF vColumnExists = 0 THEN + DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT does not exist in A_TABLE_STAT_HIST.'); + ELSE + EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP (WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE)'; + DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE dropped from A_TABLE_STAT_HIST successfully.'); + END IF; +EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR dropping WORKFLOW_SUCCESS_* columns from A_TABLE_STAT_HIST: ' || SQLERRM); + RAISE; +END; +/ + +PROMPT +PROMPT Rollback step 100 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns removed from both statistics tables. +PROMPT IS_WORKFLOW_SUCCESS_REQUIRED removed from A_SOURCE_FILE_CONFIG, A_TABLE_STAT and A_TABLE_STAT_HIST. +PROMPT WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE removed from A_TABLE_STAT and A_TABLE_STAT_HIST. +PROMPT diff --git a/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql b/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql index a607171..80aa0c4 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql +++ b/MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql @@ -31,8 +31,8 @@ PROMPT ========================================================================= PROMPT MARS-1409 Installation Starting PROMPT ============================================================================ PROMPT Package: CT_MRDS.FILE_MANAGER v3.X.X -PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED, ARCHIVAL_STRATEGY to A_TABLE_STAT/HIST -PROMPT Purpose: Direct tracking of workflow history keys in file registration; self-documenting statistics records +PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED; add ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_* columns to A_TABLE_STAT/HIST +PROMPT Purpose: Direct tracking of workflow history keys in file registration; self-documenting statistics records; separate total vs workflow-success statistics PROMPT Steps: 12 (DDL x2, ENV_MANAGER Update, FILE_MANAGER Update, FILE_ARCHIVER Update, Trigger Update, Verification, Tracking, Version Verification) PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; @@ -56,57 +56,57 @@ PROMPT ========================================================================= PROMPT PROMPT ============================================================================ -PROMPT STEP 2: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns to A_TABLE_STAT and A_TABLE_STAT_HIST +PROMPT STEP 2: Add ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_FILE_COUNT/ROW_COUNT/SIZE columns to A_TABLE_STAT and A_TABLE_STAT_HIST PROMPT ============================================================================ -@@10_MARS_1409_add_archival_strategy_column.sql +@@02_MARS_1409_add_archival_strategy_column.sql PROMPT PROMPT ============================================================================ PROMPT STEP 3: Update ENV_MANAGER package specification PROMPT ============================================================================ -@@02_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql +@@03_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql PROMPT PROMPT ============================================================================ PROMPT STEP 4: Update ENV_MANAGER package body PROMPT ============================================================================ -@@03_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql +@@04_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql PROMPT PROMPT ============================================================================ PROMPT STEP 5: Update FILE_MANAGER package specification PROMPT ============================================================================ -@@04_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql +@@05_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql PROMPT PROMPT ============================================================================ PROMPT STEP 6: Update FILE_MANAGER package body PROMPT ============================================================================ -@@05_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql +@@06_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql PROMPT PROMPT ============================================================================ PROMPT STEP 7: Update FILE_ARCHIVER package specification PROMPT ============================================================================ -@@06_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql +@@07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql PROMPT PROMPT ============================================================================ PROMPT STEP 8: Update FILE_ARCHIVER package body PROMPT ============================================================================ -@@07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql +@@08_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql PROMPT PROMPT ============================================================================ PROMPT STEP 9: Update A_WORKFLOW_HISTORY trigger PROMPT ============================================================================ -@@08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql +@@09_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql PROMPT PROMPT ============================================================================ PROMPT STEP 10: Verify installation PROMPT ============================================================================ -@@09_MARS_1409_verify_installation.sql +@@10_MARS_1409_verify_installation.sql PROMPT PROMPT ============================================================================ diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql new file mode 100644 index 0000000..08f32e5 --- /dev/null +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql @@ -0,0 +1,116 @@ +-- ==================================================================== +-- A_SOURCE_FILE_CONFIG Table +-- ==================================================================== +-- Purpose: Store source file configuration and processing rules +-- MARS-1049: Added ENCODING column for CSV character set support +-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation +-- MARS-1409: Added IS_WORKFLOW_SUCCESS_REQUIRED flag for workflow bypass +-- ==================================================================== + +CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( + A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE, + A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE, + SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG' + SOURCE_FILE_ID VARCHAR2(200), + SOURCE_FILE_DESC VARCHAR2(2000), + SOURCE_FILE_NAME_PATTERN VARCHAR2(200), + TABLE_ID VARCHAR2(200), + TEMPLATE_TABLE_NAME VARCHAR2(200), + CONTAINER_FILE_KEY NUMBER(38,0), + ARCHIVE_THRESHOLD_DAYS NUMBER(4,0), + ARCHIVE_THRESHOLD_FILES_COUNT NUMBER(38,0), + ARCHIVE_THRESHOLD_BYTES_SUM NUMBER(38,0), + ODS_SCHEMA_NAME VARCHAR2(100), + ARCHIVE_THRESHOLD_ROWS_COUNT NUMBER(38,0), + HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3), + ARCHIVAL_STRATEGY VARCHAR2(50), + MINIMUM_AGE_MONTHS NUMBER(3,0), + ENCODING VARCHAR2(50) DEFAULT 'UTF8', + IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL, + IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL, + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL, + CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')), + CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')), + CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')), + CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), + CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), + CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), + CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID) +) TABLESPACE "DATA"; + +-- Primary key index (from production export) +CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK" +ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY") +TABLESPACE "DATA"; + +-- Unique constraint index (from production export) +CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1" +ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID") +TABLESPACE "DATA"; + +-- Column comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS + 'Primary key - unique identifier for source file configuration record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS + 'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS + 'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS + 'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS + 'Human-readable description of the source file and its purpose'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS + 'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS + 'Identifier for the target table where data will be loaded (without schema prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS + 'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS + 'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS + 'Threshold for THRESHOLD_BASED strategy: archive data older than N days'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS + 'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS + 'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS + 'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS + 'Schema name where ODS external tables are created (typically ODS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS + 'Number of hours before table statistics expire and need to be recalculated'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS + 'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS + 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS + 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS + 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS + 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS + 'Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409'; + +GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT.sql b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT.sql index 74db266..c99835b 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT.sql +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT.sql @@ -18,11 +18,19 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT ( CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, ARCHIVAL_STRATEGY VARCHAR2(30), ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1), + WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_SIZE NUMBER(38,0), CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY) ) TABLESPACE "DATA"; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG when statistics were gathered. Y=OVER_ARCH_THRESOLD counts include only WORKFLOW_SUCCESSFUL=Y rows. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_FILE_COUNT IS 'Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_ROW_COUNT IS 'Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_SIZE IS 'Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; -- Unique constraint index (from production export) CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1" diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT_HIST.sql b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT_HIST.sql index 71cbbb3..825e0be 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT_HIST.sql +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT_HIST.sql @@ -17,8 +17,16 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST ( ARCH_THRESHOLD_DAYS NUMBER(4,0), CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, ARCHIVAL_STRATEGY VARCHAR2(30), - ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) + ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1), + WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0), + WORKFLOW_SUCCESS_SIZE NUMBER(38,0) ) TABLESPACE "DATA"; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; -COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; \ No newline at end of file +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG when statistics were gathered. Y=OVER_ARCH_THRESOLD counts include only WORKFLOW_SUCCESSFUL=Y rows. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_FILE_COUNT IS 'Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_ROW_COUNT IS 'Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_SIZE IS 'Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409'; \ No newline at end of file diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb index 1b56755..b009ed7 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb @@ -166,7 +166,7 @@ AS join CT_MRDS.a_workflow_history h on s.a_workflow_history_key = h.a_workflow_history_key where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' - and h.WORKFLOW_SUCCESSFUL = ''Y'' + ' || CASE WHEN vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN 'and h.WORKFLOW_SUCCESSFUL = ''Y''' ELSE '' END || ' group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')' ; @@ -199,7 +199,7 @@ AS on s.a_workflow_history_key = h.a_workflow_history_key and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||''' and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||''' - and h.WORKFLOW_SUCCESSFUL = ''Y'' + '|| CASE WHEN vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN 'and h.WORKFLOW_SUCCESSFUL = ''Y''' ELSE '' END ||' ' ; vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; @@ -450,8 +450,9 @@ AS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vTableName VARCHAR2(200); vQuery VARCHAR2(32000); - vWhereClause VARCHAR2(4000); - vOdsBucketUri VARCHAR2(1000); + vWhereClause VARCHAR2(4000); + vOverThresholdWhereClause VARCHAR2(4000); + vOdsBucketUri VARCHAR2(1000); BEGIN vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); @@ -467,13 +468,28 @@ AS -- Get ODS bucket URI before building query vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + -- Build WHERE clause for OVER_ARCH_THRESOLD columns: + -- Combines archival strategy time-condition with optional workflow success filter. + -- IS_WORKFLOW_SUCCESS_REQUIRED='Y': only files with WORKFLOW_SUCCESSFUL='Y' are counted as eligible. + -- IS_WORKFLOW_SUCCESS_REQUIRED='N': all files passing the time-condition are counted as eligible. + IF vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN + vOverThresholdWhereClause := vWhereClause || ' AND workflow_successful = ''Y'''; + ELSE + vOverThresholdWhereClause := vWhereClause; + END IF; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOverThresholdWhereClause','DEBUG',vOverThresholdWhereClause); + -- Use strategy-based WHERE clause for statistics (MARS-828) + -- FILE_COUNT, ROW_COUNT, SIZE: all files regardless of workflow success (never zero due to workflow filter) + -- OVER_ARCH_THRESOLD_*: IS_WORKFLOW_SUCCESS_REQUIRED-aware count of eligible files + -- WORKFLOW_SUCCESS_*: informational count of files with WORKFLOW_SUCCESSFUL='Y' vQuery := 'with tmp as ( select s.* ,file$name as filename ,h.workflow_start + ,h.workflow_successful , to_char(h.workflow_start,''yyyy'') as year , to_char(h.workflow_start,''mm'') as month from '||vTableName||' s @@ -482,7 +498,10 @@ AS ) , tmp_gr as ( select - filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start + filename + ,count(*) as row_count_per_file + ,min(workflow_start) as workflow_start + ,max(workflow_successful) as workflow_successful from tmp group by filename ) @@ -491,15 +510,19 @@ AS ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY ,'''||vTableName||''' as TABLE_NAME ,count(*) as FILE_COUNT - ,nvl(sum(case when ' || vWhereClause || ' then 1 else 0 end), 0) as OLD_FILE_COUNT + ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then 1 else 0 end), 0) as OLD_FILE_COUNT ,nvl(sum(row_count_per_file), 0) as ROW_COUNT - ,nvl(sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end), 0) as OLD_ROW_COUNT + ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then row_count_per_file else 0 end), 0) as OLD_ROW_COUNT ,nvl(sum(r.bytes), 0) as BYTES - ,nvl(sum(case when ' || vWhereClause || ' then r.bytes else 0 end), 0) as OLD_BYTES + ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then r.bytes else 0 end), 0) as OLD_BYTES ,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS ,systimestamp as CREATED ,'''||vSourceFileConfig.ARCHIVAL_STRATEGY||''' as ARCHIVAL_STRATEGY ,'||COALESCE(TO_CHAR(vSourceFileConfig.MINIMUM_AGE_MONTHS), 'NULL')||' as ARCH_MINIMUM_AGE_MONTHS + ,'''||vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED||''' as IS_WORKFLOW_SUCCESS_REQUIRED + ,nvl(sum(case when workflow_successful = ''Y'' then 1 else 0 end), 0) as WORKFLOW_SUCCESS_FILE_COUNT + ,nvl(sum(case when workflow_successful = ''Y'' then row_count_per_file else 0 end), 0) as WORKFLOW_SUCCESS_ROW_COUNT + ,nvl(sum(case when workflow_successful = ''Y'' then r.bytes else 0 end), 0) as WORKFLOW_SUCCESS_SIZE from tmp_gr t join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''', @@ -528,6 +551,10 @@ AS vStats.CREATED := SYSTIMESTAMP; vStats.ARCHIVAL_STRATEGY := vSourceFileConfig.ARCHIVAL_STRATEGY; vStats.ARCH_MINIMUM_AGE_MONTHS := vSourceFileConfig.MINIMUM_AGE_MONTHS; + vStats.IS_WORKFLOW_SUCCESS_REQUIRED := vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED; + vStats.WORKFLOW_SUCCESS_FILE_COUNT := 0; + vStats.WORKFLOW_SUCCESS_ROW_COUNT := 0; + vStats.WORKFLOW_SUCCESS_SIZE := 0; ELSE RAISE; END IF; diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg index 2ec4608..2f51e2b 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg @@ -17,12 +17,15 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.1'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-13 09:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-17 11:00:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.4.0 (2026-03-17): MARS-1409 - Added IS_WORKFLOW_SUCCESS_REQUIRED flag to A_SOURCE_FILE_CONFIG (DEFAULT Y). ' || + 'Y=standard DBT flow (WORKFLOW_SUCCESSFUL=Y required), N=bypass for manual/non-DBT sources. ' || + 'Flag value stored in A_TABLE_STAT and A_TABLE_STAT_HIST for full audit of statistics basis.' || CHR(13)||CHR(10) || '3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) || '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) || '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb index 7015998..f4dcef9 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb @@ -1643,7 +1643,12 @@ AS ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL - ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049 + ,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409 + ,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828 + ,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828 + ,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828 + ,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828 ) IS vSourceFileConfigKey PLS_INTEGER; vSourceKeyExists PLS_INTEGER := 0; @@ -1659,10 +1664,15 @@ AS ,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||'''' ,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||'''' ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY + ,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409 + ,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828 + ,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||'''' -- MARS-828 + ,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828 + ,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828 )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); - INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING) - VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding); + INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS) + VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths); COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION @@ -1772,6 +1782,12 @@ AS ||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM ||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS + ||cgBL||pLevel||'ENCODING = '||pSourceFileConfig.ENCODING + ||cgBL||pLevel||'IS_ARCHIVE_ENABLED = '||pSourceFileConfig.IS_ARCHIVE_ENABLED + ||cgBL||pLevel||'IS_KEEP_IN_TRASH = '||pSourceFileConfig.IS_KEEP_IN_TRASH + ||cgBL||pLevel||'ARCHIVAL_STRATEGY = '||pSourceFileConfig.ARCHIVAL_STRATEGY + ||cgBL||pLevel||'MINIMUM_AGE_MONTHS = '||pSourceFileConfig.MINIMUM_AGE_MONTHS + ||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED ||cgBL||pLevel||''||'--------------------------------' ; diff --git a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg index 0b22c58..59669d3 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg +++ b/MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg @@ -17,12 +17,14 @@ AS **/ -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) - PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.6.1'; - PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-13 09:00:00'; + PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.6.3'; + PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-17 12:30:00'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; -- Version History (Latest changes first) VERSION_HISTORY CONSTANT VARCHAR2(4000) := + '3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) || + '3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) || '3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) || '3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) || '3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) || @@ -443,13 +445,23 @@ AS * @name ADD_SOURCE_FILE_CONFIG * @desc Insert a new record to A_SOURCE_FILE_CONFIG table. * MARS-1049: Added pEncoding parameter for CSV character set specification. + * MARS-1409: Added pIsWorkflowSuccessRequired parameter. + * MARS-828: Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths. * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2') * If NULL, no CHARACTERSET clause is added to external table definitions + * @param pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow) + * 'N' = archive regardless of workflow status (bypass for manual/non-DBT sources) + * @param pIsArchiveEnabled - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled + * @param pIsKeepInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly + * @param pArchivalStrategy - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL + * @param pMinimumAgeMonths - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy) * @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( * pSourceKey => 'C2D', pSourceFileType => 'INPUT', * pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS', * pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS', - * pEncoding => 'UTF8' + * pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y', + * pIsArchiveEnabled => 'Y', pIsKeepInTrash => 'N', + * pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3 * ); **/ PROCEDURE ADD_SOURCE_FILE_CONFIG ( @@ -461,7 +473,12 @@ AS ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL - ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR + ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049 + ,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409 + ,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828 + ,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828 + ,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828 + ,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828 ); diff --git a/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql b/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql index 9898ad8..bb83fd6 100644 --- a/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql +++ b/MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql @@ -32,7 +32,7 @@ PROMPT MARS-1409 Rollback Starting PROMPT ============================================================================ PROMPT Package: CT_MRDS.FILE_MANAGER PROMPT Change: Remove A_WORKFLOW_HISTORY_KEY column and restore previous version -PROMPT Steps: 11 (Restore FILE_ARCHIVER, Restore FILE_MANAGER, Restore ENV_MANAGER, Restore trigger, Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns, Clear data, Drop column, Verify) +PROMPT Steps: 11 (Restore FILE_ARCHIVER, Restore FILE_MANAGER, Restore ENV_MANAGER, Restore trigger, Drop ARCHIVAL_STRATEGY/ARCH_MINIMUM_AGE_MONTHS/IS_WORKFLOW_SUCCESS_REQUIRED/WORKFLOW_SUCCESS_* columns, Drop column, Verify) PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; PROMPT ============================================================================ @@ -91,9 +91,9 @@ PROMPT ========================================================================= PROMPT PROMPT ============================================================================ -PROMPT STEP 8: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns from A_TABLE_STAT and A_TABLE_STAT_HIST +PROMPT STEP 8: Drop ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_FILE_COUNT/ROW_COUNT/SIZE columns from A_TABLE_STAT and A_TABLE_STAT_HIST PROMPT ============================================================================ -@@100_MARS_1409_rollback_archival_strategy_column.sql +@@98_MARS_1409_rollback_archival_strategy_column.sql PROMPT PROMPT ============================================================================ diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql index 739cfc9..08f32e5 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql @@ -4,6 +4,7 @@ -- Purpose: Store source file configuration and processing rules -- MARS-1049: Added ENCODING column for CSV character set support -- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation +-- MARS-1409: Added IS_WORKFLOW_SUCCESS_REQUIRED flag for workflow bypass -- ==================================================================== CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( @@ -27,9 +28,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( ENCODING VARCHAR2(50) DEFAULT 'UTF8', IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL, IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL, + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL, CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')), CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')), + CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')), CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), @@ -107,4 +110,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS + 'Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409'; + GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql index 74db266..58e5842 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql @@ -18,11 +18,13 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT ( CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, ARCHIVAL_STRATEGY VARCHAR2(30), ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1), CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY) ) TABLESPACE "DATA"; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409'; -- Unique constraint index (from production export) CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1" diff --git a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql index 71cbbb3..d8b5405 100644 --- a/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql +++ b/MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql @@ -17,8 +17,10 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST ( ARCH_THRESHOLD_DAYS NUMBER(4,0), CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, ARCHIVAL_STRATEGY VARCHAR2(30), - ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) + ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), + IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) ) TABLESPACE "DATA"; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; -COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; \ No newline at end of file +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; +COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409'; \ No newline at end of file diff --git a/confluence/FILE_ARCHIVER_Guide.md b/confluence/FILE_ARCHIVER_Guide.md index a17d64f..dca865a 100644 --- a/confluence/FILE_ARCHIVER_Guide.md +++ b/confluence/FILE_ARCHIVER_Guide.md @@ -10,13 +10,14 @@ The FILE_ARCHIVER package provides flexible archival strategies that accommodate - **Three Archival Strategies**: THRESHOLD_BASED, MINIMUM_AGE_MONTHS (with 0=current month only), HYBRID - **Flexible Configuration**: Per-table archival strategy configuration via A_SOURCE_FILE_CONFIG +- **Workflow Bypass**: IS_WORKFLOW_SUCCESS_REQUIRED flag allows archivization of files from non-DBT sources - **Validation**: Automatic validation of strategy-specific configuration requirements ### Package Information - **Schema**: CT_MRDS - **Package**: FILE_ARCHIVER -- **Current Version**: 3.3.0 +- **Current Version**: 3.4.0 - **Dependencies**: ENV_MANAGER, FILE_MANAGER, A_SOURCE_FILE_CONFIG, A_SOURCE_FILE_RECEIVED, A_WORKFLOW_HISTORY ### Critical Prerequisites @@ -67,7 +68,7 @@ END; | Strategy | WHERE Clause Logic | Configuration Required | Primary Use Case | |----------|-------------------|----------------------|------------------| -| `THRESHOLD_BASED` | Days since workflow start > threshold | DAYS_FOR_ARCHIVE_THRESHOLD | Simple time-based archival | +| `THRESHOLD_BASED` | Days since workflow start > threshold | ARCHIVE_THRESHOLD_DAYS | Simple time-based archival | | `MINIMUM_AGE_MONTHS` | Archive data older than X months (0=current month only) | MINIMUM_AGE_MONTHS (≥0) | All sources - flexible retention (0 for LM, 6 for CSDB) | | `HYBRID` | Combines month boundary + minimum age | MINIMUM_AGE_MONTHS | Advanced retention scenarios | @@ -77,14 +78,14 @@ Archives data based on number of days since workflow start. **WHERE Clause**: ```sql -extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRESHOLD +extract(day from (systimestamp - workflow_start)) > ARCHIVE_THRESHOLD_DAYS ``` **Configuration**: ```sql UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'THRESHOLD_BASED', - DAYS_FOR_ARCHIVE_THRESHOLD = 30, + ARCHIVE_THRESHOLD_DAYS = 30, MINIMUM_AGE_MONTHS = NULL WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'C2D_DATA' @@ -185,17 +186,17 @@ END IF; **Behavior**: Archives data **only when** at least one of the following thresholds is exceeded: -1. **FILES_COUNT_OVER_ARCHIVE_THRESHOLD** - Number of files eligible for archival -2. **ROWS_COUNT_OVER_ARCHIVE_THRESHOLD** - Number of rows eligible for archival -3. **BYTES_SUM_OVER_ARCHIVE_THRESHOLD** - Total size in bytes eligible for archival +1. **ARCHIVE_THRESHOLD_FILES_COUNT** - Number of files eligible for archival +2. **ARCHIVE_THRESHOLD_ROWS_COUNT** - Number of rows eligible for archival +3. **ARCHIVE_THRESHOLD_BYTES_SUM** - Total size in bytes eligible for archival ```sql -- Executed for THRESHOLD_BASED and HYBRID strategies -IF vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD THEN +IF vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT THEN vArchivalTriggeredBy := 'FILES_COUNT'; -ELSIF vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD THEN +ELSIF vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT THEN vArchivalTriggeredBy := 'ROWS_COUNT'; -ELSIF vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD THEN +ELSIF vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM THEN vArchivalTriggeredBy := 'BYTES_SUM'; END IF; ``` @@ -206,9 +207,9 @@ END IF; ```sql -- Set archival thresholds for THRESHOLD_BASED strategy UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, -- Archive when 10+ files eligible - ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000, -- Archive when 100k+ rows eligible - BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 104857600 -- Archive when 100MB+ eligible +SET ARCHIVE_THRESHOLD_FILES_COUNT = 10, -- Archive when 10+ files eligible + ARCHIVE_THRESHOLD_ROWS_COUNT = 100000, -- Archive when 100k+ rows eligible + ARCHIVE_THRESHOLD_BYTES_SUM = 104857600 -- Archive when 100MB+ eligible WHERE ARCHIVAL_STRATEGY = 'THRESHOLD_BASED' AND TABLE_ID = 'YOUR_TABLE'; ``` @@ -243,15 +244,15 @@ WHERE ...; ## Archival Control Configuration -### ARCHIVE_ENABLED Column +### IS_ARCHIVE_ENABLED Column Controls whether archival is enabled for specific table configuration. -**Column**: `A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED` (VARCHAR2(1), DEFAULT 'Y') +**Column**: `A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED` (CHAR(1), DEFAULT 'N' NOT NULL) **Values**: -- `'Y'` (default) - Table is eligible for archival processing -- `'N'` - Table is excluded from archival (batch operations skip this config) +- `'Y'` - Table is eligible for archival processing +- `'N'` (default) - Table is excluded from archival (batch operations skip this config) **Use Cases**: - Disable archival for specific tables without removing configuration @@ -262,7 +263,7 @@ Controls whether archival is enabled for specific table configuration. ```sql -- Disable archival for specific table UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVE_ENABLED = 'N' +SET IS_ARCHIVE_ENABLED = 'N' WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'CSDB' AND TABLE_ID = 'CSDB_DEBT'; @@ -270,7 +271,7 @@ COMMIT; -- Re-enable archival UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVE_ENABLED = 'Y' +SET IS_ARCHIVE_ENABLED = 'Y' WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'CSDB' AND TABLE_ID = 'CSDB_DEBT'; @@ -280,22 +281,22 @@ COMMIT; SELECT SOURCE_FILE_ID, TABLE_ID, - ARCHIVE_ENABLED, + IS_ARCHIVE_ENABLED, ARCHIVAL_STRATEGY FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' ORDER BY SOURCE_FILE_ID, TABLE_ID; ``` -### KEEP_IN_TRASH Column +### IS_KEEP_IN_TRASH Column Controls TRASH folder retention policy for archived files. -**Column**: `A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH` (VARCHAR2(1), DEFAULT 'Y') +**Column**: `A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL) **Values**: -- `'Y'` (default) - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED) -- `'N'` - CSV files deleted from TRASH folder after archival (status: ARCHIVED_AND_PURGED) +- `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED) +- `'N'` (default) - CSV files deleted from TRASH folder after archival (status: ARCHIVED_AND_PURGED) **Benefits of TRASH Retention (TRUE)**: - Safety net for rollback if archival issues discovered @@ -311,7 +312,7 @@ Controls TRASH folder retention policy for archived files. ```sql -- Production: Keep files in TRASH (recommended) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET KEEP_IN_TRASH = 'Y' +SET IS_KEEP_IN_TRASH = 'Y' WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'LM' AND TABLE_ID LIKE 'LM_%'; @@ -319,20 +320,64 @@ COMMIT; -- Test environment: Cleanup TRASH to save storage UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET KEEP_IN_TRASH = 'N' +SET IS_KEEP_IN_TRASH = 'N' WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'TEST_SOURCE'; COMMIT; -- Bulk configuration by source UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET KEEP_IN_TRASH = 'Y' +SET IS_KEEP_IN_TRASH = 'Y' WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM'); COMMIT; ``` +### IS_WORKFLOW_SUCCESS_REQUIRED Column -## Data Lifecycle Workflow +Controls whether archivization requires `WORKFLOW_SUCCESSFUL='Y'` in A_WORKFLOW_HISTORY. Added in MARS-1409. + +**Column**: `A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED` (CHAR(1), DEFAULT 'Y' NOT NULL) + +**Values**: +- `'Y'` (default) - Only files with `WORKFLOW_SUCCESSFUL='Y'` are eligible for archivization (standard Airflow+DBT flow) +- `'N'` - Archivization proceeds regardless of workflow completion status (bypass for manual/non-DBT sources) + +**Use Cases**: +- `'Y'`: All standard INBOX-validated sources (LM, CSDB, C2D) - ensures only fully-processed files are archived +- `'N'`: Legacy data migrated via `DATA_EXPORTER`, manual uploads, or any source without DBT workflow tracking + +**GATHER_TABLE_STAT behavior**: +- `'Y'`: Statistics (file count, row count, byte sum) counted only from files with `WORKFLOW_SUCCESSFUL='Y'` +- `'N'`: Statistics counted from all INGESTED files regardless of workflow outcome + +**Configuration Example**: +```sql +-- Standard source: require DBT workflow completion (default) +UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG +SET IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' +WHERE SOURCE_FILE_TYPE = 'INPUT' + AND SOURCE_FILE_ID = 'LM'; +COMMIT; + +-- Non-DBT source: bypass workflow check +UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG +SET IS_WORKFLOW_SUCCESS_REQUIRED = 'N' +WHERE SOURCE_FILE_TYPE = 'INPUT' + AND SOURCE_FILE_ID = 'MANUAL_UPLOAD'; +COMMIT; + +-- Or set at configuration time via ADD_SOURCE_FILE_CONFIG +CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( + pSourceKey => 'MANUAL', + pSourceFileType => 'INPUT', + pSourceFileId => 'MANUAL_UPLOAD', + pSourceFileDesc => 'Manual data upload without DBT', + pSourceFileNamePattern => 'manual_*.csv', + pTableId => 'MY_TABLE', + pTemplateTableName => 'CT_ET_TEMPLATES.MY_TABLE', + pIsWorkflowSuccessRequired => 'N' -- bypass workflow check +); +``` ### Status Tracking in A_SOURCE_FILE_RECEIVED @@ -348,7 +393,7 @@ INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional) **Status Descriptions**: - **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket - **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket) -- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when KEEP_IN_TRASH='N') +- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEEP_IN_TRASH='N') **Associated Columns Updated During Archival**: ```sql @@ -390,9 +435,9 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC 2.1 TRASH Subfolder (DATA Bucket - File Retention) ├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME) ├─ Stores CSV files after archival to Parquet - ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by KEEP_IN_TRASH config) + ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEEP_IN_TRASH config) ├─ Enables rollback if archival issues occur - └─ Optional cleanup: ARCHIVED_AND_PURGED (when KEEP_IN_TRASH = 'N') + └─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEEP_IN_TRASH = 'N') 3. ARCHIVE Bucket (Long-term Storage) ├─ Historical data in Parquet format @@ -402,14 +447,14 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC **Key Procedures**: - `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause - - TRASH folder retention controlled by `KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG + - TRASH folder retention controlled by `IS_KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG - `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling - **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey` - **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey` - **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll` - **Error Handling**: Continues processing other tables on individual failures - - **Filtering**: Respects `ARCHIVE_ENABLED='Y'` (skips disabled configurations) - - **Individual TRASH Policy**: Each table's `KEEP_IN_TRASH` setting applied independently + - **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations) + - **Individual TRASH Policy**: Each table's `IS_KEEP_IN_TRASH` setting applied independently - **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables - `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy - `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic @@ -419,7 +464,7 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC **Archival Execution**: ```sql --- Single table archival (TRASH retention controlled by KEEP_IN_TRASH config) +-- Single table archival (TRASH retention controlled by IS_KEEP_IN_TRASH config) BEGIN CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA( pSourceFileConfigKey => vSourceFileConfigKey @@ -451,11 +496,11 @@ END; **Strategy-Based Filtering**: - Package retrieves ARCHIVAL_STRATEGY from A_SOURCE_FILE_CONFIG - GET_ARCHIVAL_WHERE_CLAUSE generates appropriate WHERE clause -- Only tables with ARCHIVE_ENABLED = 'Y' are processed +- Only tables with IS_ARCHIVE_ENABLED = 'Y' are processed - Data matching criteria moved from ODS to ARCHIVE bucket - CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/) - Parquet format with Hive-style partitioning applied to ARCHIVE bucket -- TRASH retention controlled by KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG +- TRASH retention controlled by IS_KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG ### Automatic Rollback Mechanism @@ -465,7 +510,7 @@ FILE_ARCHIVER implements **automatic rollback** to ensure data integrity if arch 1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket 2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED' 3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket) -4. **Optional Cleanup**: If KEEP_IN_TRASH='N', files deleted from TRASH +4. **Optional Cleanup**: If IS_KEEP_IN_TRASH='N', files deleted from TRASH **Automatic Rollback Trigger**: If **any error occurs** during step 3 (Move to TRASH), the system: @@ -655,7 +700,7 @@ SELECT TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, - DAYS_FOR_ARCHIVE_THRESHOLD + ARCHIVE_THRESHOLD_DAYS FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' ORDER BY A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID; @@ -679,8 +724,8 @@ ORDER BY ARCHIVAL_STRATEGY; UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', MINIMUM_AGE_MONTHS = 6, - ARCHIVE_ENABLED = 'Y', -- Enable archival - KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety + IS_ARCHIVE_ENABLED = 'Y', -- Enable archival + IS_KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'CSDB' AND TABLE_ID = 'CSDB_DEBT'; @@ -688,13 +733,13 @@ COMMIT; -- Disable archival temporarily for troubleshooting UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET ARCHIVE_ENABLED = 'N' -- Batch operations will skip this table +SET IS_ARCHIVE_ENABLED = 'N' -- Batch operations will skip this table WHERE TABLE_ID = 'CSDB_DEBT'; COMMIT; -- Configure TRASH cleanup for test environment UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival +SET IS_KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival WHERE SOURCE_FILE_TYPE = 'INPUT' AND SOURCE_FILE_ID = 'TEST_SOURCE'; COMMIT; @@ -705,21 +750,21 @@ SELECT TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, - ARCHIVE_ENABLED, - KEEP_IN_TRASH + IS_ARCHIVE_ENABLED, + IS_KEEP_IN_TRASH FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' ORDER BY SOURCE_FILE_ID, TABLE_ID; -- Summary by archival status SELECT - ARCHIVE_ENABLED, - KEEP_IN_TRASH, + IS_ARCHIVE_ENABLED, + IS_KEEP_IN_TRASH, COUNT(*) AS TABLE_COUNT FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = 'INPUT' -GROUP BY ARCHIVE_ENABLED, KEEP_IN_TRASH -ORDER BY ARCHIVE_ENABLED DESC, KEEP_IN_TRASH DESC; +GROUP BY IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH +ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEEP_IN_TRASH DESC; ``` ## Release 01 Configuration @@ -829,11 +874,11 @@ JOIN CT_ODS.A_LOAD_HISTORY LH ON SFR.A_WORKFLOW_HISTORY_KEY = LH.A_WORKFLOW_HIST JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' AND SFR.PROCESSING_STATUS = 'INGESTED' - AND SFC.ARCHIVE_ENABLED = 'Y' + AND SFC.IS_ARCHIVE_ENABLED = 'Y' ORDER BY LH.LOAD_START; -- Note: MINIMUM_AGE_MONTHS archives immediately (threshold-independent) --- If files not archived, check ARCHIVE_ENABLED='Y' and run ARCHIVE_TABLE_DATA +-- If files not archived, check IS_ARCHIVE_ENABLED='Y' and run ARCHIVE_TABLE_DATA ``` **Scenario B**: **THRESHOLD_BASED** or **HYBRID** strategy not archiving @@ -843,9 +888,9 @@ SELECT SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, - SFC.FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THRESHOLD, - SFC.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THRESHOLD, - SFC.BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THRESHOLD, + SFC.ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THRESHOLD, + SFC.ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THRESHOLD, + SFC.ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THRESHOLD, COUNT(SFR.A_SOURCE_FILE_RECEIVED_KEY) AS CURRENT_FILES, SUM(SFR.TOTAL_RECORDS) AS CURRENT_ROWS, SUM(SFR.FILE_SIZE_BYTES) AS CURRENT_BYTES @@ -854,13 +899,13 @@ LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY AND SFR.PROCESSING_STATUS = 'INGESTED' WHERE SFC.ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'HYBRID') - AND SFC.ARCHIVE_ENABLED = 'Y' + AND SFC.IS_ARCHIVE_ENABLED = 'Y' AND SFC.A_SOURCE_FILE_CONFIG_KEY = :yourConfigKey GROUP BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, - SFC.FILES_COUNT_OVER_ARCHIVE_THRESHOLD, - SFC.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD, - SFC.BYTES_SUM_OVER_ARCHIVE_THRESHOLD; + SFC.ARCHIVE_THRESHOLD_FILES_COUNT, + SFC.ARCHIVE_THRESHOLD_ROWS_COUNT, + SFC.ARCHIVE_THRESHOLD_BYTES_SUM; -- Expected: At least ONE threshold (FILE/ROW/BYTE) must be exceeded -- If no threshold exceeded, archival will NOT trigger (threshold-dependent behavior) @@ -903,7 +948,7 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI **Symptoms**: Files not deleted from TRASH after archival -**Cause**: Configuration has `KEEP_IN_TRASH='Y'` (retain files in TRASH) +**Cause**: Configuration has `IS_KEEP_IN_TRASH='Y'` (retain files in TRASH) **Verification**: ```sql @@ -911,8 +956,8 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI SELECT SOURCE_FILE_ID, TABLE_ID, - KEEP_IN_TRASH, - CASE KEEP_IN_TRASH + IS_KEEP_IN_TRASH, + CASE IS_KEEP_IN_TRASH WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)' WHEN 'N' THEN 'Files DELETED immediately after archival' END AS TRASH_BEHAVIOR @@ -924,7 +969,7 @@ WHERE TABLE_ID = 'YOUR_TABLE'; ```sql -- Option A: Change configuration to auto-delete (permanent change) UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG -SET KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival +SET IS_KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival WHERE TABLE_ID = 'YOUR_TABLE'; COMMIT; @@ -997,7 +1042,7 @@ SELECT TABLE_ID, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, - DAYS_FOR_ARCHIVE_THRESHOLD + ARCHIVE_THRESHOLD_DAYS FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE TABLE_ID = 'YOUR_TABLE'; @@ -1087,7 +1132,7 @@ SELECT SFR.FILE_SIZE_BYTES, SFR.UPDATED_AT AS ARCHIVED_AT, TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH, - SFC.KEEP_IN_TRASH AS TRASH_POLICY + SFC.IS_KEEP_IN_TRASH AS TRASH_POLICY FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' @@ -1102,8 +1147,8 @@ SELECT SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, - SFC.ARCHIVE_ENABLED, - SFC.KEEP_IN_TRASH, + SFC.IS_ARCHIVE_ENABLED, + SFC.IS_KEEP_IN_TRASH, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED, @@ -1113,7 +1158,7 @@ LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = S WHERE SFC.SOURCE_FILE_TYPE = 'INPUT' GROUP BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, - SFC.ARCHIVE_ENABLED, SFC.KEEP_IN_TRASH + SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEEP_IN_TRASH ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID; ``` @@ -1134,7 +1179,7 @@ FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY JOIN CT_ODS.A_LOAD_HISTORY LH ON SFR.A_WORKFLOW_HISTORY_KEY = LH.A_WORKFLOW_HISTORY_KEY WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' - AND SFC.ARCHIVE_ENABLED = 'Y' + AND SFC.IS_ARCHIVE_ENABLED = 'Y' AND SFR.PROCESSING_STATUS = 'INGESTED' AND LH.LOAD_START < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -SFC.MINIMUM_AGE_MONTHS) GROUP BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.MINIMUM_AGE_MONTHS @@ -1173,20 +1218,30 @@ SELECT ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB, MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH, MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH, - SFC.KEEP_IN_TRASH AS POLICY + SFC.IS_KEEP_IN_TRASH AS POLICY FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -GROUP BY SFC.SOURCE_FILE_ID, SFC.KEEP_IN_TRASH +GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEEP_IN_TRASH ORDER BY SIZE_GB DESC; ``` ## Version History -### v3.3.0 (Current - 2026-02-11) +### v3.4.0 (Current - 2026-03-17) +- **MARS-1409**: Added `IS_WORKFLOW_SUCCESS_REQUIRED` flag to A_SOURCE_FILE_CONFIG + - `'Y'` (default) = archivization requires `WORKFLOW_SUCCESSFUL='Y'` in A_WORKFLOW_HISTORY (standard Airflow+DBT flow) + - `'N'` = archive regardless of workflow status (bypass for manual/non-DBT sources) +- `IS_WORKFLOW_SUCCESS_REQUIRED` stored in A_TABLE_STAT and A_TABLE_STAT_HIST at statistics gather time +- GATHER_TABLE_STAT: conditional `WORKFLOW_SUCCESSFUL='Y'` filter controlled by the flag +- ARCHIVE_TABLE_DATA: conditional `WORKFLOW_SUCCESSFUL='Y'` filter controlled by the flag +- Added `pIsWorkflowSuccessRequired` parameter to FILE_MANAGER.ADD_SOURCE_FILE_CONFIG +- FILE_MANAGER updated to v3.6.2+ + +### v3.3.0 (2026-02-11) - **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA -- Added `ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control -- Added `KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter) +- Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control +- Added `IS_KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter) - Added batch procedures with 3-level granularity (config/source/all): - ARCHIVE_ALL - Batch archival procedure - GATHER_TABLE_STAT_ALL - Batch statistics procedure @@ -1226,7 +1281,7 @@ ORDER BY SIZE_GB DESC; ### v2.0.0 (Legacy) - Initial FILE_ARCHIVER package - THRESHOLD_BASED archival only -- Fixed DAYS_FOR_ARCHIVE_THRESHOLD configuration +- Fixed ARCHIVE_THRESHOLD_DAYS configuration ## Related Documentation @@ -1337,7 +1392,7 @@ ORDER BY SIZE_GB DESC; ### TRASH Folder Retention Best Practices -1. **Default Behavior (KEEP_IN_TRASH = 'Y' - Recommended)**: +1. **Default Behavior (IS_KEEP_IN_TRASH = 'Y' - Recommended)**: - Keeps CSV files in TRASH folder after archival - Provides safety net for rollback if archival issues occur - Supports compliance and audit requirements @@ -1346,11 +1401,11 @@ ORDER BY SIZE_GB DESC; - Configuration: ```sql UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG - SET KEEP_IN_TRASH = 'Y' + SET IS_KEEP_IN_TRASH = 'Y' WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; ``` -2. **TRASH Cleanup (KEEP_IN_TRASH = 'N')**: +2. **TRASH Cleanup (IS_KEEP_IN_TRASH = 'N')**: - Deletes CSV files from TRASH folder after successful archival - Reduces storage costs in DATA bucket - Status: ARCHIVED_AND_PURGED @@ -1358,7 +1413,7 @@ ORDER BY SIZE_GB DESC; - Configuration: ```sql UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG - SET KEEP_IN_TRASH = 'N' + SET IS_KEEP_IN_TRASH = 'N' WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; ```