admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: admin:0725119b45f1837f77399b40f4ad491b9a6d6ac1
Branches Tags
admin:main admin:develop
...
compare: admin:a7db9b67bc539cff1be6bfb18153e8c058bd7352
Branches Tags
admin:main admin:develop

3 Commits
0725119b45 ... a7db9b67bc

Author SHA1 Message Date
Grzegorz Michalski
a7db9b67bc Merge branch 'main' of https://git.itbi.mywire.org/admin/mars 2026-03-18 18:19:19 +01:00
Grzegorz Michalski
ce9b6eeff6 feat(FILE_MANAGER): Update package version to 3.6.3 and enhance ADD_SOURCE_FILE_CONFIG with new parameters for archival control 
- Bump package version to 3.6.3 and update build date.
- Add new parameters: pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG.
- Include pIsWorkflowSuccessRequired parameter to control workflow success requirement for archival.
- Update version history to reflect changes.

feat(A_SOURCE_FILE_CONFIG): Modify table structure to include new archival control flags

- Add IS_WORKFLOW_SUCCESS_REQUIRED column to A_SOURCE_FILE_CONFIG for workflow bypass functionality.
- Update constraints and comments for new columns.
- Ensure backward compatibility with default values.

fix(A_TABLE_STAT, A_TABLE_STAT_HIST): Extend table structures to accommodate new workflow success tracking

- Add IS_WORKFLOW_SUCCESS_REQUIRED column to both A_TABLE_STAT and A_TABLE_STAT_HIST.
- Update comments to clarify the purpose of new columns.

docs(FILE_ARCHIVER_Guide): Revise documentation to reflect new archival features and configurations

- Document new IS_WORKFLOW_SUCCESS_REQUIRED flag and its implications for archival processes.
- Update examples and configurations to align with recent changes in the database schema.
- Ensure clarity on archival strategies and their configurations.
 2026-03-18 18:19:04 +01:00
Grzegorz Michalski
896e67bcb9 feat: Refactor A_SOURCE_FILE_CONFIG table structure and update comments for clarity 2026-03-17 10:58:01 +01:00
25 changed files with 1053 additions and 402 deletions
Expand all files Collapse all files

348
MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_add_archival_strategy_column.sql Normal file
View File

@@ -0,0 +1,348 @@
-- ============================================================================
-- MARS-1409 Step 10: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns
-- to A_TABLE_STAT and A_TABLE_STAT_HIST tables;
-- add IS_WORKFLOW_SUCCESS_REQUIRED to A_SOURCE_FILE_CONFIG
-- ============================================================================
-- Purpose: Make statistics records fully self-documenting:
-- ARCHIVAL_STRATEGY - strategy in effect when stats were gathered
-- (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID)
-- ARCH_MINIMUM_AGE_MONTHS - minimum age threshold in months; complements
-- ARCH_THRESHOLD_DAYS:
-- THRESHOLD_BASED -> ARCH_THRESHOLD_DAYS set, ARCH_MINIMUM_AGE_MONTHS NULL
-- MINIMUM_AGE_MONTHS -> ARCH_THRESHOLD_DAYS NULL, ARCH_MINIMUM_AGE_MONTHS set
-- HYBRID -> both columns set
-- Prerequisites: A_TABLE_STAT and A_TABLE_STAT_HIST tables exist
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- ----------------------------------------------------------------------------
-- ARCHIVAL_STRATEGY
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
ARCHIVAL_STRATEGY VARCHAR2(30)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS
''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
ARCHIVAL_STRATEGY VARCHAR2(30)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS
''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- ARCH_MINIMUM_AGE_MONTHS
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS
''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS
''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- IS_WORKFLOW_SUCCESS_REQUIRED (A_TABLE_STAT, A_TABLE_STAT_HIST)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS
''Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows regardless. Added MARS-1409''';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS
''Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows regardless. Added MARS-1409''';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- IS_WORKFLOW_SUCCESS_REQUIRED (A_SOURCE_FILE_CONFIG)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding IS_WORKFLOW_SUCCESS_REQUIRED column to A_SOURCE_FILE_CONFIG...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_SOURCE_FILE_CONFIG'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED already exists in A_SOURCE_FILE_CONFIG.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD (
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT ''Y'' NOT NULL
)';
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN (''Y'', ''N''))';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
''Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409''';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED added to A_SOURCE_FILE_CONFIG successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding IS_WORKFLOW_SUCCESS_REQUIRED to A_SOURCE_FILE_CONFIG: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE
-- (A_TABLE_STAT)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE columns to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_SIZE NUMBER(38,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_FILE_COUNT IS
''Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_ROW_COUNT IS
''Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_SIZE IS
''Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding WORKFLOW_SUCCESS_* columns to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE
-- (A_TABLE_STAT_HIST)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE columns to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_SIZE NUMBER(38,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_FILE_COUNT IS
''Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_ROW_COUNT IS
''Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_SIZE IS
''Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409''';
DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding WORKFLOW_SUCCESS_* columns to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Step 10 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns added to both statistics tables.
PROMPT IS_WORKFLOW_SUCCESS_REQUIRED added to A_SOURCE_FILE_CONFIG, A_TABLE_STAT and A_TABLE_STAT_HIST.
PROMPT WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE added to A_TABLE_STAT and A_TABLE_STAT_HIST.
PROMPT

0
MARS_Packages/REL02_POST/MARS-1409/02_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql → MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/03_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql → MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/04_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql → MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/05_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql → MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/06_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql → MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql → MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
View File

0
MARS_Packages/REL02_POST/MARS-1409/08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql → MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql
View File

122
MARS_Packages/REL02_POST/MARS-1409/100_MARS_1409_rollback_archival_strategy_column.sql
View File

@@ -1,122 +0,0 @@
-- ============================================================================
-- MARS-1409 Rollback Step 100: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS
-- columns from A_TABLE_STAT and A_TABLE_STAT_HIST
-- ============================================================================
-- Purpose: Rollback of step 10 - remove both stat self-documentation columns
-- Prerequisites: Step 10 was applied
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- ----------------------------------------------------------------------------
-- ARCH_MINIMUM_AGE_MONTHS (drop first - added second)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCH_MINIMUM_AGE_MONTHS';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCH_MINIMUM_AGE_MONTHS';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- ARCHIVAL_STRATEGY
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCHIVAL_STRATEGY';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCHIVAL_STRATEGY';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Rollback step 100 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns removed from both statistics tables.
PROMPT

153
MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_add_archival_strategy_column.sql
View File

@@ -1,153 +0,0 @@
-- ============================================================================
-- MARS-1409 Step 10: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns
-- to A_TABLE_STAT and A_TABLE_STAT_HIST tables
-- ============================================================================
-- Purpose: Make statistics records fully self-documenting:
-- ARCHIVAL_STRATEGY - strategy in effect when stats were gathered
-- (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID)
-- ARCH_MINIMUM_AGE_MONTHS - minimum age threshold in months; complements
-- ARCH_THRESHOLD_DAYS:
-- THRESHOLD_BASED -> ARCH_THRESHOLD_DAYS set, ARCH_MINIMUM_AGE_MONTHS NULL
-- MINIMUM_AGE_MONTHS -> ARCH_THRESHOLD_DAYS NULL, ARCH_MINIMUM_AGE_MONTHS set
-- HYBRID -> both columns set
-- Prerequisites: A_TABLE_STAT and A_TABLE_STAT_HIST tables exist
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- ----------------------------------------------------------------------------
-- ARCHIVAL_STRATEGY
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
ARCHIVAL_STRATEGY VARCHAR2(30)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS
''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Adding ARCHIVAL_STRATEGY column to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
ARCHIVAL_STRATEGY VARCHAR2(30)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS
''Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCHIVAL_STRATEGY to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- ARCH_MINIMUM_AGE_MONTHS
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT ADD (
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS
''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Adding ARCH_MINIMUM_AGE_MONTHS column to A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists > 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS already exists in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE '
ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST ADD (
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0)
)';
EXECUTE IMMEDIATE '
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS
''Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)''';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS added to A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR adding ARCH_MINIMUM_AGE_MONTHS to A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Step 10 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns added to both statistics tables.
PROMPT

0
MARS_Packages/REL02_POST/MARS-1409/09_MARS_1409_verify_installation.sql → MARS_Packages/REL02_POST/MARS-1409/10_MARS_1409_verify_installation.sql
View File

264
MARS_Packages/REL02_POST/MARS-1409/98_MARS_1409_rollback_archival_strategy_column.sql Normal file
View File

@@ -0,0 +1,264 @@
-- ============================================================================
-- MARS-1409 Rollback Step 100: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS
-- columns from A_TABLE_STAT and A_TABLE_STAT_HIST;
-- drop IS_WORKFLOW_SUCCESS_REQUIRED from A_SOURCE_FILE_CONFIG
-- ============================================================================
-- Purpose: Rollback of step 10 - remove both stat self-documentation columns
-- Prerequisites: Step 10 was applied
-- ============================================================================
SET SERVEROUTPUT ON SIZE UNLIMITED
WHENEVER SQLERROR EXIT SQL.SQLCODE
-- ----------------------------------------------------------------------------
-- ARCH_MINIMUM_AGE_MONTHS (drop first - added second)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCH_MINIMUM_AGE_MONTHS';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping ARCH_MINIMUM_AGE_MONTHS column from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCH_MINIMUM_AGE_MONTHS';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCH_MINIMUM_AGE_MONTHS does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCH_MINIMUM_AGE_MONTHS';
DBMS_OUTPUT.PUT_LINE('Column ARCH_MINIMUM_AGE_MONTHS dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCH_MINIMUM_AGE_MONTHS from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- ARCHIVAL_STRATEGY
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN ARCHIVAL_STRATEGY';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping ARCHIVAL_STRATEGY column from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'ARCHIVAL_STRATEGY';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column ARCHIVAL_STRATEGY does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN ARCHIVAL_STRATEGY';
DBMS_OUTPUT.PUT_LINE('Column ARCHIVAL_STRATEGY dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping ARCHIVAL_STRATEGY from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- IS_WORKFLOW_SUCCESS_REQUIRED (A_TABLE_STAT, A_TABLE_STAT_HIST)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- IS_WORKFLOW_SUCCESS_REQUIRED (A_SOURCE_FILE_CONFIG)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping IS_WORKFLOW_SUCCESS_REQUIRED column from A_SOURCE_FILE_CONFIG...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_SOURCE_FILE_CONFIG'
AND COLUMN_NAME = 'IS_WORKFLOW_SUCCESS_REQUIRED';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_WORKFLOW_SUCCESS_REQUIRED does not exist in A_SOURCE_FILE_CONFIG.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED';
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP COLUMN IS_WORKFLOW_SUCCESS_REQUIRED';
DBMS_OUTPUT.PUT_LINE('Column IS_WORKFLOW_SUCCESS_REQUIRED dropped from A_SOURCE_FILE_CONFIG successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping IS_WORKFLOW_SUCCESS_REQUIRED from A_SOURCE_FILE_CONFIG: ' || SQLERRM);
RAISE;
END;
/
-- ----------------------------------------------------------------------------
-- WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE
-- (drop from A_TABLE_STAT and A_TABLE_STAT_HIST)
-- ----------------------------------------------------------------------------
PROMPT
PROMPT Dropping WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE from A_TABLE_STAT...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT'
AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT does not exist in A_TABLE_STAT.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT DROP (WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE)';
DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE dropped from A_TABLE_STAT successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping WORKFLOW_SUCCESS_* columns from A_TABLE_STAT: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Dropping WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE from A_TABLE_STAT_HIST...
DECLARE
vColumnExists NUMBER := 0;
BEGIN
SELECT COUNT(*) INTO vColumnExists
FROM ALL_TAB_COLUMNS
WHERE OWNER = 'CT_MRDS'
AND TABLE_NAME = 'A_TABLE_STAT_HIST'
AND COLUMN_NAME = 'WORKFLOW_SUCCESS_FILE_COUNT';
IF vColumnExists = 0 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column WORKFLOW_SUCCESS_FILE_COUNT does not exist in A_TABLE_STAT_HIST.');
ELSE
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_TABLE_STAT_HIST DROP (WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE)';
DBMS_OUTPUT.PUT_LINE('Columns WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE dropped from A_TABLE_STAT_HIST successfully.');
END IF;
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('ERROR dropping WORKFLOW_SUCCESS_* columns from A_TABLE_STAT_HIST: ' || SQLERRM);
RAISE;
END;
/
PROMPT
PROMPT Rollback step 100 completed: ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns removed from both statistics tables.
PROMPT IS_WORKFLOW_SUCCESS_REQUIRED removed from A_SOURCE_FILE_CONFIG, A_TABLE_STAT and A_TABLE_STAT_HIST.
PROMPT WORKFLOW_SUCCESS_FILE_COUNT, WORKFLOW_SUCCESS_ROW_COUNT, WORKFLOW_SUCCESS_SIZE removed from A_TABLE_STAT and A_TABLE_STAT_HIST.
PROMPT

24
MARS_Packages/REL02_POST/MARS-1409/install_mars1409.sql
View File

@@ -31,8 +31,8 @@ PROMPT =========================================================================
PROMPT MARS-1409 Installation Starting PROMPT MARS-1409 Installation Starting
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT Package: CT_MRDS.FILE_MANAGER v3.X.X PROMPT Package: CT_MRDS.FILE_MANAGER v3.X.X
PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED, ARCHIVAL_STRATEGY to A_TABLE_STAT/HIST PROMPT Change: Add A_WORKFLOW_HISTORY_KEY to A_SOURCE_FILE_RECEIVED; add ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_* columns to A_TABLE_STAT/HIST
PROMPT Purpose: Direct tracking of workflow history keys in file registration; self-documenting statistics records PROMPT Purpose: Direct tracking of workflow history keys in file registration; self-documenting statistics records; separate total vs workflow-success statistics
PROMPT Steps: 12 (DDL x2, ENV_MANAGER Update, FILE_MANAGER Update, FILE_ARCHIVER Update, Trigger Update, Verification, Tracking, Version Verification) PROMPT Steps: 12 (DDL x2, ENV_MANAGER Update, FILE_MANAGER Update, FILE_ARCHIVER Update, Trigger Update, Verification, Tracking, Version Verification)
PROMPT Timestamp: PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
@@ -56,57 +56,57 @@ PROMPT =========================================================================
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 2: Add ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns to A_TABLE_STAT and A_TABLE_STAT_HIST PROMPT STEP 2: Add ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_FILE_COUNT/ROW_COUNT/SIZE columns to A_TABLE_STAT and A_TABLE_STAT_HIST
PROMPT ============================================================================ PROMPT ============================================================================
@@10_MARS_1409_add_archival_strategy_column.sql @@02_MARS_1409_add_archival_strategy_column.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 3: Update ENV_MANAGER package specification PROMPT STEP 3: Update ENV_MANAGER package specification
PROMPT ============================================================================ PROMPT ============================================================================
@@02_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql @@03_MARS_1409_install_CT_MRDS_ENV_MANAGER_SPEC.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 4: Update ENV_MANAGER package body PROMPT STEP 4: Update ENV_MANAGER package body
PROMPT ============================================================================ PROMPT ============================================================================
@@03_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql @@04_MARS_1409_install_CT_MRDS_ENV_MANAGER_BODY.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 5: Update FILE_MANAGER package specification PROMPT STEP 5: Update FILE_MANAGER package specification
PROMPT ============================================================================ PROMPT ============================================================================
@@04_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql @@05_MARS_1409_install_CT_MRDS_FILE_MANAGER_SPEC.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 6: Update FILE_MANAGER package body PROMPT STEP 6: Update FILE_MANAGER package body
PROMPT ============================================================================ PROMPT ============================================================================
@@05_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql @@06_MARS_1409_install_CT_MRDS_FILE_MANAGER_BODY.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 7: Update FILE_ARCHIVER package specification PROMPT STEP 7: Update FILE_ARCHIVER package specification
PROMPT ============================================================================ PROMPT ============================================================================
@@06_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql @@07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 8: Update FILE_ARCHIVER package body PROMPT STEP 8: Update FILE_ARCHIVER package body
PROMPT ============================================================================ PROMPT ============================================================================
@@07_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql @@08_MARS_1409_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 9: Update A_WORKFLOW_HISTORY trigger PROMPT STEP 9: Update A_WORKFLOW_HISTORY trigger
PROMPT ============================================================================ PROMPT ============================================================================
@@08_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql @@09_MARS_1409_install_CT_MRDS_A_WORKFLOW_HISTORY.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 10: Verify installation PROMPT STEP 10: Verify installation
PROMPT ============================================================================ PROMPT ============================================================================
@@09_MARS_1409_verify_installation.sql @@10_MARS_1409_verify_installation.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================

116
MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql Normal file
View File

@@ -0,0 +1,116 @@
-- ====================================================================
-- A_SOURCE_FILE_CONFIG Table
-- ====================================================================
-- Purpose: Store source file configuration and processing rules
-- MARS-1049: Added ENCODING column for CSV character set support
-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation
-- MARS-1409: Added IS_WORKFLOW_SUCCESS_REQUIRED flag for workflow bypass
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE,
SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG'
SOURCE_FILE_ID VARCHAR2(200),
SOURCE_FILE_DESC VARCHAR2(2000),
SOURCE_FILE_NAME_PATTERN VARCHAR2(200),
TABLE_ID VARCHAR2(200),
TEMPLATE_TABLE_NAME VARCHAR2(200),
CONTAINER_FILE_KEY NUMBER(38,0),
ARCHIVE_THRESHOLD_DAYS NUMBER(4,0),
ARCHIVE_THRESHOLD_FILES_COUNT NUMBER(38,0),
ARCHIVE_THRESHOLD_BYTES_SUM NUMBER(38,0),
ODS_SCHEMA_NAME VARCHAR2(100),
ARCHIVE_THRESHOLD_ROWS_COUNT NUMBER(38,0),
HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3),
ARCHIVAL_STRATEGY VARCHAR2(50),
MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID)
) TABLESPACE "DATA";
-- Primary key index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY")
TABLESPACE "DATA";
-- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID")
TABLESPACE "DATA";
-- Column comments
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
'Primary key - unique identifier for source file configuration record';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
'Human-readable description of the source file and its purpose';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
'Identifier for the target table where data will be loaded (without schema prefix)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
'Schema name where ODS external tables are created (typically ODS)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
'Number of hours before table statistics expire and need to be recalculated';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
'Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;

8
MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT.sql
View File

@@ -18,11 +18,19 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT (
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30), ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1),
WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_SIZE NUMBER(38,0),
CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY) CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY)
) TABLESPACE "DATA"; ) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG when statistics were gathered. Y=OVER_ARCH_THRESOLD counts include only WORKFLOW_SUCCESSFUL=Y rows. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_FILE_COUNT IS 'Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_ROW_COUNT IS 'Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.WORKFLOW_SUCCESS_SIZE IS 'Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';
-- Unique constraint index (from production export) -- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1" CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1"

12
MARS_Packages/REL02_POST/MARS-1409/new_version/A_TABLE_STAT_HIST.sql
View File

@@ -17,8 +17,16 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST (
ARCH_THRESHOLD_DAYS NUMBER(4,0), ARCH_THRESHOLD_DAYS NUMBER(4,0),
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30), ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1),
WORKFLOW_SUCCESS_FILE_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_ROW_COUNT NUMBER(38,0),
WORKFLOW_SUCCESS_SIZE NUMBER(38,0)
) TABLESPACE "DATA"; ) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG when statistics were gathered. Y=OVER_ARCH_THRESOLD counts include only WORKFLOW_SUCCESSFUL=Y rows. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_FILE_COUNT IS 'Count of files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_ROW_COUNT IS 'Row count for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.WORKFLOW_SUCCESS_SIZE IS 'Size in bytes for files with WORKFLOW_SUCCESSFUL=Y. Always populated regardless of IS_WORKFLOW_SUCCESS_REQUIRED flag. Added MARS-1409';

43
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb
View File

@@ -166,7 +166,7 @@ AS
join CT_MRDS.a_workflow_history h join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key on s.a_workflow_history_key = h.a_workflow_history_key
where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || '
and h.WORKFLOW_SUCCESSFUL = ''Y'' ' || CASE WHEN vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN 'and h.WORKFLOW_SUCCESSFUL = ''Y''' ELSE '' END || '
group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')' group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')'
; ;
@@ -199,7 +199,7 @@ AS
on s.a_workflow_history_key = h.a_workflow_history_key on s.a_workflow_history_key = h.a_workflow_history_key
and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||''' and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||'''
and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||''' and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||'''
and h.WORKFLOW_SUCCESSFUL = ''Y'' '|| CASE WHEN vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN 'and h.WORKFLOW_SUCCESSFUL = ''Y''' ELSE '' END ||'
' '
; ;
vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/'; vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
@@ -450,8 +450,9 @@ AS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableName VARCHAR2(200); vTableName VARCHAR2(200);
vQuery VARCHAR2(32000); vQuery VARCHAR2(32000);
vWhereClause VARCHAR2(4000); vWhereClause VARCHAR2(4000);
vOdsBucketUri VARCHAR2(1000); vOverThresholdWhereClause VARCHAR2(4000);
vOdsBucketUri VARCHAR2(1000);
BEGIN BEGIN
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'))); vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
@@ -467,13 +468,28 @@ AS
-- Get ODS bucket URI before building query -- Get ODS bucket URI before building query
vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
-- Build WHERE clause for OVER_ARCH_THRESOLD columns:
-- Combines archival strategy time-condition with optional workflow success filter.
-- IS_WORKFLOW_SUCCESS_REQUIRED='Y': only files with WORKFLOW_SUCCESSFUL='Y' are counted as eligible.
-- IS_WORKFLOW_SUCCESS_REQUIRED='N': all files passing the time-condition are counted as eligible.
IF vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED = 'Y' THEN
vOverThresholdWhereClause := vWhereClause || ' AND workflow_successful = ''Y''';
ELSE
vOverThresholdWhereClause := vWhereClause;
END IF;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOverThresholdWhereClause','DEBUG',vOverThresholdWhereClause);
-- Use strategy-based WHERE clause for statistics (MARS-828) -- Use strategy-based WHERE clause for statistics (MARS-828)
-- FILE_COUNT, ROW_COUNT, SIZE: all files regardless of workflow success (never zero due to workflow filter)
-- OVER_ARCH_THRESOLD_*: IS_WORKFLOW_SUCCESS_REQUIRED-aware count of eligible files
-- WORKFLOW_SUCCESS_*: informational count of files with WORKFLOW_SUCCESSFUL='Y'
vQuery := vQuery :=
'with tmp as ( 'with tmp as (
select select
s.* s.*
,file$name as filename ,file$name as filename
,h.workflow_start ,h.workflow_start
,h.workflow_successful
, to_char(h.workflow_start,''yyyy'') as year , to_char(h.workflow_start,''yyyy'') as year
, to_char(h.workflow_start,''mm'') as month , to_char(h.workflow_start,''mm'') as month
from '||vTableName||' s from '||vTableName||' s
@@ -482,7 +498,10 @@ AS
) )
, tmp_gr as ( , tmp_gr as (
select select
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start filename
,count(*) as row_count_per_file
,min(workflow_start) as workflow_start
,max(workflow_successful) as workflow_successful
from tmp from tmp
group by filename group by filename
) )
@@ -491,15 +510,19 @@ AS
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY ,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
,'''||vTableName||''' as TABLE_NAME ,'''||vTableName||''' as TABLE_NAME
,count(*) as FILE_COUNT ,count(*) as FILE_COUNT
,nvl(sum(case when ' || vWhereClause || ' then 1 else 0 end), 0) as OLD_FILE_COUNT ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then 1 else 0 end), 0) as OLD_FILE_COUNT
,nvl(sum(row_count_per_file), 0) as ROW_COUNT ,nvl(sum(row_count_per_file), 0) as ROW_COUNT
,nvl(sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end), 0) as OLD_ROW_COUNT ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then row_count_per_file else 0 end), 0) as OLD_ROW_COUNT
,nvl(sum(r.bytes), 0) as BYTES ,nvl(sum(r.bytes), 0) as BYTES
,nvl(sum(case when ' || vWhereClause || ' then r.bytes else 0 end), 0) as OLD_BYTES ,nvl(sum(case when ' || vOverThresholdWhereClause || ' then r.bytes else 0 end), 0) as OLD_BYTES
,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS ,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS
,systimestamp as CREATED ,systimestamp as CREATED
,'''||vSourceFileConfig.ARCHIVAL_STRATEGY||''' as ARCHIVAL_STRATEGY ,'''||vSourceFileConfig.ARCHIVAL_STRATEGY||''' as ARCHIVAL_STRATEGY
,'||COALESCE(TO_CHAR(vSourceFileConfig.MINIMUM_AGE_MONTHS), 'NULL')||' as ARCH_MINIMUM_AGE_MONTHS ,'||COALESCE(TO_CHAR(vSourceFileConfig.MINIMUM_AGE_MONTHS), 'NULL')||' as ARCH_MINIMUM_AGE_MONTHS
,'''||vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED||''' as IS_WORKFLOW_SUCCESS_REQUIRED
,nvl(sum(case when workflow_successful = ''Y'' then 1 else 0 end), 0) as WORKFLOW_SUCCESS_FILE_COUNT
,nvl(sum(case when workflow_successful = ''Y'' then row_count_per_file else 0 end), 0) as WORKFLOW_SUCCESS_ROW_COUNT
,nvl(sum(case when workflow_successful = ''Y'' then r.bytes else 0 end), 0) as WORKFLOW_SUCCESS_SIZE
from tmp_gr t from tmp_gr t
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS( join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''', credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''',
@@ -528,6 +551,10 @@ AS
vStats.CREATED := SYSTIMESTAMP; vStats.CREATED := SYSTIMESTAMP;
vStats.ARCHIVAL_STRATEGY := vSourceFileConfig.ARCHIVAL_STRATEGY; vStats.ARCHIVAL_STRATEGY := vSourceFileConfig.ARCHIVAL_STRATEGY;
vStats.ARCH_MINIMUM_AGE_MONTHS := vSourceFileConfig.MINIMUM_AGE_MONTHS; vStats.ARCH_MINIMUM_AGE_MONTHS := vSourceFileConfig.MINIMUM_AGE_MONTHS;
vStats.IS_WORKFLOW_SUCCESS_REQUIRED := vSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED;
vStats.WORKFLOW_SUCCESS_FILE_COUNT := 0;
vStats.WORKFLOW_SUCCESS_ROW_COUNT := 0;
vStats.WORKFLOW_SUCCESS_SIZE := 0;
ELSE ELSE
RAISE; RAISE;
END IF; END IF;

7
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg
View File

@@ -17,12 +17,15 @@ AS
**/ **/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.1'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-13 09:00:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-17 11:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.4.0 (2026-03-17): MARS-1409 - Added IS_WORKFLOW_SUCCESS_REQUIRED flag to A_SOURCE_FILE_CONFIG (DEFAULT Y). ' ||
'Y=standard DBT flow (WORKFLOW_SUCCESSFUL=Y required), N=bypass for manual/non-DBT sources. ' ||
'Flag value stored in A_TABLE_STAT and A_TABLE_STAT_HIST for full audit of statistics basis.' || CHR(13)||CHR(10) ||
'3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) || '3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) ||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) || '3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) || '3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||

22
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb
View File

@@ -1643,7 +1643,12 @@ AS
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
) IS ) IS
vSourceFileConfigKey PLS_INTEGER; vSourceFileConfigKey PLS_INTEGER;
vSourceKeyExists PLS_INTEGER := 0; vSourceKeyExists PLS_INTEGER := 0;
@@ -1659,10 +1664,15 @@ AS
,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||'''' ,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||''''
,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||'''' ,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||''''
,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY
,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409
,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828
,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||'''' -- MARS-828
,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828
,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828
)); ));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING) INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding); VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths);
COMMIT; COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION EXCEPTION
@@ -1772,6 +1782,12 @@ AS
||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM ||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM
||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT ||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT
||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
||cgBL||pLevel||'ENCODING = '||pSourceFileConfig.ENCODING
||cgBL||pLevel||'IS_ARCHIVE_ENABLED = '||pSourceFileConfig.IS_ARCHIVE_ENABLED
||cgBL||pLevel||'IS_KEEP_IN_TRASH = '||pSourceFileConfig.IS_KEEP_IN_TRASH
||cgBL||pLevel||'ARCHIVAL_STRATEGY = '||pSourceFileConfig.ARCHIVAL_STRATEGY
||cgBL||pLevel||'MINIMUM_AGE_MONTHS = '||pSourceFileConfig.MINIMUM_AGE_MONTHS
||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED
||cgBL||pLevel||''||'--------------------------------' ||cgBL||pLevel||''||'--------------------------------'
; ;

25
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg
View File

@@ -17,12 +17,14 @@ AS
**/ **/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH) -- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.6.1'; PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.6.3';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-13 09:00:00'; PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-17 12:30:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski'; PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first) -- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) := VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.6.3 (2026-03-17): MARS-828 - Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG; FORMAT_CONFIG now shows all A_SOURCE_FILE_CONFIG columns' || CHR(13)||CHR(10) ||
'3.6.2 (2026-03-17): MARS-1409 - Added pIsWorkflowSuccessRequired parameter to ADD_SOURCE_FILE_CONFIG; IS_WORKFLOW_SUCCESS_REQUIRED shown in GET_DET_SOURCE_FILE_CONFIG_INFO output' || CHR(13)||CHR(10) ||
'3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) || '3.6.1 (2026-03-13): MARS-1468 - Fixed CHAR/NCHAR/NVARCHAR2 column definitions in GENERATE_EXTERNAL_TABLE_PARAMS: CHAR now uses char_used/char_length semantics; NCHAR/NVARCHAR2 use char_length (data_length stores bytes in AL16UTF16)' || CHR(13)||CHR(10) ||
'3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) || '3.6.0 (2026-02-27): MARS-1409 - Added A_WORKFLOW_HISTORY_KEY tracking in A_SOURCE_FILE_RECEIVED. Each file now stores its workflow execution key extracted during VALIDATE_SOURCE_FILE_RECEIVED' || CHR(13)||CHR(10) ||
'3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) || '3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) ||
@@ -443,13 +445,23 @@ AS
* @name ADD_SOURCE_FILE_CONFIG * @name ADD_SOURCE_FILE_CONFIG
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table. * @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
* MARS-1049: Added pEncoding parameter for CSV character set specification. * MARS-1049: Added pEncoding parameter for CSV character set specification.
* MARS-1409: Added pIsWorkflowSuccessRequired parameter.
* MARS-828: Added pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths.
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2') * @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
* If NULL, no CHARACTERSET clause is added to external table definitions * If NULL, no CHARACTERSET clause is added to external table definitions
* @param pIsWorkflowSuccessRequired - 'Y' (default) = archivization requires WORKFLOW_SUCCESSFUL='Y' (standard DBT flow)
* 'N' = archive regardless of workflow status (bypass for manual/non-DBT sources)
* @param pIsArchiveEnabled - 'Y' = enable automatic archivization for this config; 'N' (default) = disabled
* @param pIsKeepInTrash - 'Y' = move files to trash before purge; 'N' (default) = purge directly
* @param pArchivalStrategy - Archival strategy: 'MINIMUM_AGE_MONTHS' or NULL
* @param pMinimumAgeMonths - Minimum age in months before file eligible for archivization (used with MINIMUM_AGE_MONTHS strategy)
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG( * @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
* pSourceKey => 'C2D', pSourceFileType => 'INPUT', * pSourceKey => 'C2D', pSourceFileType => 'INPUT',
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS', * pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS', * pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
* pEncoding => 'UTF8' * pEncoding => 'UTF8', pIsWorkflowSuccessRequired => 'Y',
* pIsArchiveEnabled => 'Y', pIsKeepInTrash => 'N',
* pArchivalStrategy => 'MINIMUM_AGE_MONTHS', pMinimumAgeMonths => 3
* ); * );
**/ **/
PROCEDURE ADD_SOURCE_FILE_CONFIG ( PROCEDURE ADD_SOURCE_FILE_CONFIG (
@@ -461,7 +473,12 @@ AS
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
); );

6
MARS_Packages/REL02_POST/MARS-1409/rollback_mars1409.sql
View File

@@ -32,7 +32,7 @@ PROMPT MARS-1409 Rollback Starting
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT Package: CT_MRDS.FILE_MANAGER PROMPT Package: CT_MRDS.FILE_MANAGER
PROMPT Change: Remove A_WORKFLOW_HISTORY_KEY column and restore previous version PROMPT Change: Remove A_WORKFLOW_HISTORY_KEY column and restore previous version
PROMPT Steps: 11 (Restore FILE_ARCHIVER, Restore FILE_MANAGER, Restore ENV_MANAGER, Restore trigger, Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns, Clear data, Drop column, Verify) PROMPT Steps: 11 (Restore FILE_ARCHIVER, Restore FILE_MANAGER, Restore ENV_MANAGER, Restore trigger, Drop ARCHIVAL_STRATEGY/ARCH_MINIMUM_AGE_MONTHS/IS_WORKFLOW_SUCCESS_REQUIRED/WORKFLOW_SUCCESS_* columns, Drop column, Verify)
PROMPT Timestamp: PROMPT Timestamp:
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL;
PROMPT ============================================================================ PROMPT ============================================================================
@@ -91,9 +91,9 @@ PROMPT =========================================================================
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================
PROMPT STEP 8: Drop ARCHIVAL_STRATEGY and ARCH_MINIMUM_AGE_MONTHS columns from A_TABLE_STAT and A_TABLE_STAT_HIST PROMPT STEP 8: Drop ARCHIVAL_STRATEGY, ARCH_MINIMUM_AGE_MONTHS, IS_WORKFLOW_SUCCESS_REQUIRED and WORKFLOW_SUCCESS_FILE_COUNT/ROW_COUNT/SIZE columns from A_TABLE_STAT and A_TABLE_STAT_HIST
PROMPT ============================================================================ PROMPT ============================================================================
@@100_MARS_1409_rollback_archival_strategy_column.sql @@98_MARS_1409_rollback_archival_strategy_column.sql
PROMPT PROMPT
PROMPT ============================================================================ PROMPT ============================================================================

86
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql
View File

@@ -4,6 +4,7 @@
-- Purpose: Store source file configuration and processing rules -- Purpose: Store source file configuration and processing rules
-- MARS-1049: Added ENCODING column for CSV character set support -- MARS-1049: Added ENCODING column for CSV character set support
-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation -- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation
-- MARS-1409: Added IS_WORKFLOW_SUCCESS_REQUIRED flag for workflow bypass
-- ==================================================================== -- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ( CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
@@ -16,20 +17,22 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
TABLE_ID VARCHAR2(200), TABLE_ID VARCHAR2(200),
TEMPLATE_TABLE_NAME VARCHAR2(200), TEMPLATE_TABLE_NAME VARCHAR2(200),
CONTAINER_FILE_KEY NUMBER(38,0), CONTAINER_FILE_KEY NUMBER(38,0),
DAYS_FOR_ARCHIVE_THRESHOLD NUMBER(4,0), ARCHIVE_THRESHOLD_DAYS NUMBER(4,0),
FILES_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), ARCHIVE_THRESHOLD_FILES_COUNT NUMBER(38,0),
BYTES_SUM_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), ARCHIVE_THRESHOLD_BYTES_SUM NUMBER(38,0),
ODS_SCHEMA_NAME VARCHAR2(100), ODS_SCHEMA_NAME VARCHAR2(100),
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0), ARCHIVE_THRESHOLD_ROWS_COUNT NUMBER(38,0),
HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3), HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3),
ARCHIVAL_STRATEGY VARCHAR2(50), ARCHIVAL_STRATEGY VARCHAR2(50),
MINIMUM_AGE_MONTHS NUMBER(3,0), MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8', ENCODING VARCHAR2(50) DEFAULT 'UTF8',
ARCHIVE_ENABLED CHAR(1) DEFAULT 'Y' NOT NULL, IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL, IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY), CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_ARCHIVE_ENABLED CHECK (ARCHIVE_ENABLED IN ('Y', 'N')), CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_KEEP_IN_TRASH CHECK (KEEP_IN_TRASH IN ('Y', 'N')), CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')), CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY), CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY), CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
@@ -47,10 +50,67 @@ ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABL
TABLESPACE "DATA"; TABLESPACE "DATA";
-- Column comments -- Column comments
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828'; 'Primary key - unique identifier for source file configuration record';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0'; COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0'; 'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
'Human-readable description of the source file and its purpose';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
'Identifier for the target table where data will be loaded (without schema prefix)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
'Schema name where ODS external tables are created (typically ODS)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
'Number of hours before table statistics expire and need to be recalculated';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
'Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql
View File

@@ -18,11 +18,13 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT (
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30), ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0), ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1),
CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY) CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY)
) TABLESPACE "DATA"; ) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409';
-- Unique constraint index (from production export) -- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1" CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1"

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql
View File

@@ -17,8 +17,10 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST (
ARCH_THRESHOLD_DAYS NUMBER(4,0), ARCH_THRESHOLD_DAYS NUMBER(4,0),
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP, CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30), ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0) ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1)
) TABLESPACE "DATA"; ) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)'; COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409';

211
confluence/FILE_ARCHIVER_Guide.md
View File

@@ -10,13 +10,14 @@ The FILE_ARCHIVER package provides flexible archival strategies that accommodate
- **Three Archival Strategies**: THRESHOLD_BASED, MINIMUM_AGE_MONTHS (with 0=current month only), HYBRID - **Three Archival Strategies**: THRESHOLD_BASED, MINIMUM_AGE_MONTHS (with 0=current month only), HYBRID
- **Flexible Configuration**: Per-table archival strategy configuration via A_SOURCE_FILE_CONFIG - **Flexible Configuration**: Per-table archival strategy configuration via A_SOURCE_FILE_CONFIG
- **Workflow Bypass**: IS_WORKFLOW_SUCCESS_REQUIRED flag allows archivization of files from non-DBT sources
- **Validation**: Automatic validation of strategy-specific configuration requirements - **Validation**: Automatic validation of strategy-specific configuration requirements
### Package Information ### Package Information
- **Schema**: CT_MRDS - **Schema**: CT_MRDS
- **Package**: FILE_ARCHIVER - **Package**: FILE_ARCHIVER
- **Current Version**: 3.3.0 - **Current Version**: 3.4.0
- **Dependencies**: ENV_MANAGER, FILE_MANAGER, A_SOURCE_FILE_CONFIG, A_SOURCE_FILE_RECEIVED, A_WORKFLOW_HISTORY - **Dependencies**: ENV_MANAGER, FILE_MANAGER, A_SOURCE_FILE_CONFIG, A_SOURCE_FILE_RECEIVED, A_WORKFLOW_HISTORY
### Critical Prerequisites ### Critical Prerequisites
@@ -67,7 +68,7 @@ END;
| Strategy | WHERE Clause Logic | Configuration Required | Primary Use Case | | Strategy | WHERE Clause Logic | Configuration Required | Primary Use Case |
|----------|-------------------|----------------------|------------------| |----------|-------------------|----------------------|------------------|
| `THRESHOLD_BASED` | Days since workflow start > threshold | DAYS_FOR_ARCHIVE_THRESHOLD | Simple time-based archival | | `THRESHOLD_BASED` | Days since workflow start > threshold | ARCHIVE_THRESHOLD_DAYS | Simple time-based archival |
| `MINIMUM_AGE_MONTHS` | Archive data older than X months (0=current month only) | MINIMUM_AGE_MONTHS (≥0) | All sources - flexible retention (0 for LM, 6 for CSDB) | | `MINIMUM_AGE_MONTHS` | Archive data older than X months (0=current month only) | MINIMUM_AGE_MONTHS (≥0) | All sources - flexible retention (0 for LM, 6 for CSDB) |
| `HYBRID` | Combines month boundary + minimum age | MINIMUM_AGE_MONTHS | Advanced retention scenarios | | `HYBRID` | Combines month boundary + minimum age | MINIMUM_AGE_MONTHS | Advanced retention scenarios |
@@ -77,14 +78,14 @@ Archives data based on number of days since workflow start.
**WHERE Clause**: **WHERE Clause**:
```sql ```sql
extract(day from (systimestamp - workflow_start)) > DAYS_FOR_ARCHIVE_THRESHOLD extract(day from (systimestamp - workflow_start)) > ARCHIVE_THRESHOLD_DAYS
``` ```
**Configuration**: **Configuration**:
```sql ```sql
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVAL_STRATEGY = 'THRESHOLD_BASED', SET ARCHIVAL_STRATEGY = 'THRESHOLD_BASED',
DAYS_FOR_ARCHIVE_THRESHOLD = 30, ARCHIVE_THRESHOLD_DAYS = 30,
MINIMUM_AGE_MONTHS = NULL MINIMUM_AGE_MONTHS = NULL
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'C2D_DATA' AND SOURCE_FILE_ID = 'C2D_DATA'
@@ -185,17 +186,17 @@ END IF;
**Behavior**: Archives data **only when** at least one of the following thresholds is exceeded: **Behavior**: Archives data **only when** at least one of the following thresholds is exceeded:
1. **FILES_COUNT_OVER_ARCHIVE_THRESHOLD** - Number of files eligible for archival 1. **ARCHIVE_THRESHOLD_FILES_COUNT** - Number of files eligible for archival
2. **ROWS_COUNT_OVER_ARCHIVE_THRESHOLD** - Number of rows eligible for archival 2. **ARCHIVE_THRESHOLD_ROWS_COUNT** - Number of rows eligible for archival
3. **BYTES_SUM_OVER_ARCHIVE_THRESHOLD** - Total size in bytes eligible for archival 3. **ARCHIVE_THRESHOLD_BYTES_SUM** - Total size in bytes eligible for archival
```sql ```sql
-- Executed for THRESHOLD_BASED and HYBRID strategies -- Executed for THRESHOLD_BASED and HYBRID strategies
IF vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD THEN IF vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT THEN
vArchivalTriggeredBy := 'FILES_COUNT'; vArchivalTriggeredBy := 'FILES_COUNT';
ELSIF vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD THEN ELSIF vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT THEN
vArchivalTriggeredBy := 'ROWS_COUNT'; vArchivalTriggeredBy := 'ROWS_COUNT';
ELSIF vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD THEN ELSIF vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM THEN
vArchivalTriggeredBy := 'BYTES_SUM'; vArchivalTriggeredBy := 'BYTES_SUM';
END IF; END IF;
``` ```
@@ -206,9 +207,9 @@ END IF;
```sql ```sql
-- Set archival thresholds for THRESHOLD_BASED strategy -- Set archival thresholds for THRESHOLD_BASED strategy
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET FILES_COUNT_OVER_ARCHIVE_THRESHOLD = 10, -- Archive when 10+ files eligible SET ARCHIVE_THRESHOLD_FILES_COUNT = 10, -- Archive when 10+ files eligible
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = 100000, -- Archive when 100k+ rows eligible ARCHIVE_THRESHOLD_ROWS_COUNT = 100000, -- Archive when 100k+ rows eligible
BYTES_SUM_OVER_ARCHIVE_THRESHOLD = 104857600 -- Archive when 100MB+ eligible ARCHIVE_THRESHOLD_BYTES_SUM = 104857600 -- Archive when 100MB+ eligible
WHERE ARCHIVAL_STRATEGY = 'THRESHOLD_BASED' WHERE ARCHIVAL_STRATEGY = 'THRESHOLD_BASED'
AND TABLE_ID = 'YOUR_TABLE'; AND TABLE_ID = 'YOUR_TABLE';
``` ```
@@ -243,15 +244,15 @@ WHERE ...;
## Archival Control Configuration ## Archival Control Configuration
### ARCHIVE_ENABLED Column ### IS_ARCHIVE_ENABLED Column
Controls whether archival is enabled for specific table configuration. Controls whether archival is enabled for specific table configuration.
**Column**: `A_SOURCE_FILE_CONFIG.ARCHIVE_ENABLED` (VARCHAR2(1), DEFAULT 'Y') **Column**: `A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED` (CHAR(1), DEFAULT 'N' NOT NULL)
**Values**: **Values**:
- `'Y'` (default) - Table is eligible for archival processing - `'Y'` - Table is eligible for archival processing
- `'N'` - Table is excluded from archival (batch operations skip this config) - `'N'` (default) - Table is excluded from archival (batch operations skip this config)
**Use Cases**: **Use Cases**:
- Disable archival for specific tables without removing configuration - Disable archival for specific tables without removing configuration
@@ -262,7 +263,7 @@ Controls whether archival is enabled for specific table configuration.
```sql ```sql
-- Disable archival for specific table -- Disable archival for specific table
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVE_ENABLED = 'N' SET IS_ARCHIVE_ENABLED = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'CSDB' AND SOURCE_FILE_ID = 'CSDB'
AND TABLE_ID = 'CSDB_DEBT'; AND TABLE_ID = 'CSDB_DEBT';
@@ -270,7 +271,7 @@ COMMIT;
-- Re-enable archival -- Re-enable archival
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVE_ENABLED = 'Y' SET IS_ARCHIVE_ENABLED = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'CSDB' AND SOURCE_FILE_ID = 'CSDB'
AND TABLE_ID = 'CSDB_DEBT'; AND TABLE_ID = 'CSDB_DEBT';
@@ -280,22 +281,22 @@ COMMIT;
SELECT SELECT
SOURCE_FILE_ID, SOURCE_FILE_ID,
TABLE_ID, TABLE_ID,
ARCHIVE_ENABLED, IS_ARCHIVE_ENABLED,
ARCHIVAL_STRATEGY ARCHIVAL_STRATEGY
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
ORDER BY SOURCE_FILE_ID, TABLE_ID; ORDER BY SOURCE_FILE_ID, TABLE_ID;
``` ```
### KEEP_IN_TRASH Column ### IS_KEEP_IN_TRASH Column
Controls TRASH folder retention policy for archived files. Controls TRASH folder retention policy for archived files.
**Column**: `A_SOURCE_FILE_CONFIG.KEEP_IN_TRASH` (VARCHAR2(1), DEFAULT 'Y') **Column**: `A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH` (CHAR(1), DEFAULT 'N' NOT NULL)
**Values**: **Values**:
- `'Y'` (default) - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED) - `'Y'` - CSV files kept in TRASH folder after archival (status: ARCHIVED_AND_TRASHED)
- `'N'` - CSV files deleted from TRASH folder after archival (status: ARCHIVED_AND_PURGED) - `'N'` (default) - CSV files deleted from TRASH folder after archival (status: ARCHIVED_AND_PURGED)
**Benefits of TRASH Retention (TRUE)**: **Benefits of TRASH Retention (TRUE)**:
- Safety net for rollback if archival issues discovered - Safety net for rollback if archival issues discovered
@@ -311,7 +312,7 @@ Controls TRASH folder retention policy for archived files.
```sql ```sql
-- Production: Keep files in TRASH (recommended) -- Production: Keep files in TRASH (recommended)
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'Y' SET IS_KEEP_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'LM' AND SOURCE_FILE_ID = 'LM'
AND TABLE_ID LIKE 'LM_%'; AND TABLE_ID LIKE 'LM_%';
@@ -319,20 +320,64 @@ COMMIT;
-- Test environment: Cleanup TRASH to save storage -- Test environment: Cleanup TRASH to save storage
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'N' SET IS_KEEP_IN_TRASH = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'TEST_SOURCE'; AND SOURCE_FILE_ID = 'TEST_SOURCE';
COMMIT; COMMIT;
-- Bulk configuration by source -- Bulk configuration by source
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'Y' SET IS_KEEP_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM'); AND SOURCE_FILE_ID IN ('CSDB', 'C2D', 'LM');
COMMIT; COMMIT;
``` ```
### IS_WORKFLOW_SUCCESS_REQUIRED Column
## Data Lifecycle Workflow Controls whether archivization requires `WORKFLOW_SUCCESSFUL='Y'` in A_WORKFLOW_HISTORY. Added in MARS-1409.
**Column**: `A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED` (CHAR(1), DEFAULT 'Y' NOT NULL)
**Values**:
- `'Y'` (default) - Only files with `WORKFLOW_SUCCESSFUL='Y'` are eligible for archivization (standard Airflow+DBT flow)
- `'N'` - Archivization proceeds regardless of workflow completion status (bypass for manual/non-DBT sources)
**Use Cases**:
- `'Y'`: All standard INBOX-validated sources (LM, CSDB, C2D) - ensures only fully-processed files are archived
- `'N'`: Legacy data migrated via `DATA_EXPORTER`, manual uploads, or any source without DBT workflow tracking
**GATHER_TABLE_STAT behavior**:
- `'Y'`: Statistics (file count, row count, byte sum) counted only from files with `WORKFLOW_SUCCESSFUL='Y'`
- `'N'`: Statistics counted from all INGESTED files regardless of workflow outcome
**Configuration Example**:
```sql
-- Standard source: require DBT workflow completion (default)
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_WORKFLOW_SUCCESS_REQUIRED = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'LM';
COMMIT;
-- Non-DBT source: bypass workflow check
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET IS_WORKFLOW_SUCCESS_REQUIRED = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'MANUAL_UPLOAD';
COMMIT;
-- Or set at configuration time via ADD_SOURCE_FILE_CONFIG
CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'MANUAL',
pSourceFileType => 'INPUT',
pSourceFileId => 'MANUAL_UPLOAD',
pSourceFileDesc => 'Manual data upload without DBT',
pSourceFileNamePattern => 'manual_*.csv',
pTableId => 'MY_TABLE',
pTemplateTableName => 'CT_ET_TEMPLATES.MY_TABLE',
pIsWorkflowSuccessRequired => 'N' -- bypass workflow check
);
```
### Status Tracking in A_SOURCE_FILE_RECEIVED ### Status Tracking in A_SOURCE_FILE_RECEIVED
@@ -348,7 +393,7 @@ INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
**Status Descriptions**: **Status Descriptions**:
- **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket - **INGESTED**: File successfully processed through Airflow+DBT, residing in ODS bucket
- **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket) - **ARCHIVED_AND_TRASHED**: File archived to Parquet in ARCHIVE bucket, CSV retained in TRASH folder (DATA bucket)
- **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when KEEP_IN_TRASH='N') - **ARCHIVED_AND_PURGED**: File archived to Parquet, CSV deleted from TRASH folder (when IS_KEEP_IN_TRASH='N')
**Associated Columns Updated During Archival**: **Associated Columns Updated During Archival**:
```sql ```sql
@@ -390,9 +435,9 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
2.1 TRASH Subfolder (DATA Bucket - File Retention) 2.1 TRASH Subfolder (DATA Bucket - File Retention)
├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME) ├─ Located in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
├─ Stores CSV files after archival to Parquet ├─ Stores CSV files after archival to Parquet
├─ Status: ARCHIVED_AND_TRASHED (default, controlled by KEEP_IN_TRASH config) ├─ Status: ARCHIVED_AND_TRASHED (default, controlled by IS_KEEP_IN_TRASH config)
├─ Enables rollback if archival issues occur ├─ Enables rollback if archival issues occur
└─ Optional cleanup: ARCHIVED_AND_PURGED (when KEEP_IN_TRASH = 'N') └─ Optional cleanup: ARCHIVED_AND_PURGED (when IS_KEEP_IN_TRASH = 'N')
3. ARCHIVE Bucket (Long-term Storage) 3. ARCHIVE Bucket (Long-term Storage)
├─ Historical data in Parquet format ├─ Historical data in Parquet format
@@ -402,14 +447,14 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
**Key Procedures**: **Key Procedures**:
- `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause - `ARCHIVE_TABLE_DATA(pSourceFileConfigKey)` - Main archival procedure using strategy-specific WHERE clause
- TRASH folder retention controlled by `KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG - TRASH folder retention controlled by `IS_KEEP_IN_TRASH` column in A_SOURCE_FILE_CONFIG
- `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling - `ARCHIVE_ALL(pSourceFileConfigKey, pSourceKey, pArchiveAll)` - Batch archival with 3-level granularity and error handling
- **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey` - **Level 3 (Highest Priority)**: Single configuration via `pSourceFileConfigKey`
- **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey` - **Level 2 (Medium Priority)**: All configurations for source via `pSourceKey`
- **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll` - **Level 1 (Lowest Priority)**: All configurations system-wide via `pArchiveAll`
- **Error Handling**: Continues processing other tables on individual failures - **Error Handling**: Continues processing other tables on individual failures
- **Filtering**: Respects `ARCHIVE_ENABLED='Y'` (skips disabled configurations) - **Filtering**: Respects `IS_ARCHIVE_ENABLED='Y'` (skips disabled configurations)
- **Individual TRASH Policy**: Each table's `KEEP_IN_TRASH` setting applied independently - **Individual TRASH Policy**: Each table's `IS_KEEP_IN_TRASH` setting applied independently
- **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables - **Summary Reporting**: Returns counts of Archived/Skipped/Failed tables
- `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy - `GET_ARCHIVAL_WHERE_CLAUSE` - Returns WHERE clause based on configured strategy
- `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic - `GATHER_TABLE_STAT` - Calculates archival statistics using strategy logic
@@ -419,7 +464,7 @@ https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/namespace/b/archive/o/ARC
**Archival Execution**: **Archival Execution**:
```sql ```sql
-- Single table archival (TRASH retention controlled by KEEP_IN_TRASH config) -- Single table archival (TRASH retention controlled by IS_KEEP_IN_TRASH config)
BEGIN BEGIN
CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA( CT_MRDS.FILE_ARCHIVER.ARCHIVE_TABLE_DATA(
pSourceFileConfigKey => vSourceFileConfigKey pSourceFileConfigKey => vSourceFileConfigKey
@@ -451,11 +496,11 @@ END;
**Strategy-Based Filtering**: **Strategy-Based Filtering**:
- Package retrieves ARCHIVAL_STRATEGY from A_SOURCE_FILE_CONFIG - Package retrieves ARCHIVAL_STRATEGY from A_SOURCE_FILE_CONFIG
- GET_ARCHIVAL_WHERE_CLAUSE generates appropriate WHERE clause - GET_ARCHIVAL_WHERE_CLAUSE generates appropriate WHERE clause
- Only tables with ARCHIVE_ENABLED = 'Y' are processed - Only tables with IS_ARCHIVE_ENABLED = 'Y' are processed
- Data matching criteria moved from ODS to ARCHIVE bucket - Data matching criteria moved from ODS to ARCHIVE bucket
- CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/) - CSV files moved to TRASH subfolder in DATA bucket (ODS/ → TRASH/)
- Parquet format with Hive-style partitioning applied to ARCHIVE bucket - Parquet format with Hive-style partitioning applied to ARCHIVE bucket
- TRASH retention controlled by KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG - TRASH retention controlled by IS_KEEP_IN_TRASH column in A_SOURCE_FILE_CONFIG
### Automatic Rollback Mechanism ### Automatic Rollback Mechanism
@@ -465,7 +510,7 @@ FILE_ARCHIVER implements **automatic rollback** to ensure data integrity if arch
1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket 1. **Export to ARCHIVE**: Data exported to Parquet format in ARCHIVE bucket
2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED' 2. **Status Update**: A_SOURCE_FILE_RECEIVED records updated to 'ARCHIVED_AND_TRASHED'
3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket) 3. **Move to TRASH**: CSV files moved from ODS to TRASH folder (DATA bucket)
4. **Optional Cleanup**: If KEEP_IN_TRASH='N', files deleted from TRASH 4. **Optional Cleanup**: If IS_KEEP_IN_TRASH='N', files deleted from TRASH
**Automatic Rollback Trigger**: **Automatic Rollback Trigger**:
If **any error occurs** during step 3 (Move to TRASH), the system: If **any error occurs** during step 3 (Move to TRASH), the system:
@@ -655,7 +700,7 @@ SELECT
TABLE_ID, TABLE_ID,
ARCHIVAL_STRATEGY, ARCHIVAL_STRATEGY,
MINIMUM_AGE_MONTHS, MINIMUM_AGE_MONTHS,
DAYS_FOR_ARCHIVE_THRESHOLD ARCHIVE_THRESHOLD_DAYS
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
ORDER BY A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID; ORDER BY A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID;
@@ -679,8 +724,8 @@ ORDER BY ARCHIVAL_STRATEGY;
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS', SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
MINIMUM_AGE_MONTHS = 6, MINIMUM_AGE_MONTHS = 6,
ARCHIVE_ENABLED = 'Y', -- Enable archival IS_ARCHIVE_ENABLED = 'Y', -- Enable archival
KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety IS_KEEP_IN_TRASH = 'Y' -- Keep files in TRASH for safety
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'CSDB' AND SOURCE_FILE_ID = 'CSDB'
AND TABLE_ID = 'CSDB_DEBT'; AND TABLE_ID = 'CSDB_DEBT';
@@ -688,13 +733,13 @@ COMMIT;
-- Disable archival temporarily for troubleshooting -- Disable archival temporarily for troubleshooting
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET ARCHIVE_ENABLED = 'N' -- Batch operations will skip this table SET IS_ARCHIVE_ENABLED = 'N' -- Batch operations will skip this table
WHERE TABLE_ID = 'CSDB_DEBT'; WHERE TABLE_ID = 'CSDB_DEBT';
COMMIT; COMMIT;
-- Configure TRASH cleanup for test environment -- Configure TRASH cleanup for test environment
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival SET IS_KEEP_IN_TRASH = 'N' -- Delete files from TRASH after archival
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
AND SOURCE_FILE_ID = 'TEST_SOURCE'; AND SOURCE_FILE_ID = 'TEST_SOURCE';
COMMIT; COMMIT;
@@ -705,21 +750,21 @@ SELECT
TABLE_ID, TABLE_ID,
ARCHIVAL_STRATEGY, ARCHIVAL_STRATEGY,
MINIMUM_AGE_MONTHS, MINIMUM_AGE_MONTHS,
ARCHIVE_ENABLED, IS_ARCHIVE_ENABLED,
KEEP_IN_TRASH IS_KEEP_IN_TRASH
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
ORDER BY SOURCE_FILE_ID, TABLE_ID; ORDER BY SOURCE_FILE_ID, TABLE_ID;
-- Summary by archival status -- Summary by archival status
SELECT SELECT
ARCHIVE_ENABLED, IS_ARCHIVE_ENABLED,
KEEP_IN_TRASH, IS_KEEP_IN_TRASH,
COUNT(*) AS TABLE_COUNT COUNT(*) AS TABLE_COUNT
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT' WHERE SOURCE_FILE_TYPE = 'INPUT'
GROUP BY ARCHIVE_ENABLED, KEEP_IN_TRASH GROUP BY IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH
ORDER BY ARCHIVE_ENABLED DESC, KEEP_IN_TRASH DESC; ORDER BY IS_ARCHIVE_ENABLED DESC, IS_KEEP_IN_TRASH DESC;
``` ```
## Release 01 Configuration ## Release 01 Configuration
@@ -829,11 +874,11 @@ JOIN CT_ODS.A_LOAD_HISTORY LH ON SFR.A_WORKFLOW_HISTORY_KEY = LH.A_WORKFLOW_HIST
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
AND SFR.PROCESSING_STATUS = 'INGESTED' AND SFR.PROCESSING_STATUS = 'INGESTED'
AND SFC.ARCHIVE_ENABLED = 'Y' AND SFC.IS_ARCHIVE_ENABLED = 'Y'
ORDER BY LH.LOAD_START; ORDER BY LH.LOAD_START;
-- Note: MINIMUM_AGE_MONTHS archives immediately (threshold-independent) -- Note: MINIMUM_AGE_MONTHS archives immediately (threshold-independent)
-- If files not archived, check ARCHIVE_ENABLED='Y' and run ARCHIVE_TABLE_DATA -- If files not archived, check IS_ARCHIVE_ENABLED='Y' and run ARCHIVE_TABLE_DATA
``` ```
**Scenario B**: **THRESHOLD_BASED** or **HYBRID** strategy not archiving **Scenario B**: **THRESHOLD_BASED** or **HYBRID** strategy not archiving
@@ -843,9 +888,9 @@ SELECT
SFC.SOURCE_FILE_ID, SFC.SOURCE_FILE_ID,
SFC.TABLE_ID, SFC.TABLE_ID,
SFC.ARCHIVAL_STRATEGY, SFC.ARCHIVAL_STRATEGY,
SFC.FILES_COUNT_OVER_ARCHIVE_THRESHOLD AS FILE_THRESHOLD, SFC.ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THRESHOLD,
SFC.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD AS ROW_THRESHOLD, SFC.ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THRESHOLD,
SFC.BYTES_SUM_OVER_ARCHIVE_THRESHOLD AS BYTE_THRESHOLD, SFC.ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THRESHOLD,
COUNT(SFR.A_SOURCE_FILE_RECEIVED_KEY) AS CURRENT_FILES, COUNT(SFR.A_SOURCE_FILE_RECEIVED_KEY) AS CURRENT_FILES,
SUM(SFR.TOTAL_RECORDS) AS CURRENT_ROWS, SUM(SFR.TOTAL_RECORDS) AS CURRENT_ROWS,
SUM(SFR.FILE_SIZE_BYTES) AS CURRENT_BYTES SUM(SFR.FILE_SIZE_BYTES) AS CURRENT_BYTES
@@ -854,13 +899,13 @@ LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY
AND SFR.PROCESSING_STATUS = 'INGESTED' AND SFR.PROCESSING_STATUS = 'INGESTED'
WHERE SFC.ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'HYBRID') WHERE SFC.ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'HYBRID')
AND SFC.ARCHIVE_ENABLED = 'Y' AND SFC.IS_ARCHIVE_ENABLED = 'Y'
AND SFC.A_SOURCE_FILE_CONFIG_KEY = :yourConfigKey AND SFC.A_SOURCE_FILE_CONFIG_KEY = :yourConfigKey
GROUP BY GROUP BY
SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY,
SFC.FILES_COUNT_OVER_ARCHIVE_THRESHOLD, SFC.ARCHIVE_THRESHOLD_FILES_COUNT,
SFC.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD, SFC.ARCHIVE_THRESHOLD_ROWS_COUNT,
SFC.BYTES_SUM_OVER_ARCHIVE_THRESHOLD; SFC.ARCHIVE_THRESHOLD_BYTES_SUM;
-- Expected: At least ONE threshold (FILE/ROW/BYTE) must be exceeded -- Expected: At least ONE threshold (FILE/ROW/BYTE) must be exceeded
-- If no threshold exceeded, archival will NOT trigger (threshold-dependent behavior) -- If no threshold exceeded, archival will NOT trigger (threshold-dependent behavior)
@@ -903,7 +948,7 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
**Symptoms**: Files not deleted from TRASH after archival **Symptoms**: Files not deleted from TRASH after archival
**Cause**: Configuration has `KEEP_IN_TRASH='Y'` (retain files in TRASH) **Cause**: Configuration has `IS_KEEP_IN_TRASH='Y'` (retain files in TRASH)
**Verification**: **Verification**:
```sql ```sql
@@ -911,8 +956,8 @@ WHERE object_name LIKE 'ARCHIVE/LM/STANDING_FACILITIES/PARTITION_YEAR=2026/PARTI
SELECT SELECT
SOURCE_FILE_ID, SOURCE_FILE_ID,
TABLE_ID, TABLE_ID,
KEEP_IN_TRASH, IS_KEEP_IN_TRASH,
CASE KEEP_IN_TRASH CASE IS_KEEP_IN_TRASH
WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)' WHEN 'Y' THEN 'Files RETAINED in TRASH (manual purge required)'
WHEN 'N' THEN 'Files DELETED immediately after archival' WHEN 'N' THEN 'Files DELETED immediately after archival'
END AS TRASH_BEHAVIOR END AS TRASH_BEHAVIOR
@@ -924,7 +969,7 @@ WHERE TABLE_ID = 'YOUR_TABLE';
```sql ```sql
-- Option A: Change configuration to auto-delete (permanent change) -- Option A: Change configuration to auto-delete (permanent change)
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival SET IS_KEEP_IN_TRASH = 'N' -- Auto-delete from TRASH after archival
WHERE TABLE_ID = 'YOUR_TABLE'; WHERE TABLE_ID = 'YOUR_TABLE';
COMMIT; COMMIT;
@@ -997,7 +1042,7 @@ SELECT
TABLE_ID, TABLE_ID,
ARCHIVAL_STRATEGY, ARCHIVAL_STRATEGY,
MINIMUM_AGE_MONTHS, MINIMUM_AGE_MONTHS,
DAYS_FOR_ARCHIVE_THRESHOLD ARCHIVE_THRESHOLD_DAYS
FROM CT_MRDS.A_SOURCE_FILE_CONFIG FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE TABLE_ID = 'YOUR_TABLE'; WHERE TABLE_ID = 'YOUR_TABLE';
@@ -1087,7 +1132,7 @@ SELECT
SFR.FILE_SIZE_BYTES, SFR.FILE_SIZE_BYTES,
SFR.UPDATED_AT AS ARCHIVED_AT, SFR.UPDATED_AT AS ARCHIVED_AT,
TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH, TRUNC(SYSDATE - SFR.UPDATED_AT) AS DAYS_IN_TRASH,
SFC.KEEP_IN_TRASH AS TRASH_POLICY SFC.IS_KEEP_IN_TRASH AS TRASH_POLICY
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
@@ -1102,8 +1147,8 @@ SELECT
SFC.SOURCE_FILE_ID, SFC.SOURCE_FILE_ID,
SFC.TABLE_ID, SFC.TABLE_ID,
SFC.ARCHIVAL_STRATEGY, SFC.ARCHIVAL_STRATEGY,
SFC.ARCHIVE_ENABLED, SFC.IS_ARCHIVE_ENABLED,
SFC.KEEP_IN_TRASH, SFC.IS_KEEP_IN_TRASH,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'INGESTED' THEN 1 END) AS PENDING_ARCHIVE,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' THEN 1 END) AS IN_TRASH,
COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED, COUNT(CASE WHEN SFR.PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' THEN 1 END) AS PURGED,
@@ -1113,7 +1158,7 @@ LEFT JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = S
WHERE SFC.SOURCE_FILE_TYPE = 'INPUT' WHERE SFC.SOURCE_FILE_TYPE = 'INPUT'
GROUP BY GROUP BY
SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY, SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.ARCHIVAL_STRATEGY,
SFC.ARCHIVE_ENABLED, SFC.KEEP_IN_TRASH SFC.IS_ARCHIVE_ENABLED, SFC.IS_KEEP_IN_TRASH
ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID; ORDER BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID;
``` ```
@@ -1134,7 +1179,7 @@ FROM CT_MRDS.A_SOURCE_FILE_CONFIG SFC
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED SFR ON SFC.A_SOURCE_FILE_CONFIG_KEY = SFR.A_SOURCE_FILE_CONFIG_KEY
JOIN CT_ODS.A_LOAD_HISTORY LH ON SFR.A_WORKFLOW_HISTORY_KEY = LH.A_WORKFLOW_HISTORY_KEY JOIN CT_ODS.A_LOAD_HISTORY LH ON SFR.A_WORKFLOW_HISTORY_KEY = LH.A_WORKFLOW_HISTORY_KEY
WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' WHERE SFC.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
AND SFC.ARCHIVE_ENABLED = 'Y' AND SFC.IS_ARCHIVE_ENABLED = 'Y'
AND SFR.PROCESSING_STATUS = 'INGESTED' AND SFR.PROCESSING_STATUS = 'INGESTED'
AND LH.LOAD_START < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -SFC.MINIMUM_AGE_MONTHS) AND LH.LOAD_START < ADD_MONTHS(TRUNC(SYSDATE, 'MM'), -SFC.MINIMUM_AGE_MONTHS)
GROUP BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.MINIMUM_AGE_MONTHS GROUP BY SFC.SOURCE_FILE_ID, SFC.TABLE_ID, SFC.MINIMUM_AGE_MONTHS
@@ -1173,20 +1218,30 @@ SELECT
ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB, ROUND(SUM(SFR.FILE_SIZE_BYTES) / 1024 / 1024 / 1024, 2) AS SIZE_GB,
MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH, MIN(SFR.UPDATED_AT) AS OLDEST_IN_TRASH,
MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH, MAX(SFR.UPDATED_AT) AS NEWEST_IN_TRASH,
SFC.KEEP_IN_TRASH AS POLICY SFC.IS_KEEP_IN_TRASH AS POLICY
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR FROM CT_MRDS.A_SOURCE_FILE_RECEIVED SFR
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY JOIN CT_MRDS.A_SOURCE_FILE_CONFIG SFC ON SFR.A_SOURCE_FILE_CONFIG_KEY = SFC.A_SOURCE_FILE_CONFIG_KEY
WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' WHERE SFR.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
GROUP BY SFC.SOURCE_FILE_ID, SFC.KEEP_IN_TRASH GROUP BY SFC.SOURCE_FILE_ID, SFC.IS_KEEP_IN_TRASH
ORDER BY SIZE_GB DESC; ORDER BY SIZE_GB DESC;
``` ```
## Version History ## Version History
### v3.3.0 (Current - 2026-02-11) ### v3.4.0 (Current - 2026-03-17)
- **MARS-1409**: Added `IS_WORKFLOW_SUCCESS_REQUIRED` flag to A_SOURCE_FILE_CONFIG
- `'Y'` (default) = archivization requires `WORKFLOW_SUCCESSFUL='Y'` in A_WORKFLOW_HISTORY (standard Airflow+DBT flow)
- `'N'` = archive regardless of workflow status (bypass for manual/non-DBT sources)
- `IS_WORKFLOW_SUCCESS_REQUIRED` stored in A_TABLE_STAT and A_TABLE_STAT_HIST at statistics gather time
- GATHER_TABLE_STAT: conditional `WORKFLOW_SUCCESSFUL='Y'` filter controlled by the flag
- ARCHIVE_TABLE_DATA: conditional `WORKFLOW_SUCCESSFUL='Y'` filter controlled by the flag
- Added `pIsWorkflowSuccessRequired` parameter to FILE_MANAGER.ADD_SOURCE_FILE_CONFIG
- FILE_MANAGER updated to v3.6.2+
### v3.3.0 (2026-02-11)
- **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA - **BREAKING CHANGE**: Removed `pKeepInTrash` parameter from ARCHIVE_TABLE_DATA
- Added `ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control - Added `IS_ARCHIVE_ENABLED` column to A_SOURCE_FILE_CONFIG for selective archiving control
- Added `KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter) - Added `IS_KEEP_IN_TRASH` column to A_SOURCE_FILE_CONFIG (replaces pKeepInTrash parameter)
- Added batch procedures with 3-level granularity (config/source/all): - Added batch procedures with 3-level granularity (config/source/all):
- ARCHIVE_ALL - Batch archival procedure - ARCHIVE_ALL - Batch archival procedure
- GATHER_TABLE_STAT_ALL - Batch statistics procedure - GATHER_TABLE_STAT_ALL - Batch statistics procedure
@@ -1226,7 +1281,7 @@ ORDER BY SIZE_GB DESC;
### v2.0.0 (Legacy) ### v2.0.0 (Legacy)
- Initial FILE_ARCHIVER package - Initial FILE_ARCHIVER package
- THRESHOLD_BASED archival only - THRESHOLD_BASED archival only
- Fixed DAYS_FOR_ARCHIVE_THRESHOLD configuration - Fixed ARCHIVE_THRESHOLD_DAYS configuration
## Related Documentation ## Related Documentation
@@ -1337,7 +1392,7 @@ ORDER BY SIZE_GB DESC;
### TRASH Folder Retention Best Practices ### TRASH Folder Retention Best Practices
1. **Default Behavior (KEEP_IN_TRASH = 'Y' - Recommended)**: 1. **Default Behavior (IS_KEEP_IN_TRASH = 'Y' - Recommended)**:
- Keeps CSV files in TRASH folder after archival - Keeps CSV files in TRASH folder after archival
- Provides safety net for rollback if archival issues occur - Provides safety net for rollback if archival issues occur
- Supports compliance and audit requirements - Supports compliance and audit requirements
@@ -1346,11 +1401,11 @@ ORDER BY SIZE_GB DESC;
- Configuration: - Configuration:
```sql ```sql
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'Y' SET IS_KEEP_IN_TRASH = 'Y'
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
``` ```
2. **TRASH Cleanup (KEEP_IN_TRASH = 'N')**: 2. **TRASH Cleanup (IS_KEEP_IN_TRASH = 'N')**:
- Deletes CSV files from TRASH folder after successful archival - Deletes CSV files from TRASH folder after successful archival
- Reduces storage costs in DATA bucket - Reduces storage costs in DATA bucket
- Status: ARCHIVED_AND_PURGED - Status: ARCHIVED_AND_PURGED
@@ -1358,7 +1413,7 @@ ORDER BY SIZE_GB DESC;
- Configuration: - Configuration:
```sql ```sql
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
SET KEEP_IN_TRASH = 'N' SET IS_KEEP_IN_TRASH = 'N'
WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE'; WHERE SOURCE_FILE_TYPE = 'INPUT' AND TABLE_ID = 'YOUR_TABLE';
``` ```