admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FILE_MANAGER): Update package version to 3.6.3 and enhance ADD_SOURCE_FILE_CONFIG with new parameters for archival control

Browse Source 
- Bump package version to 3.6.3 and update build date.
- Add new parameters: pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths to ADD_SOURCE_FILE_CONFIG.
- Include pIsWorkflowSuccessRequired parameter to control workflow success requirement for archival.
- Update version history to reflect changes.

feat(A_SOURCE_FILE_CONFIG): Modify table structure to include new archival control flags

- Add IS_WORKFLOW_SUCCESS_REQUIRED column to A_SOURCE_FILE_CONFIG for workflow bypass functionality.
- Update constraints and comments for new columns.
- Ensure backward compatibility with default values.

fix(A_TABLE_STAT, A_TABLE_STAT_HIST): Extend table structures to accommodate new workflow success tracking

- Add IS_WORKFLOW_SUCCESS_REQUIRED column to both A_TABLE_STAT and A_TABLE_STAT_HIST.
- Update comments to clarify the purpose of new columns.

docs(FILE_ARCHIVER_Guide): Revise documentation to reflect new archival features and configurations

- Document new IS_WORKFLOW_SUCCESS_REQUIRED flag and its implications for archival processes.
- Update examples and configurations to align with recent changes in the database schema.
- Ensure clarity on archival strategies and their configurations.
This commit is contained in:
Grzegorz Michalski
2026-03-18 18:19:04 +01:00
parent 896e67bcb9
commit ce9b6eeff6
25 changed files with 986 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql
View File

@@ -4,6 +4,7 @@
-- Purpose: Store source file configuration and processing rules
-- MARS-1049: Added ENCODING column for CSV character set support
-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation
-- MARS-1409: Added IS_WORKFLOW_SUCCESS_REQUIRED flag for workflow bypass
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
@@ -27,9 +28,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
@@ -107,4 +110,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS
'Y=Archivization requires WORKFLOW_SUCCESSFUL=Y (standard DBT flow), N=Archive regardless of workflow completion status (bypass for manual/non-DBT sources). Added MARS-1409';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql
View File

@@ -18,11 +18,13 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT (
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1),
CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY)
) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409';
-- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1"

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql
View File

@@ -17,8 +17,10 @@ CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST (
ARCH_THRESHOLD_DAYS NUMBER(4,0),
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
ARCHIVAL_STRATEGY VARCHAR2(30),
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0)
ARCH_MINIMUM_AGE_MONTHS NUMBER(4,0),
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1)
) TABLESPACE "DATA";
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCHIVAL_STRATEGY IS 'Archival strategy used when statistics were gathered (THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID). Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.ARCH_MINIMUM_AGE_MONTHS IS 'Minimum age threshold in months used when statistics were gathered. Populated for MINIMUM_AGE_MONTHS and HYBRID strategies; NULL for THRESHOLD_BASED. Complements ARCH_THRESHOLD_DAYS. Populated by FILE_ARCHIVER.GATHER_TABLE_STAT (MARS-1409)';
COMMENT ON COLUMN CT_MRDS.A_TABLE_STAT_HIST.IS_WORKFLOW_SUCCESS_REQUIRED IS 'Reflects IS_WORKFLOW_SUCCESS_REQUIRED value from A_SOURCE_FILE_CONFIG at time statistics were gathered. Y=counts include only WORKFLOW_SUCCESSFUL=Y rows, N=all rows counted regardless of workflow status. Added MARS-1409';