admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(MARS-1409): Rename IS_KEEP_IN_TRASH to IS_KEPT_IN_TRASH across relevant files and update related logic

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-19 12:23:29 +01:00
parent c96bf2051f
commit 12c58f32a3
8 changed files with 54 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
MARS_Packages/REL02_POST/MARS-1409/01_MARS_1409_add_workflow_history_key_column.sql
View File

@@ -35,5 +35,21 @@ END;
/
PROMPT
PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added to A_SOURCE_FILE_RECEIVED.
PROMPT Renaming IS_KEEP_IN_TRASH to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG...
BEGIN
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEEP_IN_TRASH TO IS_KEPT_IN_TRASH';
DBMS_OUTPUT.PUT_LINE('Column IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH in A_SOURCE_FILE_CONFIG.');
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -904 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEEP_IN_TRASH does not exist (already renamed or not present).');
ELSE
RAISE;
END IF;
END;
/
PROMPT
PROMPT Step 01 completed: A_WORKFLOW_HISTORY_KEY column added and IS_KEEP_IN_TRASH renamed to IS_KEPT_IN_TRASH.
PROMPT

18
MARS_Packages/REL02_POST/MARS-1409/99_MARS_1409_rollback_workflow_history_key_column.sql
View File

@@ -26,5 +26,21 @@ END;
/
PROMPT
PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY column removed from A_SOURCE_FILE_RECEIVED.
PROMPT Renaming IS_KEPT_IN_TRASH back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG...
BEGIN
EXECUTE IMMEDIATE 'ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG RENAME COLUMN IS_KEPT_IN_TRASH TO IS_KEEP_IN_TRASH';
DBMS_OUTPUT.PUT_LINE('Column IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH in A_SOURCE_FILE_CONFIG.');
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -904 THEN
DBMS_OUTPUT.PUT_LINE('SKIP: Column IS_KEPT_IN_TRASH does not exist (already renamed back or not present).');
ELSE
RAISE;
END IF;
END;
/
PROMPT
PROMPT Rollback 99 completed: A_WORKFLOW_HISTORY_KEY removed and IS_KEPT_IN_TRASH renamed back to IS_KEEP_IN_TRASH.
PROMPT

6
MARS_Packages/REL02_POST/MARS-1409/new_version/A_SOURCE_FILE_CONFIG.sql
View File

@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
@@ -97,7 +97,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS

12
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkb
View File

@@ -120,8 +120,8 @@ AS
END IF;
-- Get TRASH policy from configuration
vKeepInTrash := (vSourceFileConfig.IS_KEEP_IN_TRASH = 'Y');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEEP_IN_TRASH=' || vSourceFileConfig.IS_KEEP_IN_TRASH, 'INFO', vParameters);
vKeepInTrash := (vSourceFileConfig.IS_KEPT_IN_TRASH = 'Y');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH policy from config: IS_KEPT_IN_TRASH=' || vSourceFileConfig.IS_KEPT_IN_TRASH, 'INFO', vParameters);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
@@ -309,10 +309,10 @@ AS
AND r.source_file_name = f.filename
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
END LOOP;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEEP_IN_TRASH=N).','INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED (config: IS_KEPT_IN_TRASH=N).','INFO');
ELSE
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEEP_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO');
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (config: IS_KEPT_IN_TRASH=Y, status: ARCHIVED_AND_TRASHED).','INFO');
END IF;
--ROLLBACK PART
@@ -1112,7 +1112,7 @@ AS
A_SOURCE_FILE_CONFIG_KEY,
TABLE_ID,
IS_ARCHIVE_ENABLED,
IS_KEEP_IN_TRASH,
IS_KEPT_IN_TRASH,
A_SOURCE_KEY
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = 'INPUT'
@@ -1137,7 +1137,7 @@ AS
ELSE
BEGIN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEEP_IN_TRASH=' || config_rec.IS_KEEP_IN_TRASH || ']',
'Archiving table ' || config_rec.TABLE_ID || ' [Source: ' || config_rec.A_SOURCE_KEY || ', Config: ' || config_rec.A_SOURCE_FILE_CONFIG_KEY || ', IS_KEPT_IN_TRASH=' || config_rec.IS_KEPT_IN_TRASH || ']',
'INFO'
);

8
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_ARCHIVER.pkg
View File

@@ -27,7 +27,7 @@ AS
'Y=standard DBT flow (WORKFLOW_SUCCESSFUL=Y required), N=bypass for manual/non-DBT sources. ' ||
'Flag value stored in A_TABLE_STAT and A_TABLE_STAT_HIST for full audit of statistics basis.' || CHR(13)||CHR(10) ||
'3.3.1 (2026-03-13): Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) ||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEPT_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
@@ -55,7 +55,7 @@ AS
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
**/
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
@@ -66,7 +66,7 @@ AS
* @desc Function wrapper for ARCHIVE_TABLE_DATA procedure.
* Returns SQLCODE for Python library integration.
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
* @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
* @ex_rslt 0 (success) or error code
**/
@@ -141,7 +141,7 @@ AS
* @name ARCHIVE_ALL
* @desc Multi-level batch archival procedure with three granularity levels.
* Only processes configurations where IS_ARCHIVE_ENABLED='Y'.
* TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column.
* TRASH policy for each table is controlled by individual IS_KEPT_IN_TRASH column.
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority)
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)

6
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkb
View File

@@ -1634,7 +1634,7 @@ AS
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
) IS
@@ -1659,7 +1659,7 @@ AS
,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS)
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths);
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
@@ -1770,7 +1770,7 @@ AS
||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
||cgBL||pLevel||'ENCODING = '||pSourceFileConfig.ENCODING
||cgBL||pLevel||'IS_ARCHIVE_ENABLED = '||pSourceFileConfig.IS_ARCHIVE_ENABLED
||cgBL||pLevel||'IS_KEEP_IN_TRASH = '||pSourceFileConfig.IS_KEEP_IN_TRASH
||cgBL||pLevel||'IS_KEPT_IN_TRASH = '||pSourceFileConfig.IS_KEPT_IN_TRASH
||cgBL||pLevel||'ARCHIVAL_STRATEGY = '||pSourceFileConfig.ARCHIVAL_STRATEGY
||cgBL||pLevel||'MINIMUM_AGE_MONTHS = '||pSourceFileConfig.MINIMUM_AGE_MONTHS
||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED

2
MARS_Packages/REL02_POST/MARS-1409/new_version/FILE_MANAGER.pkg
View File

@@ -476,7 +476,7 @@ AS
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049
,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409
,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828
,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828
,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828
);

6
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql
View File

@@ -27,11 +27,11 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
MINIMUM_AGE_MONTHS NUMBER(3,0),
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_KEPT_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
IS_WORKFLOW_SUCCESS_REQUIRED CHAR(1) DEFAULT 'Y' NOT NULL,
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_KEPT_IN_TRASH CHECK (IS_KEPT_IN_TRASH IN ('Y', 'N')),
CONSTRAINT CHK_IS_WORKFLOW_SUCCESS_REQUIRED CHECK (IS_WORKFLOW_SUCCESS_REQUIRED IN ('Y', 'N')),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
@@ -107,7 +107,7 @@ COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH IS
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED IS