Compare commits

...

5 Commits

Author SHA1 Message Date
Grzegorz Michalski
6468d12349 minor 2026-03-13 13:51:53 +01:00
Grzegorz Michalski
fe0f7bce18 feat: Enhance FILE_ARCHIVER package to handle empty ODS bucket scenarios with improved statistics initialization 2026-03-13 13:34:38 +01:00
Grzegorz Michalski
6b2f60f413 feat: Update FILE_ARCHIVER package to version 3.3.1 with improved handling for empty ODS bucket scenarios 2026-03-13 11:40:19 +01:00
Grzegorz Michalski
ca11debd93 minor 2026-03-13 11:35:11 +01:00
Grzegorz Michalski
24e6bce18c minor changes 2026-03-13 11:34:59 +01:00
7 changed files with 47 additions and 61 deletions

View File

@@ -63,7 +63,7 @@ PROMPT
PROMPT ============================================================================
PROMPT STEP 2: Diagnose workflow key status
PROMPT ============================================================================
@@02_MARS_1409_POSTHOOK_diagnose_workflow_key_status.sql
@@diagnose_workflow_key_status.sql
PROMPT
PROMPT ============================================================================

View File

@@ -35,7 +35,6 @@ PROMPT This will reverse all changes from MARS-1409-POSTHOOK installation.
PROMPT
PROMPT Rollback steps:
PROMPT 1. Clear A_WORKFLOW_HISTORY_KEY values from A_SOURCE_FILE_RECEIVED
PROMPT 2. Verify package versions
PROMPT ============================================================================
-- Confirm rollback with user
@@ -55,12 +54,6 @@ PROMPT STEP 1: Clear backfilled A_WORKFLOW_HISTORY_KEY values
PROMPT ============================================================================
@@91_MARS_1409_POSTHOOK_rollback_workflow_keys.sql
PROMPT
PROMPT ============================================================================
PROMPT STEP 2: Verify Package Versions
PROMPT ============================================================================
@@verify_packages_version.sql
PROMPT
PROMPT ============================================================================
PROMPT MARS-1409-POSTHOOK Rollback Complete

View File

@@ -125,3 +125,5 @@ PROMPT Review the log file for detailed results: &_filename
PROMPT ============================================================================
spool off
quit;

View File

@@ -171,7 +171,19 @@ AS
;
-- Get all files that will be archived into "vfiles" collection ("regular data files")
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
BEGIN
execute immediate vQuery bulk collect into vfiles;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -29913 THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Nothing to archive.', 'INFO', vParameters);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN;
ELSE
RAISE;
END IF;
END;
-- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
@@ -479,11 +491,11 @@ AS
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
,'''||vTableName||''' as TABLE_NAME
,count(*) as FILE_COUNT
,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT
,sum (row_count_per_file) as ROW_COUNT
,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT
,sum(r.bytes) as BYTES
,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES
,nvl(sum(case when ' || vWhereClause || ' then 1 else 0 end), 0) as OLD_FILE_COUNT
,nvl(sum(row_count_per_file), 0) as ROW_COUNT
,nvl(sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end), 0) as OLD_ROW_COUNT
,nvl(sum(r.bytes), 0) as BYTES
,nvl(sum(case when ' || vWhereClause || ' then r.bytes else 0 end), 0) as OLD_BYTES
,'||COALESCE(TO_CHAR(vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS), 'NULL')||' as ARCHIVE_THRESHOLD_DAYS
,systimestamp as CREATED
from tmp_gr t
@@ -495,7 +507,27 @@ AS
on t.filename = r.object_name'
;
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
-- MARS-1468: Handle ORA-29913 - no files in ODS bucket (empty external table location)
BEGIN
execute immediate vQuery into vStats;
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -29913 THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('No files found in ODS bucket (ORA-29913: empty location). Saving zero statistics.', 'INFO', vParameters);
vStats.A_SOURCE_FILE_CONFIG_KEY := pSourceFileConfigKey;
vStats.TABLE_NAME := vTableName;
vStats.FILE_COUNT := 0;
vStats.OVER_ARCH_THRESOLD_FILE_COUNT := 0;
vStats.ROW_COUNT := 0;
vStats.OVER_ARCH_THRESOLD_ROW_COUNT := 0;
vStats."SIZE" := 0;
vStats.OVER_ARCH_THRESOLD_SIZE := 0;
vStats.ARCH_THRESHOLD_DAYS := vSourceFileConfig.ARCHIVE_THRESHOLD_DAYS;
vStats.CREATED := SYSTIMESTAMP;
ELSE
RAISE;
END IF;
END;
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
insert into CT_MRDS.A_TABLE_STAT_HIST values vStats;

View File

@@ -17,12 +17,13 @@ AS
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00';
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.1';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-03-13 09:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.3.1 (2026-03-13): MARS-1468 - Fixed ORA-29913 handling in ARCHIVE_TABLE_DATA (graceful RETURN when ODS bucket is empty) and GATHER_TABLE_STAT (saves zero statistics instead of raising error)' || CHR(13)||CHR(10) ||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||

View File

@@ -1,44 +0,0 @@
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."A_WORKFLOW_HISTORY"
AFTER INSERT OR UPDATE OF workflow_successful ON ct_mrds.a_workflow_history
REFERENCING NEW AS new OLD AS old
FOR EACH ROW
DECLARE
v_workflow_name VARCHAR2(128);
v_wla_id NUMBER;
BEGIN
IF :new.service_name = 'ODS' AND :new.workflow_name IN (
'w_ODS_LM_STANDING_FACILITIES', 'w_ODS_CSDB_DEBT', 'w_ODS_CSDB_DEBT_DAILY', 'w_ODS_CSDB_RATINGS_FULL',
'w_ODS_TMS_LIMIT_ACCESS', 'w_ODS_TMS_PORTFOLIO_ACCESS', 'w_ODS_TMS_PORTFOLIO_TREE',
'w_ODS_TMS_COLLATERAL_INVENTORY', 'w_ODS_TOP_FULLBIDARRAY_COMPILED', 'w_ODS_TOP_ANNOUNCEMENT',
'w_ODS_TOP_ALLOTMENT_MODIFICATIONS', 'w_ODS_TOP_ALLOTMENT', 'w_ODS_CEPH_PRICING', 'w_ODS_C2D_MPEC'
) THEN
IF :new.workflow_successful = 'Y' AND :new.workflow_successful <> NVL(:old.workflow_successful, 'N') THEN
CASE
WHEN :new.workflow_name = 'w_ODS_LM_STANDING_FACILITIES' THEN v_workflow_name := 'w_ODS_LM_STANDING_FACILITY';
WHEN :new.workflow_name = 'w_ODS_TMS_LIMIT_ACCESS' THEN v_workflow_name := 'w_ODS_TMS_RAR_LIMITACCESS';
WHEN :new.workflow_name = 'w_ODS_TMS_PORTFOLIO_ACCESS' THEN v_workflow_name := 'w_ODS_TMS_RAR_PORTFOLIOACCESS';
WHEN :new.workflow_name = 'w_ODS_TMS_PORTFOLIO_TREE' THEN v_workflow_name := 'w_ODS_TMS_RAR_PORTFOLIOTREE';
WHEN :new.workflow_name = 'w_ODS_TMS_COLLATERAL_INVENTORY' THEN v_workflow_name := 'w_ODS_TMS_RAR_RARCOLLATERALINVENTORY';
WHEN :new.workflow_name = 'w_ODS_TOP_FULLBIDARRAY_COMPILED' THEN v_workflow_name := 'w_ODS_TOP_FULLBIDARRAY_COMPILED';
WHEN :new.workflow_name = 'w_ODS_TOP_ANNOUNCEMENT' THEN v_workflow_name := 'w_ODS_TOP_ANNOUNCEMENT';
WHEN :new.workflow_name = 'w_ODS_TOP_ALLOTMENT_MODIFICATIONS' THEN v_workflow_name := 'w_ODS_TOP_ALLOTMENT_MODIFICATIONS';
WHEN :new.workflow_name = 'w_ODS_TOP_ALLOTMENT' THEN v_workflow_name := 'w_ODS_TOP_ALLOTMENT';
WHEN :new.workflow_name = 'w_ODS_CEPH_PRICING' THEN v_workflow_name := 'w_ODS_CEPH_PRICING';
WHEN :new.workflow_name = 'w_ODS_C2D_MPEC' THEN v_workflow_name := 'w_ODS_C2D_MPEC';
ELSE
v_workflow_name := :new.workflow_name;
END CASE;
BEGIN
v_wla_id := TO_NUMBER(:new.orchestration_run_id);
EXCEPTION WHEN OTHERS THEN NULL;
END;
INSERT INTO ct_ods.a_load_history (
a_etl_load_set_key, workflow_name, infa_run_id, load_start, load_end, exdi_appl_req_id, exdi_correlation_id, load_successful, wla_run_id, dq_flag
) VALUES (
:new.a_workflow_history_key, v_workflow_name, NULL, :new.workflow_start, :new.workflow_end, NULL, NULL, :new.workflow_successful, v_wla_id, 'F'
);
END IF;
END IF;
END
;
/

View File

@@ -118,3 +118,5 @@ PROMPT Review the log file for detailed results: &_filename
PROMPT ============================================================================
spool off
quit;