From a4ab56e96d33e8c11caabbedbf89c6feef3ff13c Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Fri, 13 Feb 2026 12:36:45 +0100 Subject: [PATCH] Enhance MARS-835 rollback process with new scripts for file registration deletion and manual cleanup, and add process tracking identifier for exports --- .../MARS-835/01_MARS_835_install_step1.sql | 6 +- ...0_MARS_835_rollback_file_registrations.sql | 54 ++++++++ .../MARS-835/91_MARS_835_rollback_step1.sql | 72 +++++----- .../REL01_ADDITIONS/MARS-835/check_files.sql | 80 +++++++++++ .../MARS-835/manual_cleanup.sql | 126 ++++++++++++++++++ .../MARS-835/manual_cleanup_parquet_only.sql | 93 +++++++++++++ .../MARS-835/rollback_mars835.sql | 8 +- 7 files changed, 406 insertions(+), 33 deletions(-) create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-835/90_MARS_835_rollback_file_registrations.sql create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql create mode 100644 MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql index 3dbc997..f348849 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql @@ -107,7 +107,8 @@ BEGIN pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT', pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) - pRegisterExport => TRUE -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) + pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) + pProcessName => 'MARS-835' -- Process identifier for tracking ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order'); @@ -227,7 +228,8 @@ BEGIN pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY', pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) - pRegisterExport => TRUE -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) + pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES) + pProcessName => 'MARS-835' -- Process identifier for tracking ); DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order'); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/90_MARS_835_rollback_file_registrations.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/90_MARS_835_rollback_file_registrations.sql new file mode 100644 index 0000000..1de8581 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/90_MARS_835_rollback_file_registrations.sql @@ -0,0 +1,54 @@ +--============================================================================================================================= +-- MARS-835 ROLLBACK: Delete File Registration Records +--============================================================================================================================= +-- Purpose: Delete all file registration records from A_SOURCE_FILE_RECEIVED table for MARS-835 process +-- Author: Grzegorz Michalski +-- Date: 2026-02-13 +-- Related: MARS-835 - CSDB Data Export Rollback +--============================================================================================================================= + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET TIMING ON + +PROMPT ======================================================================== +PROMPT ROLLBACK: Deleting file registration records from A_SOURCE_FILE_RECEIVED +PROMPT ======================================================================== + +DECLARE + vRowCount NUMBER := 0; + vStartTime TIMESTAMP := SYSTIMESTAMP; + vEndTime TIMESTAMP; + vElapsedSeconds NUMBER; +BEGIN + DBMS_OUTPUT.PUT_LINE('Deleting all MARS-835 file registrations from A_SOURCE_FILE_RECEIVED...'); + + -- Delete all records for MARS-835 process + DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE PROCESS_NAME = 'MARS-835'; + + vRowCount := SQL%ROWCOUNT; + COMMIT; + + vEndTime := SYSTIMESTAMP; + vElapsedSeconds := EXTRACT(SECOND FROM (vEndTime - vStartTime)) + + EXTRACT(MINUTE FROM (vEndTime - vStartTime)) * 60 + + EXTRACT(HOUR FROM (vEndTime - vStartTime)) * 3600; + + DBMS_OUTPUT.PUT_LINE('========================================================================'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: File registration records deleted'); + DBMS_OUTPUT.PUT_LINE('========================================================================'); + DBMS_OUTPUT.PUT_LINE('Records deleted: ' || vRowCount); + DBMS_OUTPUT.PUT_LINE('Elapsed time: ' || ROUND(vElapsedSeconds, 2) || ' seconds'); + DBMS_OUTPUT.PUT_LINE('========================================================================'); +EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + DBMS_OUTPUT.PUT_LINE('ERROR: Failed to delete file registration records'); + DBMS_OUTPUT.PUT_LINE('Error message: ' || SQLERRM); + RAISE; +END; +/ + +--============================================================================================================================= +-- End of Script +--============================================================================================================================= diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql index 20848a8..b5e283c 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/91_MARS_835_rollback_step1.sql @@ -22,25 +22,24 @@ DECLARE vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; BEGIN -- Get bucket URIs and credential vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket'); - -- Delete CSV files from DATA bucket (only files matching export pattern) - -- Pattern matches: LEGACY_DEBT_YYYYMM.csv OR LEGACY_DEBT_YYYYMM_1_20260122T...Z.csv (Oracle timestamp) + -- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS FOR rec IN ( - SELECT object_name + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => vCredentialName, location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' )) - WHERE object_name LIKE 'LEGACY_DEBT_%' - AND object_name LIKE '%.csv' - AND REGEXP_LIKE(object_name, '^LEGACY_DEBT_[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.csv$') -- YYYYMM or YYYYMM_1_timestamp + WHERE object_name LIKE 'LEGACY_DEBT%' ) LOOP BEGIN DBMS_CLOUD.DELETE_OBJECT( @@ -48,6 +47,7 @@ BEGIN object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name ); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; EXCEPTION WHEN OTHERS THEN IF SQLCODE = -20404 THEN @@ -58,19 +58,20 @@ BEGIN END; END LOOP; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT files from HIST bucket...'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))'); - -- Delete Parquet files from HIST bucket (only files matching export pattern) - -- Pattern matches: YYYYMM.parquet OR YYYYMM_1_20260122T...Z.parquet (Oracle timestamp) + DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)'); + vFileCount := 0; + + -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS FOR rec IN ( - SELECT object_name + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => vCredentialName, location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' )) - WHERE object_name LIKE '%PARTITION_YEAR=%' -- Hive-style partitioning folders - AND object_name LIKE '%.parquet' - AND REGEXP_LIKE(object_name, '[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.parquet$') -- YYYYMM or YYYYMM_1_timestamp + WHERE object_name NOT LIKE '%/' -- Exclude directories ) LOOP BEGIN DBMS_CLOUD.DELETE_OBJECT( @@ -78,6 +79,7 @@ BEGIN object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name ); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; EXCEPTION WHEN OTHERS THEN IF SQLCODE = -20404 THEN @@ -88,7 +90,11 @@ BEGIN END; END LOOP; - DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT files deleted'); + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No DEBT Parquet files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT Parquet files deleted from ARCHIVE bucket (' || vFileCount || ' file(s))'); END; / @@ -104,25 +110,24 @@ DECLARE vDataBucketUri VARCHAR2(500); vHistBucketUri VARCHAR2(500); vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; BEGIN -- Get bucket URIs and credential vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket'); - -- Delete CSV files from DATA bucket (only files matching export pattern) - -- Pattern matches: LEGACY_DEBT_DAILY_YYYYMM.csv OR LEGACY_DEBT_DAILY_YYYYMM_1_timestamp.csv + -- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS FOR rec IN ( - SELECT object_name + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => vCredentialName, location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' )) - WHERE object_name LIKE 'LEGACY_DEBT_DAILY_%' - AND object_name LIKE '%.csv' - AND REGEXP_LIKE(object_name, '^LEGACY_DEBT_DAILY_[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.csv$') -- YYYYMM or YYYYMM_1_timestamp + WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' ) LOOP BEGIN DBMS_CLOUD.DELETE_OBJECT( @@ -130,6 +135,7 @@ BEGIN object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name ); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; EXCEPTION WHEN OTHERS THEN IF SQLCODE = -20404 THEN @@ -140,19 +146,20 @@ BEGIN END; END LOOP; - DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY files from HIST bucket...'); + DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))'); - -- Delete Parquet files from HIST bucket (only files matching export pattern) - -- Pattern matches: YYYYMM.parquet OR YYYYMM_1_timestamp.parquet + DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...'); + DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)'); + vFileCount := 0; + + -- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS FOR rec IN ( - SELECT object_name + SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => vCredentialName, location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' )) - WHERE object_name LIKE '%PARTITION_YEAR=%' -- Hive-style partitioning folders - AND object_name LIKE '%.parquet' - AND REGEXP_LIKE(object_name, '[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.parquet$') -- YYYYMM or YYYYMM_1_timestamp + WHERE object_name NOT LIKE '%/' -- Exclude directories ) LOOP BEGIN DBMS_CLOUD.DELETE_OBJECT( @@ -160,6 +167,7 @@ BEGIN object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name ); DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; EXCEPTION WHEN OTHERS THEN IF SQLCODE = -20404 THEN @@ -170,7 +178,11 @@ BEGIN END; END LOOP; - DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY files deleted'); + IF vFileCount = 0 THEN + DBMS_OUTPUT.PUT_LINE(' INFO: No DEBT_DAILY Parquet files found to delete'); + END IF; + + DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY Parquet files deleted from ARCHIVE bucket (' || vFileCount || ' file(s))'); END; / diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql new file mode 100644 index 0000000..d51f9cc --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql @@ -0,0 +1,80 @@ +SET SERVEROUTPUT ON SIZE UNLIMITED +SET DEFINE OFF + +DECLARE + vCredential VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL'; + vDataBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/data/o/'; + vArchiveBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/history/o/'; + vCount NUMBER := 0; +BEGIN + DBMS_OUTPUT.PUT_LINE('=== Checking CSV files in DATA bucket ==='); + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredential, + location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/' + )) + WHERE object_name LIKE 'LEGACY_DEBT%' + ) LOOP + vCount := vCount + 1; + DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT: ' || vCount); + + vCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredential, + location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT_DAILY/' + )) + WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' + ) LOOP + vCount := vCount + 1; + DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT_DAILY: ' || vCount); + + DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Checking Parquet files in ARCHIVE bucket ==='); + vCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredential, + location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT/' + )) + WHERE ROWNUM <= 5 + ) LOOP + vCount := vCount + 1; + DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT (first 5): ' || vCount); + + vCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredential, + location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' + )) + ) LOOP + vCount := vCount + 1; + DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT_DAILY: ' || vCount); + + DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Now testing DELETE_OBJECT ==='); + DBMS_OUTPUT.PUT_LINE('Testing delete for: ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv'); + + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredential, + object_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv' + ); + DBMS_OUTPUT.PUT_LINE('SUCCESS: File deleted'); + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE('ERROR: ' || SQLERRM); + END; +END; +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql new file mode 100644 index 0000000..33a5790 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql @@ -0,0 +1,126 @@ +--============================================================================================================================= +-- MARS-835 Manual Cleanup - Delete remaining files after rollback +--============================================================================================================================= + +SET SERVEROUTPUT ON SIZE UNLIMITED +SET DEFINE OFF + +DECLARE + vDataBucketUri VARCHAR2(500); + vArchiveBucketUri VARCHAR2(500); + vCredentialName VARCHAR2(100); + vFileCount NUMBER := 0; +BEGIN + -- Get bucket URIs and credential + vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); + vArchiveBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); + vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; + + DBMS_OUTPUT.PUT_LINE('========================================================================'); + DBMS_OUTPUT.PUT_LINE('MARS-835 Manual Cleanup'); + DBMS_OUTPUT.PUT_LINE('========================================================================'); + + -- Delete DEBT CSV files from DATA bucket + DBMS_OUTPUT.PUT_LINE(CHR(10) || '1. Deleting DEBT CSV files from DATA bucket...'); + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' + )) + WHERE object_name LIKE 'LEGACY_DEBT%' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); + END; + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); + + -- Delete DEBT_DAILY CSV files from DATA bucket + DBMS_OUTPUT.PUT_LINE(CHR(10) || '2. Deleting DEBT_DAILY CSV files from DATA bucket...'); + vFileCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' + )) + WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); + END; + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); + + -- Delete DEBT Parquet files from ARCHIVE bucket + DBMS_OUTPUT.PUT_LINE(CHR(10) || '3. Deleting DEBT Parquet files from ARCHIVE bucket...'); + vFileCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' + )) + WHERE object_name NOT LIKE '%/' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); + END; + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); + + -- Delete DEBT_DAILY Parquet files from ARCHIVE bucket + DBMS_OUTPUT.PUT_LINE(CHR(10) || '4. Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...'); + vFileCount := 0; + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' + )) + WHERE object_name NOT LIKE '%/' + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); + END; + END LOOP; + DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); + + DBMS_OUTPUT.PUT_LINE(CHR(10) || '========================================================================'); + DBMS_OUTPUT.PUT_LINE('Manual cleanup completed'); + DBMS_OUTPUT.PUT_LINE('========================================================================'); +END; +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql new file mode 100644 index 0000000..1ff5fa8 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql @@ -0,0 +1,93 @@ +-- MARS-835: Manual cleanup of Parquet files only (after bugfix) +-- Description: Removes orphaned Parquet files from ARCHIVE bucket +-- Usage: Execute as CT_MRDS user + +SET SERVEROUTPUT ON SIZE UNLIMITED + +DECLARE + vCredentialName VARCHAR2(100) := CT_MRDS.ENV_MANAGER.gvCredentialName; + vHistBucketUri VARCHAR2(200) := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); + vFileCount NUMBER := 0; + vStartTime TIMESTAMP := SYSTIMESTAMP; + vElapsedTime INTERVAL DAY TO SECOND; +BEGIN + DBMS_OUTPUT.PUT_LINE('=========================================================='); + DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP: Parquet files only'); + DBMS_OUTPUT.PUT_LINE('=========================================================='); + DBMS_OUTPUT.PUT_LINE('Start Time: ' || TO_CHAR(vStartTime, 'YYYY-MM-DD HH24:MI:SS')); + DBMS_OUTPUT.PUT_LINE('Credential: ' || vCredentialName); + DBMS_OUTPUT.PUT_LINE('Archive Bucket: ' || vHistBucketUri); + DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------'); + + -- Delete CSDB_DEBT Parquet files + DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT Parquet files...'); + vFileCount := 0; + + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' + )) + WHERE object_name NOT LIKE '%/' -- Exclude directories + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name); + END IF; + END; + END LOOP; + + DBMS_OUTPUT.PUT_LINE('CSDB_DEBT Parquet files deleted: ' || vFileCount); + + -- Delete CSDB_DEBT_DAILY Parquet files + DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT_DAILY Parquet files...'); + vFileCount := 0; + + FOR rec IN ( + SELECT object_name + FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( + credential_name => vCredentialName, + location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' + )) + WHERE object_name NOT LIKE '%/' -- Exclude directories + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => vCredentialName, + object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name + ); + DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); + vFileCount := vFileCount + 1; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -20404 THEN + DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); + ELSE + DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name); + END IF; + END; + END LOOP; + + DBMS_OUTPUT.PUT_LINE('CSDB_DEBT_DAILY Parquet files deleted: ' || vFileCount); + + -- Final summary + vElapsedTime := SYSTIMESTAMP - vStartTime; + DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------'); + DBMS_OUTPUT.PUT_LINE('End Time: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS')); + DBMS_OUTPUT.PUT_LINE('Elapsed Time: ' || vElapsedTime); + DBMS_OUTPUT.PUT_LINE('=========================================================='); + DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP COMPLETED'); + DBMS_OUTPUT.PUT_LINE('=========================================================='); +END; +/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/rollback_mars835.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/rollback_mars835.sql index a3d5c5c..7d85b79 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/rollback_mars835.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/rollback_mars835.sql @@ -59,7 +59,13 @@ PROMPT ========================================================================= PROMPT PROMPT ========================================================================= -PROMPT Step 3: Verify Rollback Completed +PROMPT Step 3: Delete File Registration Records from A_SOURCE_FILE_RECEIVED +PROMPT ========================================================================= +@@90_MARS_835_rollback_file_registrations.sql + +PROMPT +PROMPT ========================================================================= +PROMPT Step 4: Verify Rollback Completed PROMPT ========================================================================= @@99_MARS_835_verify_rollback.sql