Enhance MARS-835 rollback process with new scripts for file registration deletion and manual cleanup, and add process tracking identifier for exports
This commit is contained in:
@@ -107,7 +107,8 @@ BEGIN
|
|||||||
pParallelDegree => 16,
|
pParallelDegree => 16,
|
||||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
|
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
|
||||||
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
||||||
pRegisterExport => TRUE -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
||||||
|
pProcessName => 'MARS-835' -- Process identifier for tracking
|
||||||
);
|
);
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
|
||||||
@@ -227,7 +228,8 @@ BEGIN
|
|||||||
pParallelDegree => 16,
|
pParallelDegree => 16,
|
||||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
|
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
|
||||||
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
||||||
pRegisterExport => TRUE -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
||||||
|
pProcessName => 'MARS-835' -- Process identifier for tracking
|
||||||
);
|
);
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
--=============================================================================================================================
|
||||||
|
-- MARS-835 ROLLBACK: Delete File Registration Records
|
||||||
|
--=============================================================================================================================
|
||||||
|
-- Purpose: Delete all file registration records from A_SOURCE_FILE_RECEIVED table for MARS-835 process
|
||||||
|
-- Author: Grzegorz Michalski
|
||||||
|
-- Date: 2026-02-13
|
||||||
|
-- Related: MARS-835 - CSDB Data Export Rollback
|
||||||
|
--=============================================================================================================================
|
||||||
|
|
||||||
|
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||||
|
SET TIMING ON
|
||||||
|
|
||||||
|
PROMPT ========================================================================
|
||||||
|
PROMPT ROLLBACK: Deleting file registration records from A_SOURCE_FILE_RECEIVED
|
||||||
|
PROMPT ========================================================================
|
||||||
|
|
||||||
|
DECLARE
|
||||||
|
vRowCount NUMBER := 0;
|
||||||
|
vStartTime TIMESTAMP := SYSTIMESTAMP;
|
||||||
|
vEndTime TIMESTAMP;
|
||||||
|
vElapsedSeconds NUMBER;
|
||||||
|
BEGIN
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Deleting all MARS-835 file registrations from A_SOURCE_FILE_RECEIVED...');
|
||||||
|
|
||||||
|
-- Delete all records for MARS-835 process
|
||||||
|
DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
WHERE PROCESS_NAME = 'MARS-835';
|
||||||
|
|
||||||
|
vRowCount := SQL%ROWCOUNT;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
vEndTime := SYSTIMESTAMP;
|
||||||
|
vElapsedSeconds := EXTRACT(SECOND FROM (vEndTime - vStartTime)) +
|
||||||
|
EXTRACT(MINUTE FROM (vEndTime - vStartTime)) * 60 +
|
||||||
|
EXTRACT(HOUR FROM (vEndTime - vStartTime)) * 3600;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: File registration records deleted');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Records deleted: ' || vRowCount);
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Elapsed time: ' || ROUND(vElapsedSeconds, 2) || ' seconds');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
ROLLBACK;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('ERROR: Failed to delete file registration records');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Error message: ' || SQLERRM);
|
||||||
|
RAISE;
|
||||||
|
END;
|
||||||
|
/
|
||||||
|
|
||||||
|
--=============================================================================================================================
|
||||||
|
-- End of Script
|
||||||
|
--=============================================================================================================================
|
||||||
@@ -22,25 +22,24 @@ DECLARE
|
|||||||
vDataBucketUri VARCHAR2(500);
|
vDataBucketUri VARCHAR2(500);
|
||||||
vHistBucketUri VARCHAR2(500);
|
vHistBucketUri VARCHAR2(500);
|
||||||
vCredentialName VARCHAR2(100);
|
vCredentialName VARCHAR2(100);
|
||||||
|
vFileCount NUMBER := 0;
|
||||||
BEGIN
|
BEGIN
|
||||||
-- Get bucket URIs and credential
|
-- Get bucket URIs and credential
|
||||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT files from DATA bucket...');
|
DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...');
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
|
||||||
|
|
||||||
-- Delete CSV files from DATA bucket (only files matching export pattern)
|
-- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS
|
||||||
-- Pattern matches: LEGACY_DEBT_YYYYMM.csv OR LEGACY_DEBT_YYYYMM_1_20260122T...Z.csv (Oracle timestamp)
|
|
||||||
FOR rec IN (
|
FOR rec IN (
|
||||||
SELECT object_name
|
SELECT object_name
|
||||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
credential_name => vCredentialName,
|
credential_name => vCredentialName,
|
||||||
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
|
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
|
||||||
))
|
))
|
||||||
WHERE object_name LIKE 'LEGACY_DEBT_%'
|
WHERE object_name LIKE 'LEGACY_DEBT%'
|
||||||
AND object_name LIKE '%.csv'
|
|
||||||
AND REGEXP_LIKE(object_name, '^LEGACY_DEBT_[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.csv$') -- YYYYMM or YYYYMM_1_timestamp
|
|
||||||
) LOOP
|
) LOOP
|
||||||
BEGIN
|
BEGIN
|
||||||
DBMS_CLOUD.DELETE_OBJECT(
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
@@ -48,6 +47,7 @@ BEGIN
|
|||||||
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
|
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
|
||||||
);
|
);
|
||||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
EXCEPTION
|
EXCEPTION
|
||||||
WHEN OTHERS THEN
|
WHEN OTHERS THEN
|
||||||
IF SQLCODE = -20404 THEN
|
IF SQLCODE = -20404 THEN
|
||||||
@@ -58,19 +58,20 @@ BEGIN
|
|||||||
END;
|
END;
|
||||||
END LOOP;
|
END LOOP;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT files from HIST bucket...');
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
|
||||||
|
|
||||||
-- Delete Parquet files from HIST bucket (only files matching export pattern)
|
DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...');
|
||||||
-- Pattern matches: YYYYMM.parquet OR YYYYMM_1_20260122T...Z.parquet (Oracle timestamp)
|
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
|
||||||
|
vFileCount := 0;
|
||||||
|
|
||||||
|
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
|
||||||
FOR rec IN (
|
FOR rec IN (
|
||||||
SELECT object_name
|
SELECT object_name
|
||||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
credential_name => vCredentialName,
|
credential_name => vCredentialName,
|
||||||
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/'
|
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/'
|
||||||
))
|
))
|
||||||
WHERE object_name LIKE '%PARTITION_YEAR=%' -- Hive-style partitioning folders
|
WHERE object_name NOT LIKE '%/' -- Exclude directories
|
||||||
AND object_name LIKE '%.parquet'
|
|
||||||
AND REGEXP_LIKE(object_name, '[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.parquet$') -- YYYYMM or YYYYMM_1_timestamp
|
|
||||||
) LOOP
|
) LOOP
|
||||||
BEGIN
|
BEGIN
|
||||||
DBMS_CLOUD.DELETE_OBJECT(
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
@@ -78,6 +79,7 @@ BEGIN
|
|||||||
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name
|
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name
|
||||||
);
|
);
|
||||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
EXCEPTION
|
EXCEPTION
|
||||||
WHEN OTHERS THEN
|
WHEN OTHERS THEN
|
||||||
IF SQLCODE = -20404 THEN
|
IF SQLCODE = -20404 THEN
|
||||||
@@ -88,7 +90,11 @@ BEGIN
|
|||||||
END;
|
END;
|
||||||
END LOOP;
|
END LOOP;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT files deleted');
|
IF vFileCount = 0 THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' INFO: No DEBT Parquet files found to delete');
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT Parquet files deleted from ARCHIVE bucket (' || vFileCount || ' file(s))');
|
||||||
END;
|
END;
|
||||||
/
|
/
|
||||||
|
|
||||||
@@ -104,25 +110,24 @@ DECLARE
|
|||||||
vDataBucketUri VARCHAR2(500);
|
vDataBucketUri VARCHAR2(500);
|
||||||
vHistBucketUri VARCHAR2(500);
|
vHistBucketUri VARCHAR2(500);
|
||||||
vCredentialName VARCHAR2(100);
|
vCredentialName VARCHAR2(100);
|
||||||
|
vFileCount NUMBER := 0;
|
||||||
BEGIN
|
BEGIN
|
||||||
-- Get bucket URIs and credential
|
-- Get bucket URIs and credential
|
||||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY files from DATA bucket...');
|
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...');
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
|
||||||
|
|
||||||
-- Delete CSV files from DATA bucket (only files matching export pattern)
|
-- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS
|
||||||
-- Pattern matches: LEGACY_DEBT_DAILY_YYYYMM.csv OR LEGACY_DEBT_DAILY_YYYYMM_1_timestamp.csv
|
|
||||||
FOR rec IN (
|
FOR rec IN (
|
||||||
SELECT object_name
|
SELECT object_name
|
||||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
credential_name => vCredentialName,
|
credential_name => vCredentialName,
|
||||||
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
|
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
|
||||||
))
|
))
|
||||||
WHERE object_name LIKE 'LEGACY_DEBT_DAILY_%'
|
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
|
||||||
AND object_name LIKE '%.csv'
|
|
||||||
AND REGEXP_LIKE(object_name, '^LEGACY_DEBT_DAILY_[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.csv$') -- YYYYMM or YYYYMM_1_timestamp
|
|
||||||
) LOOP
|
) LOOP
|
||||||
BEGIN
|
BEGIN
|
||||||
DBMS_CLOUD.DELETE_OBJECT(
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
@@ -130,6 +135,7 @@ BEGIN
|
|||||||
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||||
);
|
);
|
||||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
EXCEPTION
|
EXCEPTION
|
||||||
WHEN OTHERS THEN
|
WHEN OTHERS THEN
|
||||||
IF SQLCODE = -20404 THEN
|
IF SQLCODE = -20404 THEN
|
||||||
@@ -140,19 +146,20 @@ BEGIN
|
|||||||
END;
|
END;
|
||||||
END LOOP;
|
END LOOP;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY files from HIST bucket...');
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
|
||||||
|
|
||||||
-- Delete Parquet files from HIST bucket (only files matching export pattern)
|
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
|
||||||
-- Pattern matches: YYYYMM.parquet OR YYYYMM_1_timestamp.parquet
|
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
|
||||||
|
vFileCount := 0;
|
||||||
|
|
||||||
|
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
|
||||||
FOR rec IN (
|
FOR rec IN (
|
||||||
SELECT object_name
|
SELECT object_name
|
||||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
credential_name => vCredentialName,
|
credential_name => vCredentialName,
|
||||||
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/'
|
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/'
|
||||||
))
|
))
|
||||||
WHERE object_name LIKE '%PARTITION_YEAR=%' -- Hive-style partitioning folders
|
WHERE object_name NOT LIKE '%/' -- Exclude directories
|
||||||
AND object_name LIKE '%.parquet'
|
|
||||||
AND REGEXP_LIKE(object_name, '[0-9]{6}(_[0-9]+_[0-9]{8}T[0-9]{6,}Z)?\.parquet$') -- YYYYMM or YYYYMM_1_timestamp
|
|
||||||
) LOOP
|
) LOOP
|
||||||
BEGIN
|
BEGIN
|
||||||
DBMS_CLOUD.DELETE_OBJECT(
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
@@ -160,6 +167,7 @@ BEGIN
|
|||||||
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||||
);
|
);
|
||||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
EXCEPTION
|
EXCEPTION
|
||||||
WHEN OTHERS THEN
|
WHEN OTHERS THEN
|
||||||
IF SQLCODE = -20404 THEN
|
IF SQLCODE = -20404 THEN
|
||||||
@@ -170,7 +178,11 @@ BEGIN
|
|||||||
END;
|
END;
|
||||||
END LOOP;
|
END LOOP;
|
||||||
|
|
||||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY files deleted');
|
IF vFileCount = 0 THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' INFO: No DEBT_DAILY Parquet files found to delete');
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY Parquet files deleted from ARCHIVE bucket (' || vFileCount || ' file(s))');
|
||||||
END;
|
END;
|
||||||
/
|
/
|
||||||
|
|
||||||
|
|||||||
80
MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql
Normal file
80
MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||||
|
SET DEFINE OFF
|
||||||
|
|
||||||
|
DECLARE
|
||||||
|
vCredential VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL';
|
||||||
|
vDataBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/data/o/';
|
||||||
|
vArchiveBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/history/o/';
|
||||||
|
vCount NUMBER := 0;
|
||||||
|
BEGIN
|
||||||
|
DBMS_OUTPUT.PUT_LINE('=== Checking CSV files in DATA bucket ===');
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredential,
|
||||||
|
location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/'
|
||||||
|
))
|
||||||
|
WHERE object_name LIKE 'LEGACY_DEBT%'
|
||||||
|
) LOOP
|
||||||
|
vCount := vCount + 1;
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name);
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT: ' || vCount);
|
||||||
|
|
||||||
|
vCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredential,
|
||||||
|
location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT_DAILY/'
|
||||||
|
))
|
||||||
|
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
|
||||||
|
) LOOP
|
||||||
|
vCount := vCount + 1;
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name);
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT_DAILY: ' || vCount);
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Checking Parquet files in ARCHIVE bucket ===');
|
||||||
|
vCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredential,
|
||||||
|
location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT/'
|
||||||
|
))
|
||||||
|
WHERE ROWNUM <= 5
|
||||||
|
) LOOP
|
||||||
|
vCount := vCount + 1;
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name);
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT (first 5): ' || vCount);
|
||||||
|
|
||||||
|
vCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredential,
|
||||||
|
location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/'
|
||||||
|
))
|
||||||
|
) LOOP
|
||||||
|
vCount := vCount + 1;
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name);
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT_DAILY: ' || vCount);
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Now testing DELETE_OBJECT ===');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Testing delete for: ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv');
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredential,
|
||||||
|
object_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv'
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE('SUCCESS: File deleted');
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE('ERROR: ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END;
|
||||||
|
/
|
||||||
126
MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql
Normal file
126
MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
--=============================================================================================================================
|
||||||
|
-- MARS-835 Manual Cleanup - Delete remaining files after rollback
|
||||||
|
--=============================================================================================================================
|
||||||
|
|
||||||
|
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||||
|
SET DEFINE OFF
|
||||||
|
|
||||||
|
DECLARE
|
||||||
|
vDataBucketUri VARCHAR2(500);
|
||||||
|
vArchiveBucketUri VARCHAR2(500);
|
||||||
|
vCredentialName VARCHAR2(100);
|
||||||
|
vFileCount NUMBER := 0;
|
||||||
|
BEGIN
|
||||||
|
-- Get bucket URIs and credential
|
||||||
|
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||||
|
vArchiveBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||||
|
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('MARS-835 Manual Cleanup');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
|
||||||
|
-- Delete DEBT CSV files from DATA bucket
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '1. Deleting DEBT CSV files from DATA bucket...');
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
|
||||||
|
))
|
||||||
|
WHERE object_name LIKE 'LEGACY_DEBT%'
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
-- Delete DEBT_DAILY CSV files from DATA bucket
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '2. Deleting DEBT_DAILY CSV files from DATA bucket...');
|
||||||
|
vFileCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
|
||||||
|
))
|
||||||
|
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
-- Delete DEBT Parquet files from ARCHIVE bucket
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '3. Deleting DEBT Parquet files from ARCHIVE bucket...');
|
||||||
|
vFileCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/'
|
||||||
|
))
|
||||||
|
WHERE object_name NOT LIKE '%/'
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
-- Delete DEBT_DAILY Parquet files from ARCHIVE bucket
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '4. Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
|
||||||
|
vFileCount := 0;
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/'
|
||||||
|
))
|
||||||
|
WHERE object_name NOT LIKE '%/'
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE(CHR(10) || '========================================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Manual cleanup completed');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('========================================================================');
|
||||||
|
END;
|
||||||
|
/
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
-- MARS-835: Manual cleanup of Parquet files only (after bugfix)
|
||||||
|
-- Description: Removes orphaned Parquet files from ARCHIVE bucket
|
||||||
|
-- Usage: Execute as CT_MRDS user
|
||||||
|
|
||||||
|
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||||
|
|
||||||
|
DECLARE
|
||||||
|
vCredentialName VARCHAR2(100) := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||||
|
vHistBucketUri VARCHAR2(200) := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||||
|
vFileCount NUMBER := 0;
|
||||||
|
vStartTime TIMESTAMP := SYSTIMESTAMP;
|
||||||
|
vElapsedTime INTERVAL DAY TO SECOND;
|
||||||
|
BEGIN
|
||||||
|
DBMS_OUTPUT.PUT_LINE('==========================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP: Parquet files only');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('==========================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Start Time: ' || TO_CHAR(vStartTime, 'YYYY-MM-DD HH24:MI:SS'));
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Credential: ' || vCredentialName);
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Archive Bucket: ' || vHistBucketUri);
|
||||||
|
DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------');
|
||||||
|
|
||||||
|
-- Delete CSDB_DEBT Parquet files
|
||||||
|
DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT Parquet files...');
|
||||||
|
vFileCount := 0;
|
||||||
|
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/'
|
||||||
|
))
|
||||||
|
WHERE object_name NOT LIKE '%/' -- Exclude directories
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
IF SQLCODE = -20404 THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||||
|
ELSE
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name);
|
||||||
|
END IF;
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('CSDB_DEBT Parquet files deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
-- Delete CSDB_DEBT_DAILY Parquet files
|
||||||
|
DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT_DAILY Parquet files...');
|
||||||
|
vFileCount := 0;
|
||||||
|
|
||||||
|
FOR rec IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/'
|
||||||
|
))
|
||||||
|
WHERE object_name NOT LIKE '%/' -- Exclude directories
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => vCredentialName,
|
||||||
|
object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||||
|
);
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||||
|
vFileCount := vFileCount + 1;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
IF SQLCODE = -20404 THEN
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||||
|
ELSE
|
||||||
|
DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name);
|
||||||
|
END IF;
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
DBMS_OUTPUT.PUT_LINE('CSDB_DEBT_DAILY Parquet files deleted: ' || vFileCount);
|
||||||
|
|
||||||
|
-- Final summary
|
||||||
|
vElapsedTime := SYSTIMESTAMP - vStartTime;
|
||||||
|
DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('End Time: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
|
||||||
|
DBMS_OUTPUT.PUT_LINE('Elapsed Time: ' || vElapsedTime);
|
||||||
|
DBMS_OUTPUT.PUT_LINE('==========================================================');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP COMPLETED');
|
||||||
|
DBMS_OUTPUT.PUT_LINE('==========================================================');
|
||||||
|
END;
|
||||||
|
/
|
||||||
@@ -59,7 +59,13 @@ PROMPT =========================================================================
|
|||||||
|
|
||||||
PROMPT
|
PROMPT
|
||||||
PROMPT =========================================================================
|
PROMPT =========================================================================
|
||||||
PROMPT Step 3: Verify Rollback Completed
|
PROMPT Step 3: Delete File Registration Records from A_SOURCE_FILE_RECEIVED
|
||||||
|
PROMPT =========================================================================
|
||||||
|
@@90_MARS_835_rollback_file_registrations.sql
|
||||||
|
|
||||||
|
PROMPT
|
||||||
|
PROMPT =========================================================================
|
||||||
|
PROMPT Step 4: Verify Rollback Completed
|
||||||
PROMPT =========================================================================
|
PROMPT =========================================================================
|
||||||
@@99_MARS_835_verify_rollback.sql
|
@@99_MARS_835_verify_rollback.sql
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user