diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql index f348849..33927a2 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/01_MARS_835_install_step1.sql @@ -103,7 +103,7 @@ BEGIN pBucketArea => 'DATA', pFolderName => 'ODS/CSDB/CSDB_DEBT', pMinDate => &cutoff_date, - pMaxDate => SYSDATE, + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT', pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) @@ -224,7 +224,7 @@ BEGIN pBucketArea => 'DATA', pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY', pMinDate => &cutoff_date, - pMaxDate => SYSDATE, + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 16, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY', pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036) diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/02_MARS_835_install_step2.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/02_MARS_835_install_step2.sql index 3db3bdf..aa035c2 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/02_MARS_835_install_step2.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-835/02_MARS_835_install_step2.sql @@ -33,7 +33,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_INSTR_RAT_FULL', - pMaxDate => SYSDATE, + pMinDate => DATE '1900-01-01', -- Explicit start date for clarity + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 8, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_INSTR_RAT_FULL' ); @@ -60,7 +61,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_INSTR_DESC_FULL', - pMaxDate => SYSDATE, + pMinDate => DATE '1900-01-01', -- Explicit start date for clarity + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 8, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_INSTR_DESC_FULL' ); @@ -87,7 +89,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_ISSUER_RAT_FULL', - pMaxDate => SYSDATE, + pMinDate => DATE '1900-01-01', -- Explicit start date for clarity + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 8, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_ISSUER_RAT_FULL' ); @@ -114,7 +117,8 @@ BEGIN pKeyColumnName => 'A_ETL_LOAD_SET_FK', pBucketArea => 'ARCHIVE', pFolderName => 'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL', - pMaxDate => SYSDATE, + pMinDate => DATE '1900-01-01', -- Explicit start date for clarity + pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE) pParallelDegree => 8, pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_ISSUER_DESC_FULL' ); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/README.md b/MARS_Packages/REL01_ADDITIONS/MARS-835/README.md deleted file mode 100644 index 9347e91..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# MARS-835: One-Time CSDB Data Export from Operational Database to External Tables - -## Overview -This package performs a one-time bulk export of CSDB data from operational database tables (OU_CSDB schema) to new external tables in OCI buckets. The export uses DATA_EXPORTER v2.4.0 with per-column date format handling to move historical data to either DATA bucket (CSV format) or HIST bucket (Parquet format with Hive-style partitioning). - -**Migration Strategy:** -- **Split Export (2 tables)**: DEBT, DEBT_DAILY - Last 6 months → DATA (CSV), Older data → HIST (Parquet) -- **HIST Only (4 tables)**: INSTR_RAT_FULL, INSTR_DESC_FULL, ISSUER_RAT_FULL, ISSUER_DESC_FULL - All data → HIST (Parquet) - -**Key Transformations:** -- Column rename: `A_ETL_LOAD_SET_FK` → `A_WORKFLOW_HISTORY_KEY` (all tables) -- Column removal: DEBT (2 columns), DEBT_DAILY (6 columns) not required in new structure - -## Contents -- `install_mars835.sql` - Master installation script with SPOOL logging -- `rollback_mars835.sql` - Master rollback script -- `01_MARS_835_*.sql` - Individual installation scripts -- `91_MARS_835_*.sql` - Individual rollback scripts -- `track_package_versions.sql` - Package version tracking -- `verify_packages_version.sql` - Package verification - -## Prerequisites -- Oracle Database 23ai -- ADMIN user access (required for all MARS installations) -- ENV_MANAGER v3.1.0+ -- Required schema privileges - -## Installation - -### Option 1: Master Script (Recommended) -```powershell -# IMPORTANT: Execute as ADMIN user for proper privilege management -Get-Content "MARS_Packages/REL01_POST_DEACTIVATION/MARS-835/install_mars835.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high" - -# Log file created: log/INSTALL_MARS_835__.log -``` - -### Option 2: Individual Scripts -```powershell -# IMPORTANT: Execute as ADMIN user -Get-Content "01_MARS_835_*.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high" -Get-Content "02_MARS_835_*.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high" -# ... etc -``` - -## Verification -```sql --- Verify package versions -SELECT PACKAGE_NAME.GET_VERSION() FROM DUAL; - --- Check for errors (ADMIN user checks specific schema) -SELECT * FROM ALL_ERRORS -WHERE OWNER = 'CT_MRDS' -- Replace with target schema - AND NAME = 'PACKAGE_NAME'; - --- Verify no untracked changes -SELECT ENV_MANAGER.CHECK_PACKAGE_CHANGES('CT_MRDS', 'PACKAGE_NAME') FROM DUAL; -``` - -## Rollback -```powershell -# IMPORTANT: Execute as ADMIN user -Get-Content "MARS_Packages/REL01_POST_DEACTIVATION/MARS-835/rollback_mars835.sql" | sql "ADMIN/Cloudpass#34@ggmichalski_high" - -**NOTE**: Rollback for data exports is **NOT RECOMMENDED** as it would delete exported files from OCI buckets. Only use rollback if export failed and needs to be restarted. -``` - -## Expected Changes - -### Data Export Summary -**6 CSDB tables exported from OU_CSDB schema:** - -**Group 1: Split DATA + HIST (Time Critical)** -1. **DEBT** - Last 6 months → DATA, Older → HIST -2. **DEBT_DAILY** - Last 6 months → DATA, Older → HIST - -**Group 2: HIST Only (Weekend Bulk)** -3. **INSTR_RAT_FULL** - All data → HIST -4. **INSTR_DESC_FULL** - All data → HIST -5. **ISSUER_RAT_FULL** - All data → HIST -6. **ISSUER_DESC_FULL** - All data → HIST - -### Bucket Destinations (DEV environment) -- **DATA Bucket**: `mrds_data_dev/ODS/CSDB/` (CSV format) -- **HIST Bucket**: `mrds_hist_dev/ARCHIVE/CSDB/` (Parquet with partitioning) - -### Column Mappings -- **All tables**: `A_ETL_LOAD_SET_FK` renamed to `A_WORKFLOW_HISTORY_KEY` -- **DEBT**: Removed columns: `IDIRDEPOSITORY`, `VA_BONDDURATION` -- **DEBT_DAILY**: Removed columns: `STEPID`, `PROGRAMNAME`, `PROGRAMCEILING`, `PROGRAMSTATUS`, `ISSUERNACE21SECTOR`, `INSTRUMENTQUOTATIONBASIS` - -## Testing - -### Post-Export Verification - -1. **Verify CSV files in DATA bucket** (DEBT, DEBT_DAILY - last 6 months): -```sql --- Check exported files -SELECT object_name, bytes -FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'DEF_CRED_ARN', - location_uri => 'https://objectstorage.region.oraclecloud.com/n/namespace/b/mrds_data_dev/o/ODS/CSDB/' -)) WHERE object_name LIKE '%CSDB_DEBT%'; -``` - -2. **Verify Parquet files in HIST bucket** (all 6 tables): -```sql --- Check archived files with Hive partitioning -SELECT object_name, bytes -FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => 'DEF_CRED_ARN', - location_uri => 'https://objectstorage.region.oraclecloud.com/n/namespace/b/mrds_hist_dev/o/ARCHIVE/CSDB/' -)) WHERE object_name LIKE '%PARTITION_YEAR=%'; -``` - -3. **Validate row counts match source tables**: -```sql --- Compare counts between source and exported data -SELECT COUNT(*) FROM OU_CSDB.DEBT; -SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS; -- External table pointing to DATA -SELECT COUNT(*) FROM ODS.CSDB_DEBT_ARCHIVE; -- External table pointing to HIST -``` - -4. **Verify column mappings**: -```sql --- Check A_WORKFLOW_HISTORY_KEY exists in exported data -SELECT A_WORKFLOW_HISTORY_KEY, COUNT(*) -FROM ODS.CSDB_DEBT_ARCHIVE -GROUP BY A_WORKFLOW_HISTORY_KEY; -``` - -## Known Issues - -### Timing Constraints -- **DATA exports (DEBT, DEBT_DAILY)**: Must execute during parallel old+new loads phase after Production deployment -- **HIST exports (all 6 tables)**: Can run anytime, recommended for weekend bulk execution to avoid interference - -### Environment-Specific Configuration -- Bucket names must be adjusted for each environment: - - DEV: `mrds_data_dev`, `mrds_hist_dev` - - TEST: `mrds_data_test`, `mrds_hist_test` - - PROD: `mrds_data_prod`, `mrds_hist_prod` - -### Data Cutoff Date -- Export scripts use 6-month cutoff date calculated as `ADD_MONTHS(SYSDATE, -6)` -- Verify cutoff aligns with business requirements before execution - -### One-Time Execution -- This is a **ONE-TIME data migration** package -- After successful execution, package should be **deactivated** (moved to REL01_POST_DEACTIVATION) -- Do not re-run unless explicitly required for data refresh - -## Related -- **JIRA**: MARS-835 - CSDB Data Export to External Tables -- **Confluence**: FILE_MANAGER package - MRDS - Technical Team -- **Confluence**: Table Setup Guide for FILE PROCESSOR System -- **Source Schema**: OU_CSDB (Operational Database) -- **Target Schema**: ODS (External Tables) -- **Migration Type**: One-time bulk export (deactivated post-execution) - ---- - -**Author:** Grzegorz Michalski -**Date:** 2025-12-04 -**Version:** 1.0.0 diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/REQUIRED_EXTERNAL_TABLES.md b/MARS_Packages/REL01_ADDITIONS/MARS-835/REQUIRED_EXTERNAL_TABLES.md deleted file mode 100644 index 0dedb48..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/REQUIRED_EXTERNAL_TABLES.md +++ /dev/null @@ -1,207 +0,0 @@ -# MARS-835: Required External Tables for Smart Column Mapping - -## Overview -This document lists all external tables required for MARS-835 data exports using DATA_EXPORTER v2.4.0 with Smart Column Mapping feature. - -**Purpose**: Smart Column Mapping ensures CSV files are generated with columns in the EXACT order expected by external tables, preventing NULL values due to Oracle's positional CSV mapping. - ---- - -## Required External Tables - -### Group 1: DATA Bucket (CSV Format) - **CRITICAL** - -#### 1. ODS.CSDB_DEBT_DATA_ODS -- **Source Table**: OU_CSDB.LEGACY_DEBT -- **Format**: CSV -- **Bucket**: DATA (mrds_data_dev/ODS/CSDB/CSDB_DEBT/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY (position 2 recommended) -- **Critical**: Must use Smart Column Mapping to avoid NULL values in A_WORKFLOW_HISTORY_KEY - -#### 2. ODS.CSDB_DEBT_DAILY_DATA_ODS -- **Source Table**: OU_CSDB.LEGACY_DEBT_DAILY -- **Format**: CSV -- **Bucket**: DATA (mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY (position 2 recommended) -- **Critical**: Must use Smart Column Mapping to avoid NULL values in A_WORKFLOW_HISTORY_KEY - ---- - -### Group 2: ARCHIVE Bucket (Parquet Format) - **RECOMMENDED** - -#### 3. ODS.CSDB_DEBT_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_DEBT -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY -- **Note**: Parquet uses schema-based mapping (column order less critical but Smart Column Mapping ensures consistency) - -#### 4. ODS.CSDB_DEBT_DAILY_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_DEBT_DAILY -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY - -#### 5. ODS.CSDB_INSTR_RAT_FULL_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_INSTR_RAT_FULL -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_INSTR_RAT_FULL/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY - -#### 6. ODS.CSDB_INSTR_DESC_FULL_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_INSTR_DESC_FULL -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_INSTR_DESC_FULL/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY - -#### 7. ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_ISSUER_RAT_FULL -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_ISSUER_RAT_FULL/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY - -#### 8. ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE -- **Source Table**: OU_CSDB.LEGACY_ISSUER_DESC_FULL -- **Format**: Parquet with Hive partitioning -- **Bucket**: ARCHIVE (mrds_hist_dev/ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/) -- **Key Column Mapping**: A_ETL_LOAD_SET_FK → A_WORKFLOW_HISTORY_KEY - ---- - -## External Table Column Order Requirements - -### **CRITICAL for CSV Tables** (DATA bucket): - -All CSV external tables MUST have **A_WORKFLOW_HISTORY_KEY at position 2**: - -``` -Position 1: A_KEY (NUMBER) -Position 2: A_WORKFLOW_HISTORY_KEY (NUMBER) ← MUST BE HERE! -Position 3+: Other columns in any order -``` - -**Reason**: Oracle External Tables with CSV format use **positional mapping** (ignore header row). If source table has A_ETL_LOAD_SET_FK at position 72, but CSV puts it at position 72 while external table expects A_WORKFLOW_HISTORY_KEY at position 2, the external table will try to read position 2 (which might be a DATE column) as NUMBER → conversion fails → NULL value. - -**Solution**: Smart Column Mapping (v2.4.0) generates CSV columns in EXTERNAL TABLE order, ensuring position 2 has the correct NUMBER value. - -### **OPTIONAL for Parquet Tables** (ARCHIVE bucket): - -Parquet format uses **schema-based mapping** (column names). Column order doesn't matter, but Smart Column Mapping provides consistency. - ---- - -## Creation Script Example - -### CSV External Table (CRITICAL - Correct Column Order) - -```sql --- Example: ODS.CSDB_DEBT_DATA_ODS --- IMPORTANT: A_WORKFLOW_HISTORY_KEY must be at position 2! - -BEGIN - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => 'CSDB_DEBT_DATA_ODS', - pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_TEMPLATE', - pPrefix => 'ODS/CSDB/CSDB_DEBT', - pBucketUri => CT_MRDS.ENV_MANAGER.gvDataBucketUri, - pFormat => 'CSV' -- Uses positional mapping! - ); -END; -/ - --- Verify column order (A_WORKFLOW_HISTORY_KEY should be position 2) -SELECT column_id, column_name, data_type -FROM all_tab_columns -WHERE table_name = 'CSDB_DEBT_DATA_ODS' - AND owner = 'ODS' -ORDER BY column_id; -``` - -### Parquet External Table (Optional Column Order) - -```sql --- Example: ODS.CSDB_DEBT_ARCHIVE --- Column order flexible (schema-based mapping) - -BEGIN - ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE( - pTableName => 'CSDB_DEBT_ARCHIVE', - pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_TEMPLATE', - pPrefix => 'ARCHIVE/CSDB/CSDB_DEBT', - pBucketUri => CT_MRDS.ENV_MANAGER.gvArchiveBucketUri, - pFormat => 'PARQUET' -- Uses schema-based mapping - ); -END; -/ -``` - ---- - -## Template Tables Required - -All external tables require corresponding template tables in CT_ET_TEMPLATES schema: - -- `CT_ET_TEMPLATES.CSDB_DEBT_TEMPLATE` -- `CT_ET_TEMPLATES.CSDB_DEBT_DAILY_TEMPLATE` -- `CT_ET_TEMPLATES.CSDB_INSTR_RAT_FULL_TEMPLATE` -- `CT_ET_TEMPLATES.CSDB_INSTR_DESC_FULL_TEMPLATE` -- `CT_ET_TEMPLATES.CSDB_ISSUER_RAT_FULL_TEMPLATE` -- `CT_ET_TEMPLATES.CSDB_ISSUER_DESC_FULL_TEMPLATE` - -**Note**: Template tables must be created by ADMIN or CT_ET_TEMPLATES user (MRDS_LOADER cannot create them). - ---- - -## Verification Checklist - -Before running MARS-835 exports: - -- [ ] All 8 external tables exist in ODS schema -- [ ] CSV tables (DATA bucket) have A_WORKFLOW_HISTORY_KEY at position 2 -- [ ] Template tables exist in CT_ET_TEMPLATES schema -- [ ] MRDS_LOADER has EXECUTE privilege on ODS.FILE_MANAGER_ODS -- [ ] ODS schema has access to CT_MRDS.ENV_MANAGER for logging -- [ ] DATA_EXPORTER v2.4.0 deployed with Smart Column Mapping feature - ---- - -## Testing Verification - -After export, verify A_WORKFLOW_HISTORY_KEY is not NULL: - -```sql --- CSV tables (should be 100% populated) -SELECT 'CSDB_DEBT_DATA_ODS' AS TABLE_NAME, - COUNT(*) AS TOTAL_ROWS, - COUNT(A_WORKFLOW_HISTORY_KEY) AS NON_NULL_COUNT, - ROUND(COUNT(A_WORKFLOW_HISTORY_KEY) * 100.0 / NULLIF(COUNT(*), 0), 2) AS SUCCESS_RATE_PCT -FROM ODS.CSDB_DEBT_DATA_ODS; - -SELECT 'CSDB_DEBT_DAILY_DATA_ODS' AS TABLE_NAME, - COUNT(*) AS TOTAL_ROWS, - COUNT(A_WORKFLOW_HISTORY_KEY) AS NON_NULL_COUNT, - ROUND(COUNT(A_WORKFLOW_HISTORY_KEY) * 100.0 / NULLIF(COUNT(*), 0), 2) AS SUCCESS_RATE_PCT -FROM ODS.CSDB_DEBT_DAILY_DATA_ODS; - --- Parquet tables (should also be 100% populated) -SELECT 'CSDB_DEBT_ARCHIVE' AS TABLE_NAME, - COUNT(*) AS TOTAL_ROWS, - COUNT(A_WORKFLOW_HISTORY_KEY) AS NON_NULL_COUNT, - ROUND(COUNT(A_WORKFLOW_HISTORY_KEY) * 100.0 / NULLIF(COUNT(*), 0), 2) AS SUCCESS_RATE_PCT -FROM ODS.CSDB_DEBT_ARCHIVE; -``` - -**Expected Result**: SUCCESS_RATE_PCT = 100.00 for all tables - ---- - -## Related Documentation - -- [DATA_EXPORTER v2.4.0 Smart Column Mapping Examples](../MARS-835-PREHOOK/current_version/v2.3.0/DATA_EXPORTER_v2.4.0_Smart_Column_Mapping_Examples.sql) -- [Oracle External Tables Column Order Issue](../../confluence/additions/Oracle_External_Tables_Column_Order_Issue.md) -- [MARS-835 README](README.md) - ---- - -**Last Updated**: 2026-01-09 -**Author**: GitHub Copilot (MARS-835 Update) diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql deleted file mode 100644 index d51f9cc..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/check_files.sql +++ /dev/null @@ -1,80 +0,0 @@ -SET SERVEROUTPUT ON SIZE UNLIMITED -SET DEFINE OFF - -DECLARE - vCredential VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL'; - vDataBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/data/o/'; - vArchiveBucket VARCHAR2(200) := 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frtgjxu7zl7c/b/history/o/'; - vCount NUMBER := 0; -BEGIN - DBMS_OUTPUT.PUT_LINE('=== Checking CSV files in DATA bucket ==='); - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredential, - location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/' - )) - WHERE object_name LIKE 'LEGACY_DEBT%' - ) LOOP - vCount := vCount + 1; - DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT: ' || vCount); - - vCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredential, - location_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT_DAILY/' - )) - WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' - ) LOOP - vCount := vCount + 1; - DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total CSV files DEBT_DAILY: ' || vCount); - - DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Checking Parquet files in ARCHIVE bucket ==='); - vCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredential, - location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT/' - )) - WHERE ROWNUM <= 5 - ) LOOP - vCount := vCount + 1; - DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT (first 5): ' || vCount); - - vCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredential, - location_uri => vArchiveBucket || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' - )) - ) LOOP - vCount := vCount + 1; - DBMS_OUTPUT.PUT_LINE(' [' || vCount || '] ' || rec.object_name); - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total Parquet files DEBT_DAILY: ' || vCount); - - DBMS_OUTPUT.PUT_LINE(CHR(10) || '=== Now testing DELETE_OBJECT ==='); - DBMS_OUTPUT.PUT_LINE('Testing delete for: ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv'); - - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredential, - object_uri => vDataBucket || 'ODS/CSDB/CSDB_DEBT/LEGACY_DEBT_202510_1_20260213T092239041072Z.csv' - ); - DBMS_OUTPUT.PUT_LINE('SUCCESS: File deleted'); - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('ERROR: ' || SQLERRM); - END; -END; -/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql deleted file mode 100644 index 33a5790..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup.sql +++ /dev/null @@ -1,126 +0,0 @@ ---============================================================================================================================= --- MARS-835 Manual Cleanup - Delete remaining files after rollback ---============================================================================================================================= - -SET SERVEROUTPUT ON SIZE UNLIMITED -SET DEFINE OFF - -DECLARE - vDataBucketUri VARCHAR2(500); - vArchiveBucketUri VARCHAR2(500); - vCredentialName VARCHAR2(100); - vFileCount NUMBER := 0; -BEGIN - -- Get bucket URIs and credential - vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA'); - vArchiveBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); - vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName; - - DBMS_OUTPUT.PUT_LINE('========================================================================'); - DBMS_OUTPUT.PUT_LINE('MARS-835 Manual Cleanup'); - DBMS_OUTPUT.PUT_LINE('========================================================================'); - - -- Delete DEBT CSV files from DATA bucket - DBMS_OUTPUT.PUT_LINE(CHR(10) || '1. Deleting DEBT CSV files from DATA bucket...'); - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' - )) - WHERE object_name LIKE 'LEGACY_DEBT%' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); - END; - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); - - -- Delete DEBT_DAILY CSV files from DATA bucket - DBMS_OUTPUT.PUT_LINE(CHR(10) || '2. Deleting DEBT_DAILY CSV files from DATA bucket...'); - vFileCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' - )) - WHERE object_name LIKE 'LEGACY_DEBT_DAILY%' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); - END; - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); - - -- Delete DEBT Parquet files from ARCHIVE bucket - DBMS_OUTPUT.PUT_LINE(CHR(10) || '3. Deleting DEBT Parquet files from ARCHIVE bucket...'); - vFileCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' - )) - WHERE object_name NOT LIKE '%/' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); - END; - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); - - -- Delete DEBT_DAILY Parquet files from ARCHIVE bucket - DBMS_OUTPUT.PUT_LINE(CHR(10) || '4. Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...'); - vFileCount := 0; - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' - )) - WHERE object_name NOT LIKE '%/' - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vArchiveBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || rec.object_name || ' - ' || SQLERRM); - END; - END LOOP; - DBMS_OUTPUT.PUT_LINE('Total deleted: ' || vFileCount); - - DBMS_OUTPUT.PUT_LINE(CHR(10) || '========================================================================'); - DBMS_OUTPUT.PUT_LINE('Manual cleanup completed'); - DBMS_OUTPUT.PUT_LINE('========================================================================'); -END; -/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql deleted file mode 100644 index 1ff5fa8..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/manual_cleanup_parquet_only.sql +++ /dev/null @@ -1,93 +0,0 @@ --- MARS-835: Manual cleanup of Parquet files only (after bugfix) --- Description: Removes orphaned Parquet files from ARCHIVE bucket --- Usage: Execute as CT_MRDS user - -SET SERVEROUTPUT ON SIZE UNLIMITED - -DECLARE - vCredentialName VARCHAR2(100) := CT_MRDS.ENV_MANAGER.gvCredentialName; - vHistBucketUri VARCHAR2(200) := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE'); - vFileCount NUMBER := 0; - vStartTime TIMESTAMP := SYSTIMESTAMP; - vElapsedTime INTERVAL DAY TO SECOND; -BEGIN - DBMS_OUTPUT.PUT_LINE('=========================================================='); - DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP: Parquet files only'); - DBMS_OUTPUT.PUT_LINE('=========================================================='); - DBMS_OUTPUT.PUT_LINE('Start Time: ' || TO_CHAR(vStartTime, 'YYYY-MM-DD HH24:MI:SS')); - DBMS_OUTPUT.PUT_LINE('Credential: ' || vCredentialName); - DBMS_OUTPUT.PUT_LINE('Archive Bucket: ' || vHistBucketUri); - DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------'); - - -- Delete CSDB_DEBT Parquet files - DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT Parquet files...'); - vFileCount := 0; - - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' - )) - WHERE object_name NOT LIKE '%/' -- Exclude directories - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name); - END IF; - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE('CSDB_DEBT Parquet files deleted: ' || vFileCount); - - -- Delete CSDB_DEBT_DAILY Parquet files - DBMS_OUTPUT.PUT_LINE(chr(10) || 'Deleting CSDB_DEBT_DAILY Parquet files...'); - vFileCount := 0; - - FOR rec IN ( - SELECT object_name - FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( - credential_name => vCredentialName, - location_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' - )) - WHERE object_name NOT LIKE '%/' -- Exclude directories - ) LOOP - BEGIN - DBMS_CLOUD.DELETE_OBJECT( - credential_name => vCredentialName, - object_uri => vHistBucketUri || 'ARCHIVE/CSDB/CSDB_DEBT_DAILY/' || rec.object_name - ); - DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name); - vFileCount := vFileCount + 1; - EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -20404 THEN - DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name); - ELSE - DBMS_OUTPUT.PUT_LINE(' ERROR: ' || SQLERRM || ' - ' || rec.object_name); - END IF; - END; - END LOOP; - - DBMS_OUTPUT.PUT_LINE('CSDB_DEBT_DAILY Parquet files deleted: ' || vFileCount); - - -- Final summary - vElapsedTime := SYSTIMESTAMP - vStartTime; - DBMS_OUTPUT.PUT_LINE('----------------------------------------------------------'); - DBMS_OUTPUT.PUT_LINE('End Time: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS')); - DBMS_OUTPUT.PUT_LINE('Elapsed Time: ' || vElapsedTime); - DBMS_OUTPUT.PUT_LINE('=========================================================='); - DBMS_OUTPUT.PUT_LINE('MANUAL CLEANUP COMPLETED'); - DBMS_OUTPUT.PUT_LINE('=========================================================='); -END; -/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/track_package_versions.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/track_package_versions.sql deleted file mode 100644 index 7a8f647..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/track_package_versions.sql +++ /dev/null @@ -1,92 +0,0 @@ --- =================================================================== --- Simple Package Version Tracking Script --- =================================================================== --- Purpose: Track specified Oracle package versions --- Author: Grzegorz Michalski --- Date: 2025-12-04 --- Version: 3.1.0 - List-Based Edition --- --- USAGE: --- 1. Edit package list below (add/remove packages as needed) --- 2. Include in your install/rollback script: @@track_package_versions.sql --- =================================================================== - -SET SERVEROUTPUT ON; - -DECLARE - TYPE t_package_rec IS RECORD ( - owner VARCHAR2(50), - name VARCHAR2(50), - version VARCHAR2(50) - ); - TYPE t_packages IS TABLE OF t_package_rec; - TYPE t_string_array IS TABLE OF VARCHAR2(100); - - -- =================================================================== - -- PACKAGE LIST - Edit this array to specify packages to track - -- =================================================================== - -- Add or remove entries as needed for your MARS issue - -- Format: 'SCHEMA.PACKAGE_NAME' - -- =================================================================== - vPackageList t_string_array := t_string_array( - 'CT_MRDS.FILE_MANAGER', - 'ODS.FILE_MANAGER_ODS' - ); - -- =================================================================== - - vPackages t_packages := t_packages(); - vVersion VARCHAR2(50); - vCount NUMBER := 0; - vOwner VARCHAR2(50); - vPackageName VARCHAR2(50); - vDotPos NUMBER; -BEGIN - DBMS_OUTPUT.PUT_LINE('========================================'); - DBMS_OUTPUT.PUT_LINE('Package Version Tracking'); - DBMS_OUTPUT.PUT_LINE('========================================'); - - -- Process each package in the list - FOR i IN 1..vPackageList.COUNT LOOP - vDotPos := INSTR(vPackageList(i), '.'); - IF vDotPos > 0 THEN - vOwner := SUBSTR(vPackageList(i), 1, vDotPos - 1); - vPackageName := SUBSTR(vPackageList(i), vDotPos + 1); - - BEGIN - EXECUTE IMMEDIATE 'SELECT ' || vPackageList(i) || '.GET_VERSION() FROM DUAL' - INTO vVersion; - - vPackages.EXTEND; - vPackages(vPackages.COUNT).owner := vOwner; - vPackages(vPackages.COUNT).name := vPackageName; - vPackages(vPackages.COUNT).version := vVersion; - - CT_MRDS.ENV_MANAGER.TRACK_PACKAGE_VERSION( - pPackageOwner => vOwner, - pPackageName => vPackageName, - pPackageVersion => vVersion, - pPackageBuildDate => TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'), - pPackageAuthor => 'Grzegorz Michalski' - ); - vCount := vCount + 1; - EXCEPTION - WHEN OTHERS THEN - DBMS_OUTPUT.PUT_LINE('Error tracking ' || vPackageList(i) || ': ' || SQLERRM); - END; - END IF; - END LOOP; - - -- Display results - IF vPackages.COUNT > 0 THEN - DBMS_OUTPUT.PUT_LINE('Packages tracked: ' || vCount || ' of ' || vPackages.COUNT); - FOR i IN 1..vPackages.COUNT LOOP - DBMS_OUTPUT.PUT_LINE(' ' || vPackages(i).owner || '.' || vPackages(i).name || - ' (v' || vPackages(i).version || ')'); - END LOOP; - ELSE - DBMS_OUTPUT.PUT_LINE('No packages found in list'); - END IF; - - DBMS_OUTPUT.PUT_LINE('========================================'); -END; -/ diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-835/verify_packages_version.sql b/MARS_Packages/REL01_ADDITIONS/MARS-835/verify_packages_version.sql deleted file mode 100644 index e9ef47f..0000000 --- a/MARS_Packages/REL01_ADDITIONS/MARS-835/verify_packages_version.sql +++ /dev/null @@ -1,62 +0,0 @@ --- =================================================================== --- Universal Package Version Verification Script --- =================================================================== --- Purpose: Verify all tracked Oracle packages for code changes --- Author: Grzegorz Michalski --- Date: 2025-12-04 --- Version: 1.0.0 --- --- USAGE: --- Include at the end of install/rollback scripts: @@verify_packages_version.sql --- --- OUTPUT: --- - List of all tracked packages with their current status --- - OK: Package has not changed since last tracking --- - WARNING: Package code changed without version update --- =================================================================== - -SET LINESIZE 200 -SET PAGESIZE 1000 -SET FEEDBACK OFF - -PROMPT -PROMPT ======================================== -PROMPT Package Version Verification -PROMPT ======================================== -PROMPT - -COLUMN PACKAGE_OWNER FORMAT A15 -COLUMN PACKAGE_NAME FORMAT A20 -COLUMN VERSION FORMAT A10 -COLUMN STATUS FORMAT A80 - -SELECT - PACKAGE_OWNER, - PACKAGE_NAME, - PACKAGE_VERSION AS VERSION, - CT_MRDS.ENV_MANAGER.CHECK_PACKAGE_CHANGES(PACKAGE_OWNER, PACKAGE_NAME) AS STATUS -FROM ( - SELECT - PACKAGE_OWNER, - PACKAGE_NAME, - PACKAGE_VERSION, - ROW_NUMBER() OVER (PARTITION BY PACKAGE_OWNER, PACKAGE_NAME ORDER BY TRACKING_DATE DESC) AS RN - FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING -) -WHERE RN = 1 -ORDER BY PACKAGE_OWNER, PACKAGE_NAME; - -PROMPT -PROMPT ======================================== -PROMPT Verification Complete -PROMPT ======================================== -PROMPT -PROMPT Legend: -PROMPT OK - Package has not changed since last tracking -PROMPT WARNING - Package code changed without version update -PROMPT -PROMPT For detailed hash information, use: -PROMPT SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL; -PROMPT ======================================== - -SET FEEDBACK ON