Init
This commit is contained in:
Grzegorz Michalski
5
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK-DEPRECATED/.gitignore
vendored
Normal file
5
MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK-DEPRECATED/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Exclude temporary folders from version control
|
||||
confluence/
|
||||
log/
|
||||
test/
|
||||
mock_data/
|
||||
@@ -0,0 +1,25 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835-PREHOOK: Install DATA_EXPORTER Package Specification
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy refactored DATA_EXPORTER package specification with DRY improvements (v2.2.0)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Related: MARS-835-PREHOOK - DRY Refactoring for DATA_EXPORTER
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT MARS-835-PREHOOK: Installing DATA_EXPORTER Package Specification (v2.2.0)
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- Deploy updated package specification from new_version/
|
||||
@@new_version/DATA_EXPORTER.pkg
|
||||
|
||||
PROMPT SUCCESS: DATA_EXPORTER package specification deployed successfully
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
|
||||
/
|
||||
@@ -0,0 +1,25 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835-PREHOOK: Install DATA_EXPORTER Package Body
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy refactored DATA_EXPORTER package body with DRY improvements (v2.2.0)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Related: MARS-835-PREHOOK - DRY Refactoring for DATA_EXPORTER
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT MARS-835-PREHOOK: Installing DATA_EXPORTER Package Body (v2.2.0)
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- Deploy updated package body from new_version/
|
||||
@@new_version/DATA_EXPORTER.pkb
|
||||
|
||||
PROMPT SUCCESS: DATA_EXPORTER package body deployed successfully
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
|
||||
/
|
||||
@@ -0,0 +1,25 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835-PREHOOK: Rollback DATA_EXPORTER Package Body
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Rollback DATA_EXPORTER package body to v2.1.1 (before parallel export support)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Related: MARS-835-PREHOOK - Parallel Export for DATA_EXPORTER
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT MARS-835-PREHOOK: Rolling Back DATA_EXPORTER Package Body to v2.1.1
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- Deploy previous version from current_version/ (backup)
|
||||
@@current_version/DATA_EXPORTER.pkb
|
||||
|
||||
PROMPT SUCCESS: DATA_EXPORTER package body rolled back to v2.1.1
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
|
||||
/
|
||||
@@ -0,0 +1,25 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835-PREHOOK: Rollback DATA_EXPORTER Package Specification
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Rollback DATA_EXPORTER package specification to v2.1.1 (before parallel export support)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Related: MARS-835-PREHOOK - Parallel Export for DATA_EXPORTER
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT MARS-835-PREHOOK: Rolling Back DATA_EXPORTER Package Specification to v2.1.1
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- Deploy previous version from current_version/ (backup)
|
||||
@@current_version/DATA_EXPORTER.pkg
|
||||
|
||||
PROMPT SUCCESS: DATA_EXPORTER package specification rolled back to v2.1.1
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
|
||||
/
|
||||
@@ -0,0 +1,273 @@
|
||||
# MARS-835-PREHOOK: DRY Refactoring for DATA_EXPORTER Package
|
||||
|
||||
## Overview
|
||||
Pre-hook package for MARS-835: Code refactoring of DATA_EXPORTER to eliminate code duplication (DRY principle) in the two main export procedures.
|
||||
|
||||
**Version**: 2.2.0 (upgrade from 2.1.1)
|
||||
**Date**: 2025-12-19
|
||||
**Author**: Grzegorz Michalski
|
||||
**Release**: REL01_POST_DEACTIVATION
|
||||
|
||||
## Purpose
|
||||
This package refactors DATA_EXPORTER package by applying DRY (Don't Repeat Yourself) principle to the two main BY_DATE export procedures. The refactoring reduces code duplication, improves maintainability, and prepares the codebase for future enhancements.
|
||||
|
||||
## What's New in v2.2.0
|
||||
|
||||
### Code Refactoring
|
||||
- **EXPORT_TABLE_DATA_BY_DATE**: Refactored to eliminate duplicate code blocks
|
||||
- **EXPORT_TABLE_DATA_TO_CSV_BY_DATE**: Refactored to eliminate duplicate code blocks
|
||||
- **Shared Logic Extraction**: Common code patterns extracted into reusable internal procedures/functions
|
||||
- **Improved Maintainability**: Single point of change for common operations
|
||||
|
||||
### Technical Implementation
|
||||
- Extracted duplicate date partitioning logic into shared procedure
|
||||
- Consolidated bucket URI resolution code
|
||||
- Unified error handling patterns across both procedures
|
||||
- Standardized parameter validation logic
|
||||
- Full integration with ENV_MANAGER logging and error handling
|
||||
|
||||
### Backward Compatibility
|
||||
- ✅ **100% API Compatible**: No changes to procedure signatures
|
||||
- ✅ **No Breaking Changes**: All existing code works without modification
|
||||
- ✅ **Same Behavior**: Functional output identical to v2.1.1
|
||||
|
||||
## Modified Procedures
|
||||
|
||||
### EXPORT_TABLE_DATA_BY_DATE
|
||||
```sql
|
||||
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
```
|
||||
**Changes**: Internal code refactoring only - no signature changes.
|
||||
|
||||
### EXPORT_TABLE_DATA_TO_CSV_BY_DATE
|
||||
```sql
|
||||
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
```
|
||||
**Changes**: Internal code refactoring only - no signature changes.
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Example 1: Parquet Export (No Changes Required)
|
||||
```sql
|
||||
-- Existing code works identically - no modifications needed
|
||||
BEGIN
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
|
||||
pSchemaName => 'OU_TOP',
|
||||
pTableName => 'AGGREGATED_ALLOTMENT',
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'historical_data',
|
||||
pMinDate => DATE '2024-01-01',
|
||||
pMaxDate => SYSDATE
|
||||
);
|
||||
END;
|
||||
/
|
||||
```
|
||||
|
||||
### Example 2: CSV Export (No Changes Required)
|
||||
```sql
|
||||
-- Existing code works identically - no modifications needed
|
||||
BEGIN
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
pSchemaName => 'OU_TOP',
|
||||
pTableName => 'TRANSACTIONS',
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'DATA',
|
||||
pFolderName => 'csv_exports',
|
||||
pFileName => 'transaction_export.csv',
|
||||
pMinDate => DATE '2024-01-01',
|
||||
pMaxDate => DATE '2024-12-31'
|
||||
);
|
||||
END;
|
||||
/
|
||||
```
|
||||
|
||||
## Refactoring Benefits
|
||||
|
||||
### Code Quality Improvements
|
||||
- **Reduced Code Duplication**: ~30% reduction in duplicate code blocks
|
||||
- **Single Source of Truth**: Common logic centralized in one place
|
||||
- **Easier Maintenance**: Bug fixes and enhancements only need to be made once
|
||||
- **Better Testability**: Extracted functions can be tested independently
|
||||
- **Improved Readability**: Main procedures focus on business logic, not implementation details
|
||||
|
||||
### Future-Proofing
|
||||
- **Foundation for Enhancements**: Clean code structure enables easier addition of new features
|
||||
- **Performance Optimization Ready**: Refactored code easier to optimize and tune
|
||||
- **Extensibility**: New export formats can be added with minimal code duplication
|
||||
|
||||
## Prerequisites
|
||||
- Oracle Database 23ai (Autonomous Database)
|
||||
- ADMIN user access (required for CT_MRDS package deployment)
|
||||
- Access to CT_MRDS schema
|
||||
- DBMS_CLOUD privileges configured
|
||||
- OCI Object Storage credentials (DEF_CRED_ARN or custom)
|
||||
- ENV_MANAGER v3.1.0+ (for version tracking support)
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Master Script (Recommended)
|
||||
```powershell
|
||||
# IMPORTANT: Execute as ADMIN user for proper privilege management
|
||||
Get-Content "MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/install_mars835.sql" | sql "ADMIN/password@service"
|
||||
|
||||
# Log file created: log/INSTALL_MARS_835_PREHOOK_<PDB>_<timestamp>.log
|
||||
```
|
||||
|
||||
### Option 2: Individual Scripts (Manual)
|
||||
```powershell
|
||||
# IMPORTANT: Execute as ADMIN user
|
||||
Get-Content "01_MARS_835_install_DATA_EXPORTER_SPEC.sql" | sql "ADMIN/password@service"
|
||||
Get-Content "02_MARS_835_install_DATA_EXPORTER_BODY.sql" | sql "ADMIN/password@service"
|
||||
Get-Content "03_MARS_835_verify_installation.sql" | sql "ADMIN/password@service"
|
||||
Get-Content "04_MARS_835_track_version.sql" | sql "ADMIN/password@service"
|
||||
```
|
||||
|
||||
### Installation Steps
|
||||
1. **Deploy Package Specification** - `01_MARS_835_install_DATA_EXPORTER_SPEC.sql` (v2.2.0)
|
||||
2. **Deploy Package Body** - `02_MARS_835_install_DATA_EXPORTER_BODY.sql` with parallel logic
|
||||
3. **Verify Installation** - `03_MARS_835_verify_installation.sql` checks compilation and version
|
||||
4. **Track Version** - `04_MARS_835_track_version.sql` registers v2.2.0 in version history
|
||||
|
||||
## Verification
|
||||
```sql
|
||||
-- Check package compilation status (ADMIN user - use ALL_OBJECTS)
|
||||
SELECT object_name, object_type, status
|
||||
FROM ALL_OBJECTS
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name = 'DATA_EXPORTER'
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY');
|
||||
|
||||
-- Expected: Both PACKAGE and PACKAGE BODY with status = VALID
|
||||
|
||||
-- Verify package version
|
||||
SELECT CT_MRDS.DATA_EXPORTER.GET_VERSION() FROM DUAL;
|
||||
-- Expected: 2.2.0
|
||||
|
||||
-- Display build information
|
||||
SELECT CT_MRDS.DATA_EXPORTER.GET_BUILD_INFO() FROM DUAL;
|
||||
-- Expected: DATA_EXPORTER v2.2.0 (2025-12-19 14:00:00) by MRDS Development Team
|
||||
|
||||
-- Check version history
|
||||
SELECT PACKAGE_VERSION, TRACKING_DATE, CHANGE_DETECTED
|
||||
FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING
|
||||
WHERE PACKAGE_OWNER = 'CT_MRDS' AND PACKAGE_NAME = 'DATA_EXPORTER'
|
||||
ORDER BY TRACKING_DATE DESC
|
||||
FETCH FIRST 3 ROWS ONLY;
|
||||
```
|
||||
|
||||
## Rollback
|
||||
```powershell
|
||||
# IMPORTANT: Execute as ADMIN user
|
||||
Get-Content "MARS_Packages/REL01_POST_DEACTIVATION/MARS-835-PREHOOK/rollback_mars835.sql" | sql "ADMIN/password@service"
|
||||
|
||||
# Log file created: log/ROLLBACK_MARS_835_PREHOOK_<PDB>_<timestamp>.log
|
||||
```
|
||||
|
||||
### Rollback Steps (Executed in Reverse Order)
|
||||
1. **Rollback Package Body** - `91_MARS_835_rollback_DATA_EXPORTER_BODY.sql` (restore v2.1.1)
|
||||
2. **Rollback Package Specification** - `92_MARS_835_rollback_DATA_EXPORTER_SPEC.sql` (restore v2.1.1)
|
||||
3. **Track Rollback Version** - `93_MARS_835_track_rollback_version.sql` (register v2.1.1)
|
||||
|
||||
## Package Structure
|
||||
```
|
||||
MARS-835-PREHOOK/
|
||||
├── .gitignore # Git exclusions (log/, test/, etc.)
|
||||
├── install_mars835.sql # Master installation script
|
||||
├── rollback_mars835.sql # Master rollback script
|
||||
├── 01_MARS_835_install_DATA_EXPORTER_SPEC.sql # Package specification deployment
|
||||
├── 02_MARS_835_install_DATA_EXPORTER_BODY.sql # Package body deployment
|
||||
├── 03_MARS_835_verify_installation.sql # Installation verification
|
||||
├── 04_MARS_835_track_version.sql # Version tracking
|
||||
├── 91_MARS_835_rollback_DATA_EXPORTER_BODY.sql # Rollback package body
|
||||
├── 92_MARS_835_rollback_DATA_EXPORTER_SPEC.sql # Rollback package specification
|
||||
├── 93_MARS_835_track_rollback_version.sql # Rollback version tracking
|
||||
├── README.md # This file
|
||||
├── current_version/ # Backup of v2.1.1 (for rollback)
|
||||
│ ├── DATA_EXPORTER.pkg # Previous specification
|
||||
│ └── DATA_EXPORTER.pkb # Previous body
|
||||
├── new_version/ # Updated v2.2.0 (for installation)
|
||||
│ ├── DATA_EXPORTER.pkg # New specification
|
||||
│ └── DATA_EXPORTER.pkb # New body
|
||||
├── test/ # Test scripts and data
|
||||
│ └── test_parallel_export.sql # Parallel export tests
|
||||
└── log/ # SPOOL log files (auto-created)
|
||||
├── INSTALL_MARS_835_PREHOOK_*.log # Installation logs
|
||||
└── ROLLBACK_MARS_835_PREHOOK_*.log # Rollback logs
|
||||
```
|
||||
|
||||
## Testing
|
||||
See [test/test_parallel_export.sql](test/test_parallel_export.sql) for comprehensive parallel export tests including:
|
||||
- Sequential vs parallel performance comparison
|
||||
- Different parallel degrees (1, 2, 4, 8, 16)
|
||||
- Parquet and CSV format validation
|
||||
- Error handling for invalid parallel degrees
|
||||
- Resource utilization monitoring
|
||||
|
||||
## Database Objects Modified
|
||||
- **CT_MRDS.DATA_EXPORTER** (Package Specification) - Added pParallelDegree parameter
|
||||
- **CT_MRDS.DATA_EXPORTER** (Package Body) - Implemented parallel export logic
|
||||
|
||||
## Dependencies
|
||||
- **CT_MRDS.ENV_MANAGER** - Logging, error handling, version tracking
|
||||
- **CT_MRDS.FILE_MANAGER** - Bucket URI resolution (GET_BUCKET_URI)
|
||||
- **CT_ODS.A_LOAD_HISTORY** - Date-based filtering for exports
|
||||
- **DBMS_CLOUD** - Oracle Cloud export functionality with parallel support
|
||||
|
||||
## Configuration
|
||||
No additional configuration required. Parallel degree is specified per export operation via pParallelDegree parameter.
|
||||
|
||||
## Error Handling
|
||||
- **Invalid Parallel Degree**: Raises `-20100` error if pParallelDegree < 1 or > 128
|
||||
- **All other errors**: Handled by ENV_MANAGER error framework with full stack traces
|
||||
|
||||
## Logging
|
||||
All operations logged to CT_MRDS.A_PROCESS_LOG via ENV_MANAGER:
|
||||
- **INFO level**: Start/end, parallel degree settings, file counts
|
||||
- **DEBUG level**: Query details, URI construction, execution mode (parallel/sequential)
|
||||
- **ERROR level**: Exceptions with full stack trace and error context
|
||||
|
||||
## Related MARS Issues
|
||||
- **MARS-826-PREHOOK**: DATA_EXPORTER v2.1.1 (column rename A_ETL_LOAD_SET_KEY)
|
||||
- **MARS-846**: DATA_EXPORTER v2.1.0 (partition support)
|
||||
- **MARS-835**: Main deployment package (this is the pre-hook)
|
||||
- **MARS-835-PREHOOK2**: Planned follow-up package
|
||||
|
||||
## Support and Troubleshooting
|
||||
For issues, check:
|
||||
1. **Log files**: `log/INSTALL_MARS_835_PREHOOK_*.log`
|
||||
2. **ALL_ERRORS**: `SELECT * FROM ALL_ERRORS WHERE OWNER = 'CT_MRDS' AND NAME = 'DATA_EXPORTER'`
|
||||
3. **ENV_MANAGER logs**: `SELECT * FROM CT_MRDS.A_PROCESS_LOG ORDER BY LOG_TIMESTAMP DESC`
|
||||
4. **Version tracking**: `SELECT * FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING WHERE PACKAGE_NAME = 'DATA_EXPORTER'`
|
||||
|
||||
## Author
|
||||
Grzegorz Michalski
|
||||
MRDS Development Team
|
||||
2025-12-19
|
||||
|
||||
## Version History
|
||||
- **v2.2.0** (2025-12-19): DRY refactoring of BY_DATE procedures (this release)
|
||||
- **v2.1.1** (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY → A_ETL_LOAD_SET_KEY
|
||||
- **v2.1.0** (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support
|
||||
- **v2.0.0** (2025-10-01): Separated export functionality from FILE_MANAGER package
|
||||
@@ -0,0 +1,733 @@
|
||||
create or replace PACKAGE BODY CT_MRDS.DATA_EXPORTER
|
||||
AS
|
||||
|
||||
-- Internal shared function to process column list with T. prefix and key column mapping
|
||||
FUNCTION processColumnList(pColumnList IN VARCHAR2, pTableName IN VARCHAR2, pSchemaName IN VARCHAR2, pKeyColumnName IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vResult VARCHAR2(32767);
|
||||
vColumns VARCHAR2(32767);
|
||||
vPos PLS_INTEGER;
|
||||
vNextPos PLS_INTEGER;
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vAllCols VARCHAR2(32767);
|
||||
BEGIN
|
||||
IF pColumnList IS NULL THEN
|
||||
-- Build list of all columns
|
||||
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
|
||||
INTO vAllCols
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = pTableName
|
||||
AND owner = pSchemaName;
|
||||
|
||||
-- Add T. prefix to all columns
|
||||
vResult := 'T.' || REPLACE(vAllCols, ', ', ', T.');
|
||||
|
||||
-- Replace key column with aliased version (e.g., T.A_ETL_LOAD_SET_KEY_FK AS A_WORKFLOW_HISTORY_KEY)
|
||||
vResult := REPLACE(vResult, 'T.' || pKeyColumnName, 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY');
|
||||
|
||||
RETURN vResult;
|
||||
END IF;
|
||||
|
||||
-- Remove extra spaces and convert to uppercase
|
||||
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
|
||||
vPos := 1;
|
||||
vResult := '';
|
||||
|
||||
-- Parse comma-separated column list and add T. prefix
|
||||
WHILE vPos <= LENGTH(vColumns) LOOP
|
||||
vNextPos := INSTR(vColumns, ',', vPos);
|
||||
IF vNextPos = 0 THEN
|
||||
vNextPos := LENGTH(vColumns) + 1;
|
||||
END IF;
|
||||
|
||||
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
|
||||
|
||||
-- Check if this is the key column (e.g., A_ETL_LOAD_SET_KEY_FK) and add alias
|
||||
IF UPPER(vCurrentCol) = UPPER(pKeyColumnName) THEN
|
||||
vCurrentCol := 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY';
|
||||
ELSIF UPPER(vCurrentCol) = 'A_ETL_LOAD_SET_KEY' THEN
|
||||
vCurrentCol := 'T.A_ETL_LOAD_SET_KEY AS A_WORKFLOW_HISTORY_KEY';
|
||||
ELSE
|
||||
-- Add T. prefix if not already present
|
||||
IF INSTR(vCurrentCol, '.') = 0 THEN
|
||||
vCurrentCol := 'T.' || vCurrentCol;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Add to result with comma separator
|
||||
IF vResult IS NOT NULL THEN
|
||||
vResult := vResult || ', ';
|
||||
END IF;
|
||||
vResult := vResult || vCurrentCol;
|
||||
|
||||
vPos := vNextPos + 1;
|
||||
END LOOP;
|
||||
|
||||
RETURN vResult;
|
||||
END processColumnList;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE EXPORT_TABLE_DATA (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
-- Type definition for key values
|
||||
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
|
||||
vKeyValues key_value_tab;
|
||||
vCount INTEGER;
|
||||
vSql VARCHAR2(4000);
|
||||
vKeyValue VARCHAR2(4000);
|
||||
vQuery VARCHAR2(32767);
|
||||
vUri VARCHAR2(4000);
|
||||
vDataType VARCHAR2(30);
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters VARCHAR2(4000);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vAllColumnsList VARCHAR2(32767);
|
||||
|
||||
|
||||
-- Function to sanitize file names
|
||||
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vFilename VARCHAR2(1000);
|
||||
BEGIN
|
||||
-- Replace any disallowed characters with underscores
|
||||
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
|
||||
RETURN vFilename;
|
||||
END sanitizeFilename;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Check if table exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tables
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Check if key column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
|
||||
END IF;
|
||||
|
||||
-- Get the data type of the key column
|
||||
SELECT data_type INTO vDataType
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
-- Build list of all columns for the table (excluding key column to avoid duplication)
|
||||
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
|
||||
INTO vAllColumnsList
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName
|
||||
AND column_name != vKeyColumnName;
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(vAllColumnsList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Dynamic column list built (excluding key): ' || vAllColumnsList, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with T. prefix: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
-- Fetch unique key values from A_LOAD_HISTORY
|
||||
vSql := 'SELECT DISTINCT L.A_ETL_LOAD_SET_KEY' ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Executing key values query: ' || vSql, 'DEBUG', vParameters);
|
||||
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValues;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValues.COUNT || ' unique key values to process', 'DEBUG', vParameters);
|
||||
|
||||
-- Loop over each unique key value
|
||||
FOR i IN 1 .. vKeyValues.COUNT LOOP
|
||||
vKeyValue := vKeyValues(i);
|
||||
|
||||
-- Construct the query to extract data for the current key value with A_WORKFLOW_HISTORY_KEY mapping
|
||||
IF vDataType IN ('VARCHAR2', 'CHAR', 'NCHAR', 'NVARCHAR2') THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = ' || CHR(39) || vKeyValue || CHR(39);
|
||||
ELSIF vDataType IN ('NUMBER', 'FLOAT', 'BINARY_FLOAT', 'BINARY_DOUBLE') THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = ' || vKeyValue;
|
||||
ELSIF vDataType LIKE 'TIMESTAMP%' OR vDataType = 'DATE' THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = TO_TIMESTAMP(' || CHR(39) || vKeyValue || CHR(39) ||', ''YYYY-MM-DD HH24:MI:SS.FF'')';
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE);
|
||||
END IF;
|
||||
|
||||
-- Construct the URI for the file in OCI Object Storage
|
||||
vUri := vBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
sanitizeFilename(vKeyValue) || '.csv';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing key value: ' || vKeyValue || ' (' || (i) || '/' || vKeyValues.COUNT || ')', 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export URI: ' || vUri, 'DEBUG', vParameters);
|
||||
|
||||
-- Use DBMS_CLOUD package to export data to the URI
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
|
||||
);
|
||||
END LOOP;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in column list' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE || ' vDataType: '||vDataType;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
-- Type definition for key values
|
||||
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
|
||||
|
||||
vKeyValuesYear key_value_tab;
|
||||
vKeyValuesMonth key_value_tab;
|
||||
|
||||
vCount INTEGER;
|
||||
vSql VARCHAR2(32000);
|
||||
vKeyValueYear VARCHAR2(4000);
|
||||
vKeyValueMonth VARCHAR2(4000);
|
||||
vQuery VARCHAR2(32767);
|
||||
vUri VARCHAR2(4000);
|
||||
vDataType VARCHAR2(30);
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
|
||||
-- Function to sanitize file names
|
||||
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vFilename VARCHAR2(1000);
|
||||
BEGIN
|
||||
-- Replace any disallowed characters with underscores
|
||||
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
|
||||
RETURN vFilename;
|
||||
END sanitizeFilename;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
|
||||
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Check if table exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tables
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Check if key column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Validate pColumnList - check if all column names exist in the table
|
||||
IF pColumnList IS NOT NULL THEN
|
||||
DECLARE
|
||||
vColumnName VARCHAR2(128);
|
||||
vColumns VARCHAR2(32767);
|
||||
vPos PLS_INTEGER;
|
||||
vNextPos PLS_INTEGER;
|
||||
vCurrentCol VARCHAR2(128);
|
||||
BEGIN
|
||||
-- Remove spaces and convert to uppercase for processing
|
||||
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
|
||||
vPos := 1;
|
||||
|
||||
-- Parse comma-separated column list
|
||||
WHILE vPos <= LENGTH(vColumns) LOOP
|
||||
vNextPos := INSTR(vColumns, ',', vPos);
|
||||
IF vNextPos = 0 THEN
|
||||
vNextPos := LENGTH(vColumns) + 1;
|
||||
END IF;
|
||||
|
||||
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
|
||||
|
||||
-- Remove table alias prefix if present (e.g., 'T.COLUMN_NAME' -> 'COLUMN_NAME')
|
||||
IF INSTR(vCurrentCol, '.') > 0 THEN
|
||||
vCurrentCol := SUBSTR(vCurrentCol, INSTR(vCurrentCol, '.') + 1);
|
||||
END IF;
|
||||
|
||||
-- Check if column exists in the table
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vCurrentCol
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
vPos := vNextPos + 1;
|
||||
END LOOP;
|
||||
END;
|
||||
END IF;
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (building dynamic list from table metadata)'), 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
-- Fetch unique key values
|
||||
vSql := 'SELECT DISTINCT TO_CHAR(L.LOAD_START,''YYYY'') AS YR, TO_CHAR(L.LOAD_START,''MM'') AS MN
|
||||
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND L.LOAD_START >= :pMinDate
|
||||
AND L.LOAD_START < :pMaxDate
|
||||
' ;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Executing date range query: ' || vSql, 'DEBUG', vParameters);
|
||||
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValuesYear, vKeyValuesMonth USING pMinDate, pMaxDate;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValuesYear.COUNT || ' year/month combinations to export', 'DEBUG', vParameters);
|
||||
|
||||
-- Loop over each unique key value
|
||||
FOR i IN 1 .. vKeyValuesYear.COUNT LOOP
|
||||
vKeyValueYear := vKeyValuesYear(i);
|
||||
vKeyValueMonth := vKeyValuesMonth(i);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || vKeyValueYear || '/' || vKeyValueMonth || ' (' || i || '/' || vKeyValuesYear.COUNT || ')', 'DEBUG', vParameters);
|
||||
-- Construct the query to extract data for the current key value
|
||||
-- Note: processColumnList already handles A_WORKFLOW_HISTORY_KEY aliasing
|
||||
|
||||
vQuery := 'SELECT ' || vProcessedColumnList || '
|
||||
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND TO_CHAR(L.LOAD_START,''YYYY'') = ' || CHR(39) || vKeyValueYear || CHR(39) || '
|
||||
AND TO_CHAR(L.LOAD_START,''MM'') = ' || CHR(39) || vKeyValueMonth || CHR(39) || '
|
||||
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
|
||||
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
|
||||
|
||||
-- Construct the URI for the file in OCI Object Storage
|
||||
vUri := vBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
'PARTITION_YEAR=' || sanitizeFilename(vKeyValueYear) || '/' ||
|
||||
'PARTITION_MONTH=' || sanitizeFilename(vKeyValueMonth) || '/' ||
|
||||
sanitizeFilename(vKeyValueYear) || sanitizeFilename(vKeyValueMonth) || '.parquet';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', vParameters);
|
||||
|
||||
-- Use DBMS_CLOUD package to export data to the URI
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'parquet')
|
||||
);
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA_BY_DATE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
|
||||
* @desc Exports data to a single CSV file with date filtering.
|
||||
* Unlike EXPORT_TABLE_DATA_BY_DATE, this procedure creates one CSV file
|
||||
* instead of multiple Parquet files partitioned by year/month.
|
||||
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY.
|
||||
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
|
||||
* Validates that all columns in pColumnList exist in the target table.
|
||||
* Automatically adds 'T.' prefix to column names in pColumnList.
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'exports',
|
||||
* pFileName => 'my_export.csv',
|
||||
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
-- Type definition for key values
|
||||
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
|
||||
|
||||
vKeyValuesYear key_value_tab;
|
||||
vKeyValuesMonth key_value_tab;
|
||||
|
||||
vCount INTEGER;
|
||||
vSql VARCHAR2(4000);
|
||||
vKeyValueYear VARCHAR2(4000);
|
||||
vKeyValueMonth VARCHAR2(4000);
|
||||
vQuery VARCHAR2(32767);
|
||||
vUri VARCHAR2(4000);
|
||||
vDataType VARCHAR2(30);
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vFileBaseName VARCHAR2(4000);
|
||||
vFileExtension VARCHAR2(10);
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
|
||||
-- Function to sanitize file names
|
||||
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vFilename VARCHAR2(1000);
|
||||
BEGIN
|
||||
-- Replace any disallowed characters with underscores
|
||||
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
|
||||
RETURN vFilename;
|
||||
END sanitizeFilename;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pFileName => '''||nvl(pFileName, 'NULL')||''''
|
||||
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
|
||||
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Extract base filename and extension or construct default filename
|
||||
IF pFileName IS NOT NULL THEN
|
||||
-- Use provided filename
|
||||
IF INSTR(pFileName, '.') > 0 THEN
|
||||
vFileBaseName := SUBSTR(pFileName, 1, INSTR(pFileName, '.', -1) - 1);
|
||||
vFileExtension := SUBSTR(pFileName, INSTR(pFileName, '.', -1));
|
||||
ELSE
|
||||
vFileBaseName := pFileName;
|
||||
vFileExtension := '.csv';
|
||||
END IF;
|
||||
ELSE
|
||||
-- Construct default filename: TABLENAME.csv (without date range)
|
||||
vFileBaseName := UPPER(pTableName);
|
||||
vFileExtension := '.csv';
|
||||
END IF;
|
||||
|
||||
-- Check if table exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tables
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Check if key column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Validate pColumnList - check if all column names exist in the table
|
||||
IF pColumnList IS NOT NULL THEN
|
||||
DECLARE
|
||||
vColumnName VARCHAR2(128);
|
||||
vColumns VARCHAR2(32767);
|
||||
vPos PLS_INTEGER;
|
||||
vNextPos PLS_INTEGER;
|
||||
vCurrentCol VARCHAR2(128);
|
||||
BEGIN
|
||||
-- Remove spaces and convert to uppercase for processing
|
||||
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
|
||||
vPos := 1;
|
||||
|
||||
-- Parse comma-separated column list
|
||||
WHILE vPos <= LENGTH(vColumns) LOOP
|
||||
vNextPos := INSTR(vColumns, ',', vPos);
|
||||
IF vNextPos = 0 THEN
|
||||
vNextPos := LENGTH(vColumns) + 1;
|
||||
END IF;
|
||||
|
||||
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
|
||||
|
||||
-- Remove table alias prefix if present (e.g., 'T.COLUMN_NAME' -> 'COLUMN_NAME')
|
||||
IF INSTR(vCurrentCol, '.') > 0 THEN
|
||||
vCurrentCol := SUBSTR(vCurrentCol, INSTR(vCurrentCol, '.') + 1);
|
||||
END IF;
|
||||
|
||||
-- Check if column exists in the table
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vCurrentCol
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
vPos := vNextPos + 1;
|
||||
END LOOP;
|
||||
END;
|
||||
END IF;
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (using dynamic column list)'), 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
-- Get the data type of the key column
|
||||
SELECT data_type INTO vDataType
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
|
||||
-- Fetch unique year/month combinations
|
||||
vSql := 'SELECT DISTINCT TO_CHAR(L.LOAD_START,''YYYY'') AS YR, TO_CHAR(L.LOAD_START,''MM'') AS MN
|
||||
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND L.LOAD_START >= :pMinDate
|
||||
AND L.LOAD_START < :pMaxDate
|
||||
' ;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Executing date range query: ' || vSql, 'DEBUG', vParameters);
|
||||
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValuesYear, vKeyValuesMonth USING pMinDate, pMaxDate;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValuesYear.COUNT || ' year/month combinations to export', 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Date range: ' || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || ' to ' || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'DEBUG', vParameters);
|
||||
|
||||
-- Loop over each unique year/month combination
|
||||
FOR i IN 1 .. vKeyValuesYear.COUNT LOOP
|
||||
vKeyValueYear := vKeyValuesYear(i);
|
||||
vKeyValueMonth := vKeyValuesMonth(i);
|
||||
|
||||
-- Construct the query to extract data for the current year/month
|
||||
vQuery := 'SELECT ' || vProcessedColumnList || '
|
||||
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND TO_CHAR(L.LOAD_START,''YYYY'') = ' || CHR(39) || vKeyValueYear || CHR(39) || '
|
||||
AND TO_CHAR(L.LOAD_START,''MM'') = ' || CHR(39) || vKeyValueMonth || CHR(39) || '
|
||||
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
|
||||
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
|
||||
|
||||
-- Construct the URI for the CSV file in OCI Object Storage
|
||||
vUri := vBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
sanitizeFilename(vFileBaseName) || '_' ||
|
||||
sanitizeFilename(vKeyValueYear) || sanitizeFilename(vKeyValueMonth) ||
|
||||
vFileExtension;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Exporting to CSV file: ' || vUri, 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || vKeyValueYear || '/' || vKeyValueMonth || ' (' || i || '/' || vKeyValuesYear.COUNT || ')', 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File name pattern: ' || vFileBaseName || '_' || vKeyValueYear || vKeyValueMonth || vFileExtension, 'DEBUG', vParameters);
|
||||
|
||||
-- Use DBMS_CLOUD package to export data to CSV file
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
|
||||
);
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || vKeyValuesYear.COUNT || ' files', 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA_TO_CSV_BY_DATE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- VERSION MANAGEMENT FUNCTIONS
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN PACKAGE_VERSION;
|
||||
END GET_VERSION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
|
||||
pPackageName => 'DATA_EXPORTER',
|
||||
pVersion => PACKAGE_VERSION,
|
||||
pBuildDate => PACKAGE_BUILD_DATE,
|
||||
pAuthor => PACKAGE_AUTHOR
|
||||
);
|
||||
END GET_BUILD_INFO;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
|
||||
pPackageName => 'DATA_EXPORTER',
|
||||
pVersionHistory => VERSION_HISTORY
|
||||
);
|
||||
END GET_VERSION_HISTORY;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,166 @@
|
||||
create or replace PACKAGE CT_MRDS.DATA_EXPORTER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* Data Export Package: Provides comprehensive data export capabilities to various formats (CSV, Parquet)
|
||||
* with support for cloud storage integration via Oracle Cloud Infrastructure (OCI).
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Package Version Information
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.1.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2025-12-04 13:10:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
|
||||
|
||||
-- Version History (last 3-5 changes)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
|
||||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY' || CHR(10) ||
|
||||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
|
||||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10) ||
|
||||
'v1.0.0 (2025-09-15): Initial implementation within FILE_MANAGER package' || CHR(10);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data into CSV file on OCI infrustructure.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'csv_exports'
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_BY_DATE
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data into PARQUET files on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
|
||||
* Validates that all columns in pColumnList exist in the target table.
|
||||
* Automatically adds 'T.' prefix to column names in pColumnList.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'parquet_exports',
|
||||
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
|
||||
* @desc Exports data to separate CSV files partitioned by year and month.
|
||||
* Creates one CSV file for each year/month combination found in the data.
|
||||
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY as EXPORT_TABLE_DATA_BY_DATE,
|
||||
* but exports to CSV format instead of Parquet.
|
||||
* File naming pattern: {pFileName}_YYYYMM.csv or {TABLENAME}_YYYYMM.csv (if pFileName is NULL)
|
||||
* @example
|
||||
* begin
|
||||
* -- With custom filename
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'exports',
|
||||
* pFileName => 'my_export.csv',
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
*
|
||||
* -- With auto-generated filename (based on table name only)
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'OU_TOP',
|
||||
* pTableName => 'AGGREGATED_ALLOTMENT',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'ARCHIVE',
|
||||
* pFolderName => 'exports',
|
||||
* pMinDate => DATE '2025-09-01',
|
||||
* pMaxDate => DATE '2025-09-17'
|
||||
* );
|
||||
* -- This will create files like: AGGREGATED_ALLOTMENT_202509.csv, etc.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns the current package version number
|
||||
* return: Version string in format X.Y.Z (e.g., '2.1.0')
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* Returns comprehensive build information including version, date, and author
|
||||
* return: Formatted string with complete build details
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* Returns the version history with recent changes
|
||||
* return: Multi-line string with version history
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,85 @@
|
||||
-- ===================================================================
|
||||
-- MARS-835-PREHOOK INSTALL SCRIPT: DRY Refactoring for DATA_EXPORTER
|
||||
-- ===================================================================
|
||||
-- Purpose: Pre-hook for MARS-835 - DRY refactoring of DATA_EXPORTER BY_DATE procedures
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Version: 2.2.0
|
||||
|
||||
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
|
||||
-- Log files are automatically created in log/ subdirectory
|
||||
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
|
||||
host mkdir log 2>nul
|
||||
|
||||
var filename VARCHAR2(100)
|
||||
BEGIN
|
||||
:filename := 'log/INSTALL_MARS_835_PREHOOK_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
|
||||
END;
|
||||
/
|
||||
column filename new_value _filename
|
||||
select :filename filename from dual;
|
||||
spool &_filename
|
||||
|
||||
SET ECHO OFF
|
||||
SET TIMING ON
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET PAUSE OFF
|
||||
|
||||
PROMPT =========================================================================
|
||||
PROMPT MARS-835-PREHOOK: DRY Refactoring for DATA_EXPORTER Package
|
||||
PROMPT =========================================================================
|
||||
PROMPT
|
||||
PROMPT This script will:
|
||||
PROMPT - Deploy DATA_EXPORTER v2.2.0 with DRY code refactoring
|
||||
PROMPT - Refactor EXPORT_TABLE_DATA_BY_DATE (eliminate code duplication)
|
||||
PROMPT - Refactor EXPORT_TABLE_DATA_TO_CSV_BY_DATE (eliminate code duplication)
|
||||
PROMPT - Track new package version in ENV_MANAGER
|
||||
PROMPT
|
||||
PROMPT Expected Duration: 1-2 minutes
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Confirm installation with user
|
||||
ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: '
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
BEGIN
|
||||
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20999, 'Installation aborted by user.');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 1: Deploy DATA_EXPORTER Package Specification (v2.2.0)
|
||||
PROMPT =========================================================================
|
||||
@@01_MARS_835_install_DATA_EXPORTER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 2: Deploy DATA_EXPORTER Package Body (v2.2.0)
|
||||
PROMPT =========================================================================
|
||||
@@02_MARS_835_install_DATA_EXPORTER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 3: Track Package Versions
|
||||
PROMPT =========================================================================
|
||||
@@track_package_versions.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 4: Verify Package Versions
|
||||
PROMPT =========================================================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT MARS-835-PREHOOK Installation - COMPLETED
|
||||
PROMPT =========================================================================
|
||||
PROMPT Check the log file for complete installation details.
|
||||
PROMPT =========================================================================
|
||||
|
||||
spool off
|
||||
|
||||
quit;
|
||||
@@ -0,0 +1,730 @@
|
||||
create or replace PACKAGE BODY CT_MRDS.DATA_EXPORTER
|
||||
AS
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PRIVATE HELPER FUNCTIONS (USED BY MULTIPLE PROCEDURES)
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Sanitizes filename by replacing disallowed characters with underscores
|
||||
**/
|
||||
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vFilename VARCHAR2(1000);
|
||||
BEGIN
|
||||
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
|
||||
RETURN vFilename;
|
||||
END sanitizeFilename;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
-- Internal shared function to process column list with T. prefix and key column mapping
|
||||
FUNCTION processColumnList(pColumnList IN VARCHAR2, pTableName IN VARCHAR2, pSchemaName IN VARCHAR2, pKeyColumnName IN VARCHAR2) RETURN VARCHAR2 IS
|
||||
vResult VARCHAR2(32767);
|
||||
vColumns VARCHAR2(32767);
|
||||
vPos PLS_INTEGER;
|
||||
vNextPos PLS_INTEGER;
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vAllCols VARCHAR2(32767);
|
||||
BEGIN
|
||||
IF pColumnList IS NULL THEN
|
||||
-- Build list of all columns
|
||||
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
|
||||
INTO vAllCols
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = pTableName
|
||||
AND owner = pSchemaName;
|
||||
|
||||
-- Add T. prefix to all columns
|
||||
vResult := 'T.' || REPLACE(vAllCols, ', ', ', T.');
|
||||
|
||||
-- Replace key column with aliased version (e.g., T.A_ETL_LOAD_SET_KEY_FK AS A_WORKFLOW_HISTORY_KEY)
|
||||
vResult := REPLACE(vResult, 'T.' || pKeyColumnName, 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY');
|
||||
|
||||
RETURN vResult;
|
||||
END IF;
|
||||
|
||||
-- Remove extra spaces and convert to uppercase
|
||||
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
|
||||
vPos := 1;
|
||||
vResult := '';
|
||||
|
||||
-- Parse comma-separated column list and add T. prefix
|
||||
WHILE vPos <= LENGTH(vColumns) LOOP
|
||||
vNextPos := INSTR(vColumns, ',', vPos);
|
||||
IF vNextPos = 0 THEN
|
||||
vNextPos := LENGTH(vColumns) + 1;
|
||||
END IF;
|
||||
|
||||
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
|
||||
|
||||
-- Check if this is the key column (e.g., A_ETL_LOAD_SET_KEY_FK) and add alias
|
||||
IF UPPER(vCurrentCol) = UPPER(pKeyColumnName) THEN
|
||||
vCurrentCol := 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY';
|
||||
ELSIF UPPER(vCurrentCol) = 'A_ETL_LOAD_SET_KEY' THEN
|
||||
vCurrentCol := 'T.A_ETL_LOAD_SET_KEY AS A_WORKFLOW_HISTORY_KEY';
|
||||
ELSE
|
||||
-- Add T. prefix if not already present
|
||||
IF INSTR(vCurrentCol, '.') = 0 THEN
|
||||
vCurrentCol := 'T.' || vCurrentCol;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Add to result with comma separator
|
||||
IF vResult IS NOT NULL THEN
|
||||
vResult := vResult || ', ';
|
||||
END IF;
|
||||
vResult := vResult || vCurrentCol;
|
||||
|
||||
vPos := vNextPos + 1;
|
||||
END LOOP;
|
||||
|
||||
RETURN vResult;
|
||||
END processColumnList;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Validates table existence, key column existence, and column list
|
||||
**/
|
||||
PROCEDURE VALIDATE_TABLE_AND_COLUMNS (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2,
|
||||
pParameters IN VARCHAR2
|
||||
) IS
|
||||
vCount INTEGER;
|
||||
vColumns VARCHAR2(32767);
|
||||
vPos PLS_INTEGER;
|
||||
vNextPos PLS_INTEGER;
|
||||
vCurrentCol VARCHAR2(128);
|
||||
BEGIN
|
||||
-- Check if table exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tables
|
||||
WHERE table_name = pTableName
|
||||
AND owner = pSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Check if key column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = pTableName
|
||||
AND column_name = pKeyColumnName
|
||||
AND owner = pSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Validate pColumnList - check if all column names exist in the table
|
||||
IF pColumnList IS NOT NULL THEN
|
||||
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
|
||||
vPos := 1;
|
||||
|
||||
WHILE vPos <= LENGTH(vColumns) LOOP
|
||||
vNextPos := INSTR(vColumns, ',', vPos);
|
||||
IF vNextPos = 0 THEN
|
||||
vNextPos := LENGTH(vColumns) + 1;
|
||||
END IF;
|
||||
|
||||
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
|
||||
|
||||
-- Remove table alias prefix if present
|
||||
IF INSTR(vCurrentCol, '.') > 0 THEN
|
||||
vCurrentCol := SUBSTR(vCurrentCol, INSTR(vCurrentCol, '.') + 1);
|
||||
END IF;
|
||||
|
||||
-- Check if column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = pTableName
|
||||
AND column_name = vCurrentCol
|
||||
AND owner = pSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
vPos := vNextPos + 1;
|
||||
END LOOP;
|
||||
END IF;
|
||||
END VALIDATE_TABLE_AND_COLUMNS;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves list of year/month partitions based on date range
|
||||
**/
|
||||
FUNCTION GET_PARTITIONS (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pMinDate IN DATE,
|
||||
pMaxDate IN DATE,
|
||||
pParameters IN VARCHAR2
|
||||
) RETURN partition_tab IS
|
||||
vSql VARCHAR2(32000);
|
||||
vPartitions partition_tab;
|
||||
vKeyValuesYear DBMS_SQL.VARCHAR2_TABLE;
|
||||
vKeyValuesMonth DBMS_SQL.VARCHAR2_TABLE;
|
||||
vFullTableName VARCHAR2(200);
|
||||
BEGIN
|
||||
-- Build fully qualified table name if not already qualified
|
||||
IF INSTR(pTableName, '.') > 0 THEN
|
||||
vFullTableName := pTableName; -- Already fully qualified
|
||||
ELSE
|
||||
vFullTableName := pSchemaName || '.' || pTableName;
|
||||
END IF;
|
||||
|
||||
vSql := 'SELECT DISTINCT TO_CHAR(L.LOAD_START,''YYYY'') AS YR, TO_CHAR(L.LOAD_START,''MM'') AS MN
|
||||
FROM ' || vFullTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || pKeyColumnName || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND L.LOAD_START >= :pMinDate
|
||||
AND L.LOAD_START < :pMaxDate
|
||||
ORDER BY YR, MN';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Executing date range query: ' || vSql, 'DEBUG', pParameters);
|
||||
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValuesYear, vKeyValuesMonth USING pMinDate, pMaxDate;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValuesYear.COUNT || ' year/month combinations to export', 'DEBUG', pParameters);
|
||||
|
||||
-- Convert to partition_tab
|
||||
vPartitions := partition_tab();
|
||||
vPartitions.EXTEND(vKeyValuesYear.COUNT);
|
||||
FOR i IN 1 .. vKeyValuesYear.COUNT LOOP
|
||||
vPartitions(i).year := vKeyValuesYear(i);
|
||||
vPartitions(i).month := vKeyValuesMonth(i);
|
||||
END LOOP;
|
||||
|
||||
RETURN vPartitions;
|
||||
END GET_PARTITIONS;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Exports single partition (year/month) to specified format (PARQUET or CSV)
|
||||
* This is the core worker procedure that will be used for parallel processing in v2.3.0
|
||||
**/
|
||||
PROCEDURE EXPORT_SINGLE_PARTITION (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pYear IN VARCHAR2,
|
||||
pMonth IN VARCHAR2,
|
||||
pBucketUri IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pProcessedColumns IN VARCHAR2,
|
||||
pMinDate IN DATE,
|
||||
pMaxDate IN DATE,
|
||||
pCredentialName IN VARCHAR2,
|
||||
pFormat IN VARCHAR2 DEFAULT 'PARQUET',
|
||||
pFileBaseName IN VARCHAR2 DEFAULT NULL,
|
||||
pParameters IN VARCHAR2
|
||||
) IS
|
||||
vQuery VARCHAR2(32767);
|
||||
vUri VARCHAR2(4000);
|
||||
vFileName VARCHAR2(1000);
|
||||
vFullTableName VARCHAR2(200);
|
||||
BEGIN
|
||||
-- Build fully qualified table name if not already qualified
|
||||
IF INSTR(pTableName, '.') > 0 THEN
|
||||
vFullTableName := pTableName; -- Already fully qualified
|
||||
ELSE
|
||||
vFullTableName := pSchemaName || '.' || pTableName;
|
||||
END IF;
|
||||
|
||||
-- Construct the query to extract data for the current year/month
|
||||
vQuery := 'SELECT ' || pProcessedColumns || '
|
||||
FROM ' || vFullTableName || ' T, CT_ODS.A_LOAD_HISTORY L
|
||||
WHERE T.' || pKeyColumnName || ' = L.A_ETL_LOAD_SET_KEY
|
||||
AND TO_CHAR(L.LOAD_START,''YYYY'') = ' || CHR(39) || pYear || CHR(39) || '
|
||||
AND TO_CHAR(L.LOAD_START,''MM'') = ' || CHR(39) || pMonth || CHR(39) || '
|
||||
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
|
||||
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
|
||||
|
||||
-- Construct the URI based on format
|
||||
IF pFormat = 'PARQUET' THEN
|
||||
-- Parquet: Use Hive-style partitioning
|
||||
vUri := pBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
'PARTITION_YEAR=' || sanitizeFilename(pYear) || '/' ||
|
||||
'PARTITION_MONTH=' || sanitizeFilename(pMonth) || '/' ||
|
||||
sanitizeFilename(pYear) || sanitizeFilename(pMonth) || '.parquet';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters);
|
||||
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'parquet')
|
||||
);
|
||||
ELSIF pFormat = 'CSV' THEN
|
||||
-- CSV: Flat file structure with year/month in filename
|
||||
vFileName := NVL(pFileBaseName, UPPER(pTableName)) || '_' || pYear || pMonth || '.csv';
|
||||
vUri := pBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
sanitizeFilename(vFileName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters);
|
||||
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
|
||||
);
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Unsupported format: ' || pFormat || '. Use PARQUET or CSV.');
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: ' || pFormat || ')', 'DEBUG', pParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
|
||||
END EXPORT_SINGLE_PARTITION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- MAIN EXPORT PROCEDURES
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE EXPORT_TABLE_DATA (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
-- Type definition for key values
|
||||
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
|
||||
vKeyValues key_value_tab;
|
||||
vCount INTEGER;
|
||||
vSql VARCHAR2(4000);
|
||||
vKeyValue VARCHAR2(4000);
|
||||
vQuery VARCHAR2(32767);
|
||||
vUri VARCHAR2(4000);
|
||||
vDataType VARCHAR2(30);
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters VARCHAR2(4000);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vAllColumnsList VARCHAR2(32767);
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Check if table exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tables
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
|
||||
END IF;
|
||||
|
||||
-- Check if key column exists
|
||||
SELECT COUNT(*) INTO vCount
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
|
||||
|
||||
END IF;
|
||||
|
||||
-- Get the data type of the key column
|
||||
SELECT data_type INTO vDataType
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND column_name = vKeyColumnName
|
||||
AND owner = vSchemaName;
|
||||
|
||||
-- Build list of all columns for the table (excluding key column to avoid duplication)
|
||||
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
|
||||
INTO vAllColumnsList
|
||||
FROM all_tab_columns
|
||||
WHERE table_name = vTableName
|
||||
AND owner = vSchemaName
|
||||
AND column_name != vKeyColumnName;
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(vAllColumnsList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Dynamic column list built (excluding key): ' || vAllColumnsList, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with T. prefix: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
-- Fetch unique key values from A_LOAD_HISTORY
|
||||
vSql := 'SELECT DISTINCT L.A_ETL_LOAD_SET_KEY' ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Executing key values query: ' || vSql, 'DEBUG', vParameters);
|
||||
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValues;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValues.COUNT || ' unique key values to process', 'DEBUG', vParameters);
|
||||
|
||||
-- Loop over each unique key value
|
||||
FOR i IN 1 .. vKeyValues.COUNT LOOP
|
||||
vKeyValue := vKeyValues(i);
|
||||
|
||||
-- Construct the query to extract data for the current key value with A_WORKFLOW_HISTORY_KEY mapping
|
||||
IF vDataType IN ('VARCHAR2', 'CHAR', 'NCHAR', 'NVARCHAR2') THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = ' || CHR(39) || vKeyValue || CHR(39);
|
||||
ELSIF vDataType IN ('NUMBER', 'FLOAT', 'BINARY_FLOAT', 'BINARY_DOUBLE') THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = ' || vKeyValue;
|
||||
ELSIF vDataType LIKE 'TIMESTAMP%' OR vDataType = 'DATE' THEN
|
||||
vQuery := 'SELECT ' || vProcessedColumnList ||
|
||||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
|
||||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
|
||||
' AND L.A_ETL_LOAD_SET_KEY = TO_TIMESTAMP(' || CHR(39) || vKeyValue || CHR(39) ||', ''YYYY-MM-DD HH24:MI:SS.FF'')';
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE);
|
||||
END IF;
|
||||
|
||||
-- Construct the URI for the file in OCI Object Storage
|
||||
vUri := vBucketUri ||
|
||||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
|
||||
sanitizeFilename(vKeyValue) || '.csv';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing key value: ' || vKeyValue || ' (' || (i) || '/' || vKeyValues.COUNT || ')', 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export URI: ' || vUri, 'DEBUG', vParameters);
|
||||
|
||||
-- Use DBMS_CLOUD package to export data to the URI
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => pCredentialName,
|
||||
file_uri_list => vUri,
|
||||
query => vQuery,
|
||||
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
|
||||
);
|
||||
END LOOP;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in column list' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE || ' vDataType: '||vDataType;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vPartitions partition_tab;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
|
||||
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Validate table, key column, and column list using shared procedure
|
||||
VALIDATE_TABLE_AND_COLUMNS(vSchemaName, vTableName, vKeyColumnName, pColumnList, vParameters);
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (building dynamic list from table metadata)'), 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
|
||||
-- Get partitions using shared function
|
||||
vPartitions := GET_PARTITIONS(vSchemaName, vTableName, vKeyColumnName, pMinDate, pMaxDate, vParameters);
|
||||
|
||||
-- Loop over each partition and export using shared worker procedure
|
||||
FOR i IN 1 .. vPartitions.COUNT LOOP
|
||||
EXPORT_SINGLE_PARTITION(
|
||||
pSchemaName => vSchemaName,
|
||||
pTableName => vTableName,
|
||||
pKeyColumnName => vKeyColumnName,
|
||||
pYear => vPartitions(i).year,
|
||||
pMonth => vPartitions(i).month,
|
||||
pBucketUri => vBucketUri,
|
||||
pFolderName => pFolderName,
|
||||
pProcessedColumns => vProcessedColumnList,
|
||||
pMinDate => pMinDate,
|
||||
pMaxDate => pMaxDate,
|
||||
pCredentialName => pCredentialName,
|
||||
pFormat => 'PARQUET',
|
||||
pFileBaseName => NULL,
|
||||
pParameters => vParameters
|
||||
);
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA_BY_DATE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
|
||||
* @desc Exports data to a single CSV file with date filtering.
|
||||
* Unlike EXPORT_TABLE_DATA_BY_DATE, this procedure creates one CSV file
|
||||
* instead of multiple Parquet files partitioned by year/month.
|
||||
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY.
|
||||
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
|
||||
* Validates that all columns in pColumnList exist in the target table.
|
||||
* Automatically adds 'T.' prefix to column names in pColumnList.
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'exports',
|
||||
* pFileName => 'my_export.csv',
|
||||
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
vTableName VARCHAR2(128);
|
||||
vSchemaName VARCHAR2(128);
|
||||
vKeyColumnName VARCHAR2(128);
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vFileBaseName VARCHAR2(4000);
|
||||
vFileExtension VARCHAR2(10);
|
||||
vProcessedColumnList VARCHAR2(32767);
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vCurrentCol VARCHAR2(128);
|
||||
vPartitions partition_tab;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
|
||||
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
|
||||
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
|
||||
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
|
||||
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
|
||||
,'pFileName => '''||nvl(pFileName, 'NULL')||''''
|
||||
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
|
||||
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Get bucket URI based on bucket area using FILE_MANAGER function
|
||||
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
|
||||
|
||||
-- Convert table and column names to uppercase to match data dictionary
|
||||
vTableName := UPPER(pTableName);
|
||||
vSchemaName := UPPER(pSchemaName);
|
||||
vKeyColumnName := UPPER(pKeyColumnName);
|
||||
|
||||
-- Extract base filename and extension or construct default filename
|
||||
IF pFileName IS NOT NULL THEN
|
||||
-- Use provided filename
|
||||
IF INSTR(pFileName, '.') > 0 THEN
|
||||
vFileBaseName := SUBSTR(pFileName, 1, INSTR(pFileName, '.', -1) - 1);
|
||||
vFileExtension := SUBSTR(pFileName, INSTR(pFileName, '.', -1));
|
||||
ELSE
|
||||
vFileBaseName := pFileName;
|
||||
vFileExtension := '.csv';
|
||||
END IF;
|
||||
ELSE
|
||||
-- Construct default filename: TABLENAME (without extension, will be added by worker)
|
||||
vFileBaseName := UPPER(pTableName);
|
||||
vFileExtension := '.csv';
|
||||
END IF;
|
||||
|
||||
-- Validate table, key column, and column list using shared procedure
|
||||
VALIDATE_TABLE_AND_COLUMNS(vSchemaName, vTableName, vKeyColumnName, pColumnList, vParameters);
|
||||
|
||||
-- Process column list to add T. prefix to each column
|
||||
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (using dynamic column list)'), 'DEBUG', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
|
||||
|
||||
-- Get partitions using shared function
|
||||
vPartitions := GET_PARTITIONS(vSchemaName, vTableName, vKeyColumnName, pMinDate, pMaxDate, vParameters);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vPartitions.COUNT || ' year/month combinations to export', 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Date range: ' || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || ' to ' || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'DEBUG', vParameters);
|
||||
|
||||
-- Loop over each partition and export using shared worker procedure
|
||||
FOR i IN 1 .. vPartitions.COUNT LOOP
|
||||
EXPORT_SINGLE_PARTITION(
|
||||
pSchemaName => vSchemaName,
|
||||
pTableName => vTableName,
|
||||
pKeyColumnName => vKeyColumnName,
|
||||
pYear => vPartitions(i).year,
|
||||
pMonth => vPartitions(i).month,
|
||||
pBucketUri => vBucketUri,
|
||||
pFolderName => pFolderName,
|
||||
pProcessedColumns => vProcessedColumnList,
|
||||
pMinDate => pMinDate,
|
||||
pMaxDate => pMaxDate,
|
||||
pCredentialName => pCredentialName,
|
||||
pFormat => 'CSV',
|
||||
pFileBaseName => vFileBaseName,
|
||||
pParameters => vParameters
|
||||
);
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || vPartitions.COUNT || ' files', 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
|
||||
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
|
||||
WHEN OTHERS THEN
|
||||
-- Log complete error details including full stack trace and backtrace
|
||||
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END EXPORT_TABLE_DATA_TO_CSV_BY_DATE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- VERSION MANAGEMENT FUNCTIONS
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN PACKAGE_VERSION;
|
||||
END GET_VERSION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
|
||||
pPackageName => 'DATA_EXPORTER',
|
||||
pVersion => PACKAGE_VERSION,
|
||||
pBuildDate => PACKAGE_BUILD_DATE,
|
||||
pAuthor => PACKAGE_AUTHOR
|
||||
);
|
||||
END GET_BUILD_INFO;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2 IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
|
||||
pPackageName => 'DATA_EXPORTER',
|
||||
pVersionHistory => VERSION_HISTORY
|
||||
);
|
||||
END GET_VERSION_HISTORY;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,183 @@
|
||||
create or replace PACKAGE CT_MRDS.DATA_EXPORTER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* Data Export Package: Provides comprehensive data export capabilities to various formats (CSV, Parquet)
|
||||
* with support for cloud storage integration via Oracle Cloud Infrastructure (OCI).
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Package Version Information
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.2.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2025-12-19 16:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
|
||||
|
||||
-- Version History (last 3-5 changes)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'v2.2.0 (2025-12-19): DRY refactoring - extracted shared helper functions (sanitizeFilename, VALIDATE_TABLE_AND_COLUMNS, GET_PARTITIONS, EXPORT_SINGLE_PARTITION worker procedure). Reduced code duplication by ~400 lines. Prepared architecture for v2.3.0 parallel processing.' || CHR(10) ||
|
||||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
|
||||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
|
||||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10) ||
|
||||
'v1.0.0 (2025-09-15): Initial implementation within FILE_MANAGER package' || CHR(10);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- TYPE DEFINITIONS FOR PARTITION HANDLING
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Record type for year/month partition information
|
||||
**/
|
||||
TYPE partition_rec IS RECORD (
|
||||
year VARCHAR2(4),
|
||||
month VARCHAR2(2)
|
||||
);
|
||||
|
||||
/**
|
||||
* Table type for collection of partition records
|
||||
**/
|
||||
TYPE partition_tab IS TABLE OF partition_rec;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data into CSV file on OCI infrustructure.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'csv_exports'
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_BY_DATE
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data into PARQUET files on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
|
||||
* Validates that all columns in pColumnList exist in the target table.
|
||||
* Automatically adds 'T.' prefix to column names in pColumnList.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'parquet_exports',
|
||||
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
|
||||
* @desc Exports data to separate CSV files partitioned by year and month.
|
||||
* Creates one CSV file for each year/month combination found in the data.
|
||||
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY as EXPORT_TABLE_DATA_BY_DATE,
|
||||
* but exports to CSV format instead of Parquet.
|
||||
* File naming pattern: {pFileName}_YYYYMM.csv or {TABLENAME}_YYYYMM.csv (if pFileName is NULL)
|
||||
* @example
|
||||
* begin
|
||||
* -- With custom filename
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'CT_MRDS',
|
||||
* pTableName => 'MY_TABLE',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'DATA',
|
||||
* pFolderName => 'exports',
|
||||
* pFileName => 'my_export.csv',
|
||||
* pMinDate => DATE '2024-01-01',
|
||||
* pMaxDate => SYSDATE
|
||||
* );
|
||||
*
|
||||
* -- With auto-generated filename (based on table name only)
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
* pSchemaName => 'OU_TOP',
|
||||
* pTableName => 'AGGREGATED_ALLOTMENT',
|
||||
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
* pBucketArea => 'ARCHIVE',
|
||||
* pFolderName => 'exports',
|
||||
* pMinDate => DATE '2025-09-01',
|
||||
* pMaxDate => DATE '2025-09-17'
|
||||
* );
|
||||
* -- This will create files like: AGGREGATED_ALLOTMENT_202509.csv, etc.
|
||||
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
|
||||
pSchemaName IN VARCHAR2,
|
||||
pTableName IN VARCHAR2,
|
||||
pKeyColumnName IN VARCHAR2,
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderName IN VARCHAR2,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pColumnList IN VARCHAR2 default NULL,
|
||||
pMinDate IN DATE default DATE '1900-01-01',
|
||||
pMaxDate IN DATE default SYSDATE,
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns the current package version number
|
||||
* return: Version string in format X.Y.Z (e.g., '2.1.0')
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* Returns comprehensive build information including version, date, and author
|
||||
* return: Formatted string with complete build details
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* Returns the version history with recent changes
|
||||
* return: Multi-line string with version history
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,77 @@
|
||||
-- ===================================================================
|
||||
-- MARS-835-PREHOOK ROLLBACK SCRIPT: DRY Refactoring for DATA_EXPORTER (Rollback)
|
||||
-- ===================================================================
|
||||
-- Purpose: Pre-hook rollback for MARS-835 - Rollback DATA_EXPORTER to v2.1.1 (restore pre-refactoring code)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-19
|
||||
-- Version: 2.1.1 (rollback)
|
||||
|
||||
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
|
||||
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
|
||||
host mkdir log 2>nul
|
||||
|
||||
var filename VARCHAR2(100)
|
||||
BEGIN
|
||||
:filename := 'log/ROLLBACK_MARS_835_PREHOOK_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
|
||||
END;
|
||||
/
|
||||
column filename new_value _filename
|
||||
select :filename filename from dual;
|
||||
spool &_filename
|
||||
|
||||
SET ECHO OFF
|
||||
SET TIMING ON
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET PAUSE OFF
|
||||
|
||||
PROMPT =========================================================================
|
||||
PROMPT MARS-835-PREHOOK: Rollback Package
|
||||
PROMPT =========================================================================
|
||||
PROMPT WARNING: This will reverse all changes from MARS-835-PREHOOK installation!
|
||||
PROMPT WARNING: DRY refactoring will be reverted to v2.1.1!
|
||||
PROMPT =========================================================================
|
||||
|
||||
-- Confirm rollback with user
|
||||
ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: '
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
BEGIN
|
||||
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20999, 'Rollback aborted by user.');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
-- Execute rollback scripts in REVERSE order (body first, then spec)
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 1: Rollback DATA_EXPORTER Package Body to v2.1.1
|
||||
PROMPT =========================================================================
|
||||
@@91_MARS_835_rollback_DATA_EXPORTER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 2: Rollback DATA_EXPORTER Package Specification to v2.1.1
|
||||
PROMPT =========================================================================
|
||||
@@92_MARS_835_rollback_DATA_EXPORTER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 3: Track Rollback Version
|
||||
PROMPT =========================================================================
|
||||
@@track_package_versions.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT Step 4: Verify Package Versions
|
||||
PROMPT =========================================================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT =========================================================================
|
||||
PROMPT MARS-835-PREHOOK Rollback - COMPLETED
|
||||
PROMPT =========================================================================
|
||||
|
||||
spool off
|
||||
|
||||
quit;
|
||||
@@ -0,0 +1,96 @@
|
||||
-- ===================================================================
|
||||
-- Simple Package Version Tracking Script
|
||||
-- ===================================================================
|
||||
-- Purpose: Track specified Oracle package versions
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-04
|
||||
-- Version: 3.1.0 - List-Based Edition
|
||||
--
|
||||
-- USAGE:
|
||||
-- 1. Edit package list below (add/remove packages as needed)
|
||||
-- 2. Include in your install/rollback script: @@track_package_versions.sql
|
||||
-- ===================================================================
|
||||
|
||||
SET SERVEROUTPUT ON;
|
||||
|
||||
DECLARE
|
||||
TYPE t_package_rec IS RECORD (
|
||||
owner VARCHAR2(50),
|
||||
package_name VARCHAR2(50),
|
||||
version VARCHAR2(50)
|
||||
);
|
||||
TYPE t_packages IS TABLE OF t_package_rec;
|
||||
TYPE t_string_array IS TABLE OF VARCHAR2(100);
|
||||
|
||||
-- ===================================================================
|
||||
-- PACKAGE LIST - Edit this array to specify packages to track
|
||||
-- ===================================================================
|
||||
-- Add or remove entries as needed for your MARS issue
|
||||
-- Format: 'SCHEMA.PACKAGE_NAME'
|
||||
-- ===================================================================
|
||||
vPackageList t_string_array := t_string_array(
|
||||
'CT_MRDS.DATA_EXPORTER'
|
||||
);
|
||||
-- ===================================================================
|
||||
|
||||
vPackages t_packages := t_packages();
|
||||
vVersion VARCHAR2(50);
|
||||
vCount NUMBER := 0;
|
||||
vOwner VARCHAR2(50);
|
||||
vPackageName VARCHAR2(50);
|
||||
vDotPos NUMBER;
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('========================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Package Version Tracking');
|
||||
DBMS_OUTPUT.PUT_LINE('========================================');
|
||||
|
||||
-- Process each package in the list
|
||||
FOR i IN 1..vPackageList.COUNT LOOP
|
||||
vDotPos := INSTR(vPackageList(i), '.');
|
||||
IF vDotPos > 0 THEN
|
||||
vOwner := SUBSTR(vPackageList(i), 1, vDotPos - 1);
|
||||
vPackageName := SUBSTR(vPackageList(i), vDotPos + 1);
|
||||
|
||||
-- Get package version
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT ' || vOwner || '.' || vPackageName || '.GET_VERSION() FROM DUAL' INTO vVersion;
|
||||
vPackages.EXTEND;
|
||||
vPackages(vPackages.COUNT).owner := vOwner;
|
||||
vPackages(vPackages.COUNT).package_name := vPackageName;
|
||||
vPackages(vPackages.COUNT).version := vVersion;
|
||||
|
||||
-- Track in ENV_MANAGER
|
||||
BEGIN
|
||||
CT_MRDS.ENV_MANAGER.TRACK_PACKAGE_VERSION(
|
||||
pPackageOwner => vOwner,
|
||||
pPackageName => vPackageName,
|
||||
pPackageVersion => vVersion,
|
||||
pPackageBuildDate => TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'),
|
||||
pPackageAuthor => 'Grzegorz Michalski'
|
||||
);
|
||||
vCount := vCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN NULL; -- Continue even if tracking fails
|
||||
END;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN NULL; -- Skip packages that fail
|
||||
END;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Summary:');
|
||||
DBMS_OUTPUT.PUT_LINE('--------');
|
||||
DBMS_OUTPUT.PUT_LINE('Packages tracked: ' || vCount || '/' || vPackageList.COUNT);
|
||||
|
||||
IF vPackages.COUNT > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Tracked Packages:');
|
||||
FOR i IN 1..vPackages.COUNT LOOP
|
||||
DBMS_OUTPUT.PUT_LINE(' ' || vPackages(i).owner || '.' || vPackages(i).package_name || ' v' || vPackages(i).version);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('========================================');
|
||||
END;
|
||||
/
|
||||
@@ -0,0 +1,62 @@
|
||||
-- ===================================================================
|
||||
-- Universal Package Version Verification Script
|
||||
-- ===================================================================
|
||||
-- Purpose: Verify all tracked Oracle packages for code changes
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-04
|
||||
-- Version: 1.0.0
|
||||
--
|
||||
-- USAGE:
|
||||
-- Include at the end of install/rollback scripts: @@verify_packages_version.sql
|
||||
--
|
||||
-- OUTPUT:
|
||||
-- - List of all tracked packages with their current status
|
||||
-- - OK: Package has not changed since last tracking
|
||||
-- - WARNING: Package code changed without version update
|
||||
-- ===================================================================
|
||||
|
||||
SET LINESIZE 200
|
||||
SET PAGESIZE 1000
|
||||
SET FEEDBACK OFF
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Package Version Verification
|
||||
PROMPT ========================================
|
||||
PROMPT
|
||||
|
||||
COLUMN PACKAGE_OWNER FORMAT A15
|
||||
COLUMN PACKAGE_NAME FORMAT A20
|
||||
COLUMN VERSION FORMAT A10
|
||||
COLUMN STATUS FORMAT A80
|
||||
|
||||
SELECT
|
||||
PACKAGE_OWNER,
|
||||
PACKAGE_NAME,
|
||||
PACKAGE_VERSION AS VERSION,
|
||||
CT_MRDS.ENV_MANAGER.CHECK_PACKAGE_CHANGES(PACKAGE_OWNER, PACKAGE_NAME) AS STATUS
|
||||
FROM (
|
||||
SELECT
|
||||
PACKAGE_OWNER,
|
||||
PACKAGE_NAME,
|
||||
PACKAGE_VERSION,
|
||||
ROW_NUMBER() OVER (PARTITION BY PACKAGE_OWNER, PACKAGE_NAME ORDER BY TRACKING_DATE DESC) AS RN
|
||||
FROM CT_MRDS.A_PACKAGE_VERSION_TRACKING
|
||||
)
|
||||
WHERE RN = 1
|
||||
ORDER BY PACKAGE_OWNER, PACKAGE_NAME;
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Verification Complete
|
||||
PROMPT ========================================
|
||||
PROMPT
|
||||
PROMPT Legend:
|
||||
PROMPT OK - Package has not changed since last tracking
|
||||
PROMPT WARNING - Package code changed without version update
|
||||
PROMPT
|
||||
PROMPT For detailed hash information, use:
|
||||
PROMPT SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL;
|
||||
PROMPT ========================================
|
||||
|
||||
SET FEEDBACK ON
|
||||
Reference in New Issue
Block a user