Compare commits
16 Commits
b85172ae84
...
develop
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9d4056451 | ||
|
|
60b218d211 | ||
|
|
819b6f7880 | ||
|
|
c68d5bfe2c | ||
|
|
c607bbe26e | ||
|
|
1569237306 | ||
|
|
472a724fe0 | ||
|
|
04d4f6ac02 | ||
|
|
ca5d8b320c | ||
|
|
2605896469 | ||
|
|
b588b0bb72 | ||
|
|
6060f93fde | ||
|
|
99aca3af40 | ||
|
|
1089184367 | ||
|
|
e538706896 | ||
|
|
ff034fcd68 |
@@ -25,6 +25,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported');
|
||||
@@ -46,6 +47,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported');
|
||||
@@ -67,6 +69,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported');
|
||||
|
||||
@@ -30,6 +30,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER',
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported');
|
||||
@@ -51,6 +52,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM',
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported');
|
||||
|
||||
@@ -25,6 +25,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported');
|
||||
@@ -46,6 +47,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported');
|
||||
@@ -67,6 +69,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported');
|
||||
|
||||
@@ -30,6 +30,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES',
|
||||
pParallelDegree => 8,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported');
|
||||
@@ -51,6 +52,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER',
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported');
|
||||
|
||||
@@ -26,6 +26,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER',
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported');
|
||||
@@ -47,6 +48,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM',
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported');
|
||||
|
||||
@@ -30,6 +30,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER',
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported');
|
||||
@@ -51,6 +52,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM',
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported');
|
||||
|
||||
@@ -25,6 +25,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported');
|
||||
@@ -46,6 +47,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported');
|
||||
@@ -66,8 +68,7 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 2,
|
||||
pJobClass => 'high'
|
||||
pParallelDegree => 2, pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported');
|
||||
EXCEPTION
|
||||
|
||||
@@ -25,6 +25,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported');
|
||||
@@ -46,6 +47,7 @@ BEGIN
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM',
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported');
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
-- =====================================================================
|
||||
-- Script: 01b_MARS_828_add_column_comments.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Purpose: Add comprehensive column comments for A_SOURCE_FILE_CONFIG and A_SOURCE_FILE_RECEIVED tables
|
||||
-- Description: Documents all columns to improve database maintainability and user understanding
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Adding comprehensive column comments
|
||||
PROMPT ========================================
|
||||
|
||||
-- =====================================================================
|
||||
-- A_SOURCE_FILE_CONFIG Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Adding column comments for A_SOURCE_FILE_CONFIG...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Primary key - unique identifier for source file configuration record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
|
||||
'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
|
||||
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
|
||||
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
|
||||
'Human-readable description of the source file and its purpose';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
|
||||
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
|
||||
'Identifier for the target table where data will be loaded (without schema prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
|
||||
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
|
||||
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
|
||||
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
|
||||
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
|
||||
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
|
||||
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
|
||||
'Schema name where ODS external tables are created (typically ODS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
|
||||
'Number of hours before table statistics expire and need to be recalculated';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
|
||||
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
|
||||
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
-- =====================================================================
|
||||
-- A_SOURCE_FILE_RECEIVED Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Adding column comments for A_SOURCE_FILE_RECEIVED...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS
|
||||
'Primary key - unique identifier for received file record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS
|
||||
'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS
|
||||
'MD5 checksum of file content for integrity verification and duplicate detection';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS
|
||||
'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS
|
||||
'File size in bytes';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS
|
||||
'Date when file was registered in the system (extracted from CREATED timestamp)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS
|
||||
'Name of temporary external table created for file validation (dropped after validation)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS
|
||||
'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS
|
||||
'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS
|
||||
'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)';
|
||||
|
||||
-- =====================================================================
|
||||
-- Verification
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT Verifying column comments...
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
COUNT(*) as total_columns,
|
||||
COUNT(comments) as documented_columns,
|
||||
COUNT(*) - COUNT(comments) as undocumented_columns
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
GROUP BY table_name
|
||||
ORDER BY table_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT Detailed column documentation status:
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
CASE WHEN comments IS NULL THEN 'MISSING' ELSE 'OK' END as comment_status
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
ORDER BY table_name, column_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Column comments added successfully
|
||||
PROMPT ========================================
|
||||
PROMPT A_SOURCE_FILE_CONFIG: All 20 columns documented
|
||||
PROMPT A_SOURCE_FILE_RECEIVED: All 12 columns documented
|
||||
PROMPT ========================================
|
||||
@@ -59,9 +59,23 @@ WHERE owner = 'CT_MRDS'
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_type;
|
||||
|
||||
-- 5. Check for compilation errors
|
||||
-- 5. Check FILE_MANAGER package compilation status
|
||||
PROMPT
|
||||
PROMPT 5. Checking for compilation errors...
|
||||
PROMPT 5. Checking FILE_MANAGER package status...
|
||||
SELECT
|
||||
object_name,
|
||||
object_type,
|
||||
status,
|
||||
TO_CHAR(last_ddl_time, 'YYYY-MM-DD HH24:MI:SS') as last_ddl_time
|
||||
FROM all_objects
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name = 'FILE_MANAGER'
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_type;
|
||||
|
||||
-- 6. Check for compilation errors
|
||||
PROMPT
|
||||
PROMPT 6. Checking for compilation errors (FILE_ARCHIVER)...
|
||||
SELECT
|
||||
name,
|
||||
type,
|
||||
@@ -73,14 +87,31 @@ WHERE owner = 'CT_MRDS'
|
||||
AND name = 'FILE_ARCHIVER'
|
||||
ORDER BY type, sequence;
|
||||
|
||||
-- 6. Verify package version
|
||||
-- 7. Check for compilation errors (FILE_MANAGER)
|
||||
PROMPT
|
||||
PROMPT 6. Verifying FILE_ARCHIVER version...
|
||||
SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() as package_version FROM DUAL;
|
||||
PROMPT 7. Checking for compilation errors (FILE_MANAGER)...
|
||||
SELECT
|
||||
name,
|
||||
type,
|
||||
line,
|
||||
position,
|
||||
text
|
||||
FROM all_errors
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND name = 'FILE_MANAGER'
|
||||
ORDER BY type, sequence;
|
||||
|
||||
-- 7. Test trigger validation
|
||||
-- 8. Verify package versions
|
||||
PROMPT
|
||||
PROMPT 7. Testing trigger validation (should fail)...
|
||||
PROMPT 8. Verifying package versions...
|
||||
PROMPT FILE_ARCHIVER version:
|
||||
SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() as package_version FROM DUAL;
|
||||
PROMPT FILE_MANAGER version:
|
||||
SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() as package_version FROM DUAL;
|
||||
|
||||
-- 9. Test trigger validation
|
||||
PROMPT
|
||||
PROMPT 9. Testing trigger validation (should fail)...
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
SET SERVEROUTPUT ON
|
||||
DECLARE
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
--
|
||||
-- Configuration by group:
|
||||
-- - 19 LM tables: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 100K rows OR 1GB, 24h stats
|
||||
-- - 2 CSDB DEBT: MINIMUM_AGE_MONTHS=6, 5 files OR 50K rows OR 512MB, 48h stats
|
||||
-- - 2 CSDB DEBT: MINIMUM_AGE_MONTHS=0 (current month only), 5 files OR 50K rows OR 512MB, 48h stats
|
||||
-- - 4 CSDB ratings: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 20K rows OR 256MB, 72h stats
|
||||
--
|
||||
-- Dependencies:
|
||||
@@ -33,7 +33,7 @@ PROMPT - Triggers: 10 files OR 100,000 rows OR 1 GB
|
||||
PROMPT - Stats Expiration: 24 hours
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT Tables (2):
|
||||
PROMPT - Strategy: MINIMUM_AGE_MONTHS = 6
|
||||
PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only)
|
||||
PROMPT - Triggers: 5 files OR 50,000 rows OR 512 MB
|
||||
PROMPT - Stats Expiration: 48 hours
|
||||
PROMPT
|
||||
@@ -92,16 +92,16 @@ PROMPT LM tables configuration completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 2: CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 6)
|
||||
PROMPT SECTION 2: CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 0)
|
||||
PROMPT =====================================================================
|
||||
PROMPT Thresholds: 5 files OR 50K rows OR 512MB
|
||||
PROMPT Stats expire: 48 hours
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Update CSDB DEBT tables (6-month retention)
|
||||
-- Update CSDB DEBT tables (current month only)
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
|
||||
MINIMUM_AGE_MONTHS = 6,
|
||||
MINIMUM_AGE_MONTHS = 0,
|
||||
ODS_SCHEMA_NAME = 'ODS',
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = 5,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = 50000,
|
||||
@@ -195,7 +195,7 @@ WHERE A_SOURCE_KEY = 'LM'
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 6):
|
||||
PROMPT CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 0):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
@@ -210,7 +210,7 @@ SELECT
|
||||
IS_KEEP_IN_TRASH,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
|
||||
AND MINIMUM_AGE_MONTHS = 6
|
||||
AND MINIMUM_AGE_MONTHS = 0
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT = 5
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT = 50000
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM = 536870912
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_MANAGER Package Specification v3.3.2
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_MANAGER Package Specification with MARS-828 column compatibility
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Related: MARS-828 Threshold Column Rename Compatibility
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_MANAGER Package Specification v3.3.2
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_MANAGER.pkg
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_MANAGER'
|
||||
AND OBJECT_TYPE = 'PACKAGE';
|
||||
|
||||
PROMPT SUCCESS: FILE_MANAGER Package Specification v3.3.2 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -0,0 +1,38 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_MANAGER Package Body v3.3.2
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_MANAGER Package Body with MARS-828 threshold column compatibility
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Related: MARS-828 Threshold Column Rename Compatibility
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_MANAGER Package Body v3.3.2
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_MANAGER.pkb
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_MANAGER'
|
||||
AND OBJECT_TYPE IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY OBJECT_TYPE;
|
||||
|
||||
-- Check for any compilation errors
|
||||
SELECT 'COMPILATION ERRORS FOUND' AS WARNING
|
||||
FROM ALL_ERRORS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND NAME = 'FILE_MANAGER'
|
||||
AND TYPE = 'PACKAGE BODY'
|
||||
AND ROWNUM = 1;
|
||||
|
||||
PROMPT SUCCESS: FILE_MANAGER Package Body v3.3.2 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -20,7 +20,10 @@ DROP CONSTRAINT CHK_IS_KEEP_IN_TRASH;
|
||||
-- Drop columns
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
MINIMUM_AGE_MONTHS
|
||||
);
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH
|
||||
);
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
-- =====================================================================
|
||||
-- Script: 94b_MARS_828_rollback_column_comments.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Purpose: Remove column comments added by 01b_MARS_828_add_column_comments.sql
|
||||
-- Description: Optional rollback - removes documentation but does not affect functionality
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Removing column comments (optional)
|
||||
PROMPT ========================================
|
||||
|
||||
-- =====================================================================
|
||||
-- Remove A_SOURCE_FILE_CONFIG Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Removing column comments from A_SOURCE_FILE_CONFIG...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS '';
|
||||
|
||||
-- =====================================================================
|
||||
-- Remove A_SOURCE_FILE_RECEIVED Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Removing column comments from A_SOURCE_FILE_RECEIVED...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS '';
|
||||
|
||||
-- =====================================================================
|
||||
-- Verification
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT Verifying column comments removed...
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
COUNT(*) as total_columns,
|
||||
COUNT(CASE WHEN comments IS NOT NULL AND LENGTH(comments) > 0 THEN 1 END) as documented_columns
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
GROUP BY table_name
|
||||
ORDER BY table_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Column comments removed successfully
|
||||
PROMPT ========================================
|
||||
PROMPT NOTE: This is an optional rollback step
|
||||
PROMPT Database functionality is not affected
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,10 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_MANAGER Package Specification to v3.3.1
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package specification version (pre-threshold column rename compatibility)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- WARNING: This removes MARS-828 threshold column compatibility from FILE_MANAGER
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/FILE_MANAGER.pkg
|
||||
@@ -0,0 +1,10 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_MANAGER Package Body to v3.3.1
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package body version (pre-threshold column rename compatibility)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- WARNING: This removes MARS-828 threshold column compatibility from FILE_MANAGER
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/FILE_MANAGER.pkb
|
||||
@@ -35,10 +35,10 @@ PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT MARS-828 Installation Starting
|
||||
PROMPT ============================================================================
|
||||
PROMPT Package: CT_MRDS.FILE_ARCHIVER
|
||||
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving
|
||||
PROMPT Package: CT_MRDS.FILE_ARCHIVER v3.3.0 + CT_MRDS.FILE_MANAGER v3.3.2
|
||||
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving + FILE_MANAGER compatibility
|
||||
PROMPT Purpose: Flexible archival policies per data source with file retention and config-based control
|
||||
PROMPT Steps: 11 (DDL, Rename Columns, Trigger, Statuses, Grants, Package v3.3.0, Verify, Track, Configure)
|
||||
PROMPT Steps: 14 (DDL, Rename, Comments, Trigger, Statuses, Grants, Packages, Verify, Track, Configure)
|
||||
PROMPT Timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
|
||||
PROMPT ============================================================================
|
||||
@@ -56,57 +56,72 @@ WHENEVER SQLERROR CONTINUE
|
||||
|
||||
-- Installation steps
|
||||
PROMPT
|
||||
PROMPT Step 1/11: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG
|
||||
PROMPT Step 1/12: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG
|
||||
PROMPT ==============================================================================
|
||||
@@01_MARS_828_install_add_archival_strategy_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 2/11: Renaming threshold columns for consistent naming
|
||||
PROMPT Step 2/12: Renaming threshold columns for consistent naming
|
||||
PROMPT ==========================================================
|
||||
@@01a_MARS_828_rename_threshold_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 3/11: Creating validation trigger
|
||||
PROMPT Step 3/12: Adding comprehensive column comments
|
||||
PROMPT ===============================================
|
||||
@@01b_MARS_828_add_column_comments.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 4/12: Creating validation trigger
|
||||
PROMPT ======================================
|
||||
@@02_MARS_828_install_archival_strategy_trigger.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 4/11: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED
|
||||
PROMPT Step 5/12: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED
|
||||
PROMPT ===================================================================
|
||||
@@07_MARS_828_install_add_trash_retention_statuses.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 5/11: Granting privileges on T_FILENAME to MRDS_LOADER
|
||||
PROMPT Step 6/12: Granting privileges on T_FILENAME to MRDS_LOADER
|
||||
PROMPT ==========================================================
|
||||
@@08_MARS_828_install_grant_t_filename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 6/11: Deploying FILE_ARCHIVER Package Specification v3.3.0
|
||||
PROMPT Step 7/12: Deploying FILE_ARCHIVER Package Specification v3.3.0
|
||||
PROMPT ==============================================================
|
||||
@@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 7/11: Deploying FILE_ARCHIVER Package Body v3.3.0
|
||||
PROMPT Step 8/14: Deploying FILE_ARCHIVER Package Body v3.3.0
|
||||
PROMPT ====================================================
|
||||
@@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 8/11: Verifying installation
|
||||
PROMPT =================================
|
||||
PROMPT Step 9/14: Deploying FILE_MANAGER Package Specification v3.3.2
|
||||
PROMPT =============================================================
|
||||
@@09_MARS_828_install_CT_MRDS_FILE_MANAGER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 10/14: Deploying FILE_MANAGER Package Body v3.3.2
|
||||
PROMPT ===================================================
|
||||
@@10_MARS_828_install_CT_MRDS_FILE_MANAGER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 11/14: Verifying installation
|
||||
PROMPT ==================================
|
||||
@@05_MARS_828_verify_installation.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 9/11: Tracking package versions
|
||||
PROMPT ====================================
|
||||
PROMPT Step 12/14: Tracking package versions
|
||||
PROMPT =====================================
|
||||
@@track_package_versions.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 10/11: Verifying tracked packages
|
||||
PROMPT Step 13/14: Verifying tracked packages
|
||||
PROMPT ======================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 11/11: Configuring Release 01 tables archival strategies
|
||||
PROMPT Step 14/14: Configuring Release 01 tables archival strategies
|
||||
PROMPT ============================================================
|
||||
@@06_MARS_828_configure_release01_tables.sql
|
||||
|
||||
@@ -118,8 +133,9 @@ PROMPT Completion Time:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT Installation Summary:
|
||||
PROMPT - Package: CT_MRDS.FILE_ARCHIVER
|
||||
PROMPT - Version: 3.3.0 (includes selective archiving and config-based TRASH policy)
|
||||
PROMPT - Packages Installed:
|
||||
PROMPT * CT_MRDS.FILE_ARCHIVER v3.3.0 (includes selective archiving and config-based TRASH policy)
|
||||
PROMPT * CT_MRDS.FILE_MANAGER v3.3.2 (compatible with MARS-828 threshold column renames)
|
||||
PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID
|
||||
PROMPT - Selective Archiving: IS_ARCHIVE_ENABLED column (Y=archive, N=skip)
|
||||
PROMPT - TRASH Policy: IS_KEEP_IN_TRASH column (Y=keep files, N=delete immediately)
|
||||
|
||||
@@ -47,10 +47,64 @@ ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABL
|
||||
TABLESPACE "DATA";
|
||||
|
||||
-- Column comments
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Primary key - unique identifier for source file configuration record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
|
||||
'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
|
||||
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
|
||||
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
|
||||
'Human-readable description of the source file and its purpose';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
|
||||
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
|
||||
'Identifier for the target table where data will be loaded (without schema prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
|
||||
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
|
||||
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
|
||||
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
|
||||
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
|
||||
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
|
||||
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
|
||||
'Schema name where ODS external tables are created (typically ODS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
|
||||
'Number of hours before table statistics expire and need to be recalculated';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
|
||||
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
|
||||
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;
|
||||
@@ -26,4 +26,41 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
|
||||
CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1
|
||||
ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES);
|
||||
|
||||
-- Column comments
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS
|
||||
'Primary key - unique identifier for received file record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS
|
||||
'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS
|
||||
'MD5 checksum of file content for integrity verification and duplicate detection';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS
|
||||
'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS
|
||||
'File size in bytes';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS
|
||||
'Date when file was registered in the system (extracted from CREATED timestamp)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS
|
||||
'Name of temporary external table created for file validation (dropped after validation)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS
|
||||
'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS
|
||||
'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_FILE_NAME IS
|
||||
'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)';
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE;
|
||||
@@ -166,6 +166,7 @@ AS
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || '
|
||||
and h.WORKFLOW_SUCCESSFUL = ''Y''
|
||||
group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')'
|
||||
;
|
||||
|
||||
@@ -182,11 +183,11 @@ AS
|
||||
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
on s.file$name = r.source_file_name
|
||||
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
|
||||
and r.PROCESSING_STATUS = ''INGESTED''
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||'''
|
||||
and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||'''
|
||||
and h.WORKFLOW_SUCCESSFUL = ''Y''
|
||||
'
|
||||
;
|
||||
vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||'ARCHIVE/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
|
||||
|
||||
2009
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_MANAGER.pkb
Normal file
2009
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_MANAGER.pkb
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,639 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_MANAGER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) ||
|
||||
'3.3.2 (2026-02-20): MARS-828 - Fixed threshold column names in GET_DET_SOURCE_FILE_CONFIG_INFO for MARS-828 compatibility' || CHR(13)||CHR(10) ||
|
||||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) ||
|
||||
'3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) ||
|
||||
'2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-01): Initial release with file processing and validation capabilities';
|
||||
|
||||
TYPE tSourceFileReceived IS RECORD
|
||||
(
|
||||
A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE,
|
||||
A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
SOURCE_FILE_PREFIX_INBOX VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ODS VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430),
|
||||
SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE,
|
||||
RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE,
|
||||
PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE,
|
||||
EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE
|
||||
);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgSourceFileConfigKey PLS_INTEGER;
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
|
||||
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG
|
||||
* @desc Get source file type by matching the source file name against source file type naming patterns
|
||||
* or by specifying the id of a received source file.
|
||||
* @example ...
|
||||
* @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE"
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL
|
||||
, pSourceFileReceivedKey IN NUMBER DEFAULT NULL
|
||||
, pSourceFileConfigKey IN NUMBER DEFAULT NULL)
|
||||
RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table.
|
||||
* This overload automatically determines source file type from the file name.
|
||||
* It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig.
|
||||
* Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(
|
||||
* pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'
|
||||
* ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... );
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2,
|
||||
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name SET_SOURCE_FILE_RECEIVED_STATUS
|
||||
* @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column
|
||||
* based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY
|
||||
* and provided value of pStatus parameter
|
||||
* @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION');
|
||||
**/
|
||||
PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(
|
||||
pSourceFileReceivedKey IN PLS_INTEGER,
|
||||
pStatus IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_EXTERNAL_TABLE_COLUMNS
|
||||
* @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name.
|
||||
* It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure.
|
||||
* @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual;
|
||||
* @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "REV_NUMBER" NUMBER(28,0),
|
||||
* "REF_DATE" DATE,
|
||||
* "FREE_TEXT" VARCHAR2(1000 CHAR),
|
||||
* "MLF_BS_TOTAL" NUMBER(28,10),
|
||||
* "DF_BS_TOTAL" NUMBER(28,10),
|
||||
* "MLF_SF_TOTAL" NUMBER(28,10),
|
||||
* "DF_SF_TOTAL" NUMBER(28,10)
|
||||
**/
|
||||
FUNCTION GET_EXTERNAL_TABLE_COLUMNS (
|
||||
pTargetTableTemplate IN VARCHAR2
|
||||
)
|
||||
RETURN CLOB;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252')
|
||||
* If provided, adds CHARACTERSET clause to external table definition
|
||||
* @example
|
||||
* begin
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLE(
|
||||
* pTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
|
||||
* pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/',
|
||||
* pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/',
|
||||
* pFileName => NULL,
|
||||
* pDelimiter => ',',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pTableName IN VARCHAR2,
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pPrefix IN VARCHAR2,
|
||||
pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pDelimiter IN VARCHAR2 DEFAULT ',',
|
||||
pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc Creates External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_SOURCE_FILE_RECEIVED
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
|
||||
* It validate External table build upon single file
|
||||
* provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED
|
||||
(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE.
|
||||
* It validates External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt FAILED
|
||||
**/
|
||||
FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
/**
|
||||
* @name S_VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A function which checks if SELECT query reterns any rows.
|
||||
* It trys to selects External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt PASSED
|
||||
**/
|
||||
FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DROP_EXTERNAL_TABLE
|
||||
* @desc It drops External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE DROP_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name COPY_FILE
|
||||
* @desc It copies file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS'
|
||||
* @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE COPY_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name MOVE_FILE
|
||||
* @desc It moves file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS', 'QUARANTINE'
|
||||
* @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE MOVE_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_FOLDER_CONTENTS
|
||||
* @desc It deletes all files from specified folder in the cloud storage.
|
||||
* The procedure lists all objects in the specified folder prefix and deletes them one by one.
|
||||
* pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE'
|
||||
* pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/')
|
||||
* @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/');
|
||||
**/
|
||||
PROCEDURE DELETE_FOLDER_CONTENTS(
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderPrefix IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter.
|
||||
* Ubmrella procedure that calls:
|
||||
* - REGISTER_SOURCE_FILE_RECEIVED;
|
||||
* - CREATE_EXTERNAL_TABLE;
|
||||
* - VALIDATE_SOURCE_FILE_RECEIVED;
|
||||
* - DROP_EXTERNAL_TABLE;
|
||||
* - MOVE_FILE;
|
||||
* @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
**/
|
||||
PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter and return processing result value.
|
||||
* It returns (success/failure) => 0 / -(value).
|
||||
* Ubmrella function that calls PROCESS_SOURCE_FILE procedure.
|
||||
* @example
|
||||
* declare
|
||||
* vResult PLS_INTEGER;
|
||||
* begin
|
||||
* vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult);
|
||||
* end;
|
||||
* @ex_rslt 0
|
||||
* -20021
|
||||
**/
|
||||
FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DATE_FORMAT
|
||||
* @desc Returns date format for specified template table name and column name.
|
||||
* Date is taken from configuration A_COLUMN_DATE_FORMAT table.
|
||||
* @example select FILE_MANAGER.GET_DATE_FORMAT(
|
||||
* pTemplateTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pColumnName => 'SNAPSHOT_DATE')
|
||||
* from dual;
|
||||
* @ex_rslt DD/MM/YYYY HH24:MI:SS
|
||||
**/
|
||||
FUNCTION GET_DATE_FORMAT(
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnName IN VARCHAR2
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GENERATE_EXTERNAL_TABLE_PARAMS
|
||||
* @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName.
|
||||
* @example
|
||||
* declare
|
||||
* vColumnList CLOB;
|
||||
* vFieldList CLOB;
|
||||
* begin
|
||||
* FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER'
|
||||
* ,pColumnList => vColumnList
|
||||
* ,pFieldList => vFieldList
|
||||
* );
|
||||
* DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList);
|
||||
* DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList);
|
||||
* end;
|
||||
* /
|
||||
**/
|
||||
PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnList OUT CLOB,
|
||||
pFieldList OUT CLOB
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE
|
||||
* @desc Insert a new record to A_SOURCE table.
|
||||
* pSourceKey is a PRIMARY KEY value.
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE,
|
||||
pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_SOURCE_CASCADE
|
||||
* @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables:
|
||||
* - A_SOURCE_FILE_CONFIG
|
||||
* - A_SOURCE_FILE_RECEIVED
|
||||
* - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems)
|
||||
* The procedure checks if template tables are shared before deleting date format configurations.
|
||||
* If a template table is used by multiple source systems, date formats are preserved.
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS');
|
||||
**/
|
||||
PROCEDURE DELETE_SOURCE_CASCADE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID)
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record.
|
||||
* @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(
|
||||
* pSourceFileId => 'UC_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified input parameters,
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY.
|
||||
* @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY (
|
||||
* pSourceFileType => 'INPUT'
|
||||
* ,pSourceFileId => 'UC_DISSEM'
|
||||
* ,pTableId => 'UC_NMA_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT'
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE_FILE_CONFIG
|
||||
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification.
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
|
||||
* If NULL, no CHARACTERSET clause is added to external table definitions
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
|
||||
* pSourceKey => 'C2D', pSourceFileType => 'INPUT',
|
||||
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE_FILE_CONFIG (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE
|
||||
,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE
|
||||
,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
|
||||
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
|
||||
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
|
||||
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_COLUMN_DATE_FORMAT
|
||||
* @desc Insert a new record to A_COLUMN_DATE_FORMAT table.
|
||||
**/
|
||||
PROCEDURE ADD_COLUMN_DATE_FORMAT (
|
||||
pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE
|
||||
,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE
|
||||
,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_BUCKET_URI
|
||||
* @desc Function used to get string with bucket http url.
|
||||
* Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual;
|
||||
* @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/
|
||||
**/
|
||||
FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_CONFIG_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_CONFIG record
|
||||
* for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
* pSourceFileConfigKey => 128
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
* @ex_rslt
|
||||
* Details about File Configuration:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 128
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Details about related Container Config:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 126
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Column Date Format config entries:
|
||||
* --------------------------------
|
||||
* TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM
|
||||
* ...
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_RECEIVED_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_RECEIVED record
|
||||
* for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY).
|
||||
* If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
* pSourceFileReceivedKey => 377
|
||||
* ,pIncludeConfigInfo => 1
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
*
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE
|
||||
,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_USER_LOAD_OPERATIONS
|
||||
* @desc Function returns details from USER_LOAD_OPERATIONS table
|
||||
* for specified pOperationId.
|
||||
* @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual;
|
||||
* @ex_rslt
|
||||
* Details about USER_LOAD_OPERATIONS where ID = 3608
|
||||
* --------------------------------
|
||||
* ID = 3608
|
||||
* TYPE = VALIDATE
|
||||
* SID = 31260
|
||||
* SERIAL# = 52915
|
||||
* START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE
|
||||
* UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE
|
||||
* STATUS = FAILED
|
||||
* OWNER_NAME = CT_MRDS
|
||||
* TABLE_NAME = STANDING_FACILITIES_HEADER
|
||||
* PARTITION_NAME =
|
||||
* SUBPARTITION_NAME =
|
||||
* FILE_URI_LIST =
|
||||
* ROWS_LOADED =
|
||||
* LOGFILE_TABLE = VALIDATE$3608_LOG
|
||||
* BADFILE_TABLE = VALIDATE$3608_BAD
|
||||
* STATUS_TABLE =
|
||||
* TEMPEXT_TABLE =
|
||||
* CREDENTIAL_NAME =
|
||||
* EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_USER_LOAD_OPERATIONS (
|
||||
pOperationId PLS_INTEGER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name ANALYZE_VALIDATION_ERRORS
|
||||
* @desc Wrapper function that analyzes validation errors for a source file using its received key.
|
||||
* Automatically derives template schema, table name, CSV URI and validation log table
|
||||
* from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS.
|
||||
* @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL;
|
||||
* @ex_rslt Detailed validation analysis report with column mismatches and solutions
|
||||
**/
|
||||
FUNCTION ANALYZE_VALIDATION_ERRORS(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_MANAGER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 3.2.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_MANAGER
|
||||
* Version: 3.2.0
|
||||
* Build Date: 2025-10-22 16:30:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_MANAGER Version History:
|
||||
* 3.2.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
|
||||
/
|
||||
@@ -35,10 +35,12 @@ PROMPT Rollback steps:
|
||||
PROMPT 1. Rollback TRASH retention statuses
|
||||
PROMPT 2. Revoke T_FILENAME privileges
|
||||
PROMPT 3. Remove validation trigger
|
||||
PROMPT 4. Revert threshold column renames (restore original naming)
|
||||
PROMPT 5. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH)
|
||||
PROMPT 6. Restore FILE_ARCHIVER package to v2.0.0
|
||||
PROMPT 7. Revert all archival strategies to THRESHOLD_BASED
|
||||
PROMPT 4. Remove column comments (OPTIONAL - does not affect functionality)
|
||||
PROMPT 5. Revert threshold column renames (restore original naming)
|
||||
PROMPT 6. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH)
|
||||
PROMPT 7. Restore FILE_ARCHIVER package to v2.0.0
|
||||
PROMPT 8. Restore FILE_MANAGER package to v3.3.1
|
||||
PROMPT 9. Revert all archival strategies to THRESHOLD_BASED
|
||||
PROMPT
|
||||
PROMPT Timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL;
|
||||
@@ -57,43 +59,61 @@ WHENEVER SQLERROR CONTINUE
|
||||
|
||||
-- Rollback steps (in reverse order)
|
||||
PROMPT
|
||||
PROMPT Step 1/8: Rolling back TRASH retention statuses
|
||||
PROMPT Step 1/9: Rolling back TRASH retention statuses
|
||||
PROMPT ================================================
|
||||
@@90_MARS_828_rollback_trash_retention_statuses.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 2/8: Revoking T_FILENAME privileges from MRDS_LOADER
|
||||
PROMPT Step 2/9: Revoking T_FILENAME privileges from MRDS_LOADER
|
||||
PROMPT ==========================================================
|
||||
@@95_MARS_828_rollback_grant_t_filename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 3/8: Dropping validation trigger
|
||||
PROMPT Step 3/9: Dropping validation trigger
|
||||
PROMPT ======================================
|
||||
@@93_MARS_828_rollback_trigger.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 4/8: Reverting threshold column renames
|
||||
PROMPT Step 4/9 (OPTIONAL): Removing column comments
|
||||
PROMPT ==============================================
|
||||
PROMPT NOTE: This is optional - comments do not affect functionality
|
||||
PROMPT Skipping column comments removal in standard rollback
|
||||
PROMPT Execute 94b_MARS_828_rollback_column_comments.sql manually if needed
|
||||
PROMPT
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 5/9: Reverting threshold column renames
|
||||
PROMPT =============================================
|
||||
@@94a_MARS_828_rollback_threshold_rename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 5/8: Dropping all archival configuration columns
|
||||
PROMPT Step 6/9: Dropping all archival configuration columns
|
||||
PROMPT ======================================================
|
||||
@@94_MARS_828_rollback_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 6/8: Restoring FILE_ARCHIVER Package Specification v2.0.0
|
||||
PROMPT Step 7/9: Restoring FILE_ARCHIVER Package Specification v2.0.0
|
||||
PROMPT ===============================================================
|
||||
@@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 7/8: Restoring FILE_ARCHIVER Package Body v2.0.0
|
||||
PROMPT ======================================================
|
||||
PROMPT Step 8/11: Restoring FILE_ARCHIVER Package Body v2.0.0
|
||||
PROMPT =======================================================
|
||||
@@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 8/8: Verifying tracked packages
|
||||
PROMPT =====================================
|
||||
PROMPT Step 9/11: Restoring FILE_MANAGER Package Specification v3.3.1
|
||||
PROMPT ===============================================================
|
||||
@@97_MARS_828_rollback_FILE_MANAGER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 10/11: Restoring FILE_MANAGER Package Body v3.3.1
|
||||
PROMPT ======================================================
|
||||
@@98_MARS_828_rollback_FILE_MANAGER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 11/11: Verifying tracked packages
|
||||
PROMPT ======================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
-- Verify rollback
|
||||
@@ -103,9 +123,9 @@ PROMPT =========================================
|
||||
SELECT object_name, object_type, status, last_ddl_time
|
||||
FROM all_objects
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name = 'FILE_ARCHIVER'
|
||||
AND object_name IN ('FILE_ARCHIVER', 'FILE_MANAGER')
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_type;
|
||||
ORDER BY object_name, object_type;
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
@@ -115,8 +135,9 @@ PROMPT Completion Time:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_end FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT Rollback Summary:
|
||||
PROMPT - Package: CT_MRDS.FILE_ARCHIVER
|
||||
PROMPT - Restored Version: 2.0.0 (THRESHOLD_BASED archival only)
|
||||
PROMPT - Packages Rolled Back:
|
||||
PROMPT * CT_MRDS.FILE_ARCHIVER to v2.0.0 (THRESHOLD_BASED archival only)
|
||||
PROMPT * CT_MRDS.FILE_MANAGER to v3.3.1 (pre-MARS-828 threshold column compatibility)
|
||||
PROMPT - Removed Features: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID strategies
|
||||
PROMPT
|
||||
PROMPT Log file: &_filename
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,637 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_MANAGER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) ||
|
||||
'3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) ||
|
||||
'2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-01): Initial release with file processing and validation capabilities';
|
||||
|
||||
TYPE tSourceFileReceived IS RECORD
|
||||
(
|
||||
A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE,
|
||||
A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
SOURCE_FILE_PREFIX_INBOX VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ODS VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430),
|
||||
SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE,
|
||||
RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE,
|
||||
PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE,
|
||||
EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE
|
||||
);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgSourceFileConfigKey PLS_INTEGER;
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
|
||||
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG
|
||||
* @desc Get source file type by matching the source file name against source file type naming patterns
|
||||
* or by specifying the id of a received source file.
|
||||
* @example ...
|
||||
* @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE"
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL
|
||||
, pSourceFileReceivedKey IN NUMBER DEFAULT NULL
|
||||
, pSourceFileConfigKey IN NUMBER DEFAULT NULL)
|
||||
RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table.
|
||||
* This overload automatically determines source file type from the file name.
|
||||
* It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig.
|
||||
* Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(
|
||||
* pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'
|
||||
* ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... );
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2,
|
||||
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name SET_SOURCE_FILE_RECEIVED_STATUS
|
||||
* @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column
|
||||
* based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY
|
||||
* and provided value of pStatus parameter
|
||||
* @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION');
|
||||
**/
|
||||
PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(
|
||||
pSourceFileReceivedKey IN PLS_INTEGER,
|
||||
pStatus IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_EXTERNAL_TABLE_COLUMNS
|
||||
* @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name.
|
||||
* It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure.
|
||||
* @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual;
|
||||
* @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "REV_NUMBER" NUMBER(28,0),
|
||||
* "REF_DATE" DATE,
|
||||
* "FREE_TEXT" VARCHAR2(1000 CHAR),
|
||||
* "MLF_BS_TOTAL" NUMBER(28,10),
|
||||
* "DF_BS_TOTAL" NUMBER(28,10),
|
||||
* "MLF_SF_TOTAL" NUMBER(28,10),
|
||||
* "DF_SF_TOTAL" NUMBER(28,10)
|
||||
**/
|
||||
FUNCTION GET_EXTERNAL_TABLE_COLUMNS (
|
||||
pTargetTableTemplate IN VARCHAR2
|
||||
)
|
||||
RETURN CLOB;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252')
|
||||
* If provided, adds CHARACTERSET clause to external table definition
|
||||
* @example
|
||||
* begin
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLE(
|
||||
* pTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
|
||||
* pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/',
|
||||
* pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/',
|
||||
* pFileName => NULL,
|
||||
* pDelimiter => ',',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pTableName IN VARCHAR2,
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pPrefix IN VARCHAR2,
|
||||
pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pDelimiter IN VARCHAR2 DEFAULT ',',
|
||||
pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc Creates External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_SOURCE_FILE_RECEIVED
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
|
||||
* It validate External table build upon single file
|
||||
* provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED
|
||||
(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE.
|
||||
* It validates External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt FAILED
|
||||
**/
|
||||
FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
/**
|
||||
* @name S_VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A function which checks if SELECT query reterns any rows.
|
||||
* It trys to selects External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt PASSED
|
||||
**/
|
||||
FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DROP_EXTERNAL_TABLE
|
||||
* @desc It drops External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE DROP_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name COPY_FILE
|
||||
* @desc It copies file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS'
|
||||
* @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE COPY_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name MOVE_FILE
|
||||
* @desc It moves file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS', 'QUARANTINE'
|
||||
* @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE MOVE_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_FOLDER_CONTENTS
|
||||
* @desc It deletes all files from specified folder in the cloud storage.
|
||||
* The procedure lists all objects in the specified folder prefix and deletes them one by one.
|
||||
* pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE'
|
||||
* pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/')
|
||||
* @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/');
|
||||
**/
|
||||
PROCEDURE DELETE_FOLDER_CONTENTS(
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderPrefix IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter.
|
||||
* Ubmrella procedure that calls:
|
||||
* - REGISTER_SOURCE_FILE_RECEIVED;
|
||||
* - CREATE_EXTERNAL_TABLE;
|
||||
* - VALIDATE_SOURCE_FILE_RECEIVED;
|
||||
* - DROP_EXTERNAL_TABLE;
|
||||
* - MOVE_FILE;
|
||||
* @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
**/
|
||||
PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter and return processing result value.
|
||||
* It returns (success/failure) => 0 / -(value).
|
||||
* Ubmrella function that calls PROCESS_SOURCE_FILE procedure.
|
||||
* @example
|
||||
* declare
|
||||
* vResult PLS_INTEGER;
|
||||
* begin
|
||||
* vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult);
|
||||
* end;
|
||||
* @ex_rslt 0
|
||||
* -20021
|
||||
**/
|
||||
FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DATE_FORMAT
|
||||
* @desc Returns date format for specified template table name and column name.
|
||||
* Date is taken from configuration A_COLUMN_DATE_FORMAT table.
|
||||
* @example select FILE_MANAGER.GET_DATE_FORMAT(
|
||||
* pTemplateTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pColumnName => 'SNAPSHOT_DATE')
|
||||
* from dual;
|
||||
* @ex_rslt DD/MM/YYYY HH24:MI:SS
|
||||
**/
|
||||
FUNCTION GET_DATE_FORMAT(
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnName IN VARCHAR2
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GENERATE_EXTERNAL_TABLE_PARAMS
|
||||
* @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName.
|
||||
* @example
|
||||
* declare
|
||||
* vColumnList CLOB;
|
||||
* vFieldList CLOB;
|
||||
* begin
|
||||
* FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER'
|
||||
* ,pColumnList => vColumnList
|
||||
* ,pFieldList => vFieldList
|
||||
* );
|
||||
* DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList);
|
||||
* DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList);
|
||||
* end;
|
||||
* /
|
||||
**/
|
||||
PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnList OUT CLOB,
|
||||
pFieldList OUT CLOB
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE
|
||||
* @desc Insert a new record to A_SOURCE table.
|
||||
* pSourceKey is a PRIMARY KEY value.
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE,
|
||||
pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_SOURCE_CASCADE
|
||||
* @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables:
|
||||
* - A_SOURCE_FILE_CONFIG
|
||||
* - A_SOURCE_FILE_RECEIVED
|
||||
* - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems)
|
||||
* The procedure checks if template tables are shared before deleting date format configurations.
|
||||
* If a template table is used by multiple source systems, date formats are preserved.
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS');
|
||||
**/
|
||||
PROCEDURE DELETE_SOURCE_CASCADE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID)
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record.
|
||||
* @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(
|
||||
* pSourceFileId => 'UC_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified input parameters,
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY.
|
||||
* @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY (
|
||||
* pSourceFileType => 'INPUT'
|
||||
* ,pSourceFileId => 'UC_DISSEM'
|
||||
* ,pTableId => 'UC_NMA_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT'
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE_FILE_CONFIG
|
||||
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification.
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
|
||||
* If NULL, no CHARACTERSET clause is added to external table definitions
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
|
||||
* pSourceKey => 'C2D', pSourceFileType => 'INPUT',
|
||||
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE_FILE_CONFIG (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE
|
||||
,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE
|
||||
,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
|
||||
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
|
||||
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
|
||||
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_COLUMN_DATE_FORMAT
|
||||
* @desc Insert a new record to A_COLUMN_DATE_FORMAT table.
|
||||
**/
|
||||
PROCEDURE ADD_COLUMN_DATE_FORMAT (
|
||||
pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE
|
||||
,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE
|
||||
,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_BUCKET_URI
|
||||
* @desc Function used to get string with bucket http url.
|
||||
* Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual;
|
||||
* @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/
|
||||
**/
|
||||
FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_CONFIG_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_CONFIG record
|
||||
* for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
* pSourceFileConfigKey => 128
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
* @ex_rslt
|
||||
* Details about File Configuration:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 128
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Details about related Container Config:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 126
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Column Date Format config entries:
|
||||
* --------------------------------
|
||||
* TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM
|
||||
* ...
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_RECEIVED_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_RECEIVED record
|
||||
* for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY).
|
||||
* If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
* pSourceFileReceivedKey => 377
|
||||
* ,pIncludeConfigInfo => 1
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
*
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE
|
||||
,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_USER_LOAD_OPERATIONS
|
||||
* @desc Function returns details from USER_LOAD_OPERATIONS table
|
||||
* for specified pOperationId.
|
||||
* @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual;
|
||||
* @ex_rslt
|
||||
* Details about USER_LOAD_OPERATIONS where ID = 3608
|
||||
* --------------------------------
|
||||
* ID = 3608
|
||||
* TYPE = VALIDATE
|
||||
* SID = 31260
|
||||
* SERIAL# = 52915
|
||||
* START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE
|
||||
* UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE
|
||||
* STATUS = FAILED
|
||||
* OWNER_NAME = CT_MRDS
|
||||
* TABLE_NAME = STANDING_FACILITIES_HEADER
|
||||
* PARTITION_NAME =
|
||||
* SUBPARTITION_NAME =
|
||||
* FILE_URI_LIST =
|
||||
* ROWS_LOADED =
|
||||
* LOGFILE_TABLE = VALIDATE$3608_LOG
|
||||
* BADFILE_TABLE = VALIDATE$3608_BAD
|
||||
* STATUS_TABLE =
|
||||
* TEMPEXT_TABLE =
|
||||
* CREDENTIAL_NAME =
|
||||
* EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_USER_LOAD_OPERATIONS (
|
||||
pOperationId PLS_INTEGER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name ANALYZE_VALIDATION_ERRORS
|
||||
* @desc Wrapper function that analyzes validation errors for a source file using its received key.
|
||||
* Automatically derives template schema, table name, CSV URI and validation log table
|
||||
* from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS.
|
||||
* @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL;
|
||||
* @ex_rslt Detailed validation analysis report with column mismatches and solutions
|
||||
**/
|
||||
FUNCTION ANALYZE_VALIDATION_ERRORS(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_MANAGER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 3.2.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_MANAGER
|
||||
* Version: 3.2.0
|
||||
* Build Date: 2025-10-22 16:30:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_MANAGER Version History:
|
||||
* 3.2.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
|
||||
/
|
||||
@@ -29,7 +29,8 @@ DECLARE
|
||||
-- Format: 'SCHEMA.PACKAGE_NAME'
|
||||
-- ===================================================================
|
||||
vPackageList t_string_array := t_string_array(
|
||||
'CT_MRDS.FILE_ARCHIVER'
|
||||
'CT_MRDS.FILE_ARCHIVER',
|
||||
'CT_MRDS.FILE_MANAGER'
|
||||
);
|
||||
-- ===================================================================
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ END;
|
||||
/
|
||||
|
||||
CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
|
||||
CHUNK_ID NUMBER PRIMARY KEY,
|
||||
CHUNK_ID NUMBER NOT NULL,
|
||||
TASK_NAME VARCHAR2(100) NOT NULL,
|
||||
YEAR_VALUE VARCHAR2(4) NOT NULL,
|
||||
MONTH_VALUE VARCHAR2(2) NOT NULL,
|
||||
@@ -47,14 +47,16 @@ CREATE TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS (
|
||||
STATUS VARCHAR2(30) DEFAULT 'PENDING' NOT NULL,
|
||||
ERROR_MESSAGE VARCHAR2(4000),
|
||||
EXPORT_TIMESTAMP TIMESTAMP,
|
||||
CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL
|
||||
CREATED_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL,
|
||||
CONSTRAINT PK_PARALLEL_EXPORT_CHUNKS PRIMARY KEY (TASK_NAME, CHUNK_ID)
|
||||
);
|
||||
|
||||
CREATE INDEX IX_PARALLEL_CHUNKS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(TASK_NAME);
|
||||
-- Index for status-based queries (e.g., WHERE STATUS = 'FAILED' AND TASK_NAME = ?)
|
||||
CREATE INDEX IX_PARALLEL_CHUNKS_STATUS_TASK ON CT_MRDS.A_PARALLEL_EXPORT_CHUNKS(STATUS, TASK_NAME);
|
||||
|
||||
COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Unique chunk identifier (partition number)';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name for cleanup';
|
||||
COMMENT ON TABLE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS IS 'Permanent table for parallel export chunk processing (DBMS_PARALLEL_EXECUTE) - permanent because GTT data not visible in parallel callback sessions. PK: (TASK_NAME, CHUNK_ID) ensures session isolation for concurrent exports.';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.CHUNK_ID IS 'Chunk identifier within task (partition number) - unique per TASK_NAME, not globally';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.TASK_NAME IS 'DBMS_PARALLEL_EXECUTE task name - session isolation key, part of composite PK with CHUNK_ID';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.YEAR_VALUE IS 'Partition year (YYYY)';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.MONTH_VALUE IS 'Partition month (MM)';
|
||||
COMMENT ON COLUMN CT_MRDS.A_PARALLEL_EXPORT_CHUNKS.SCHEMA_NAME IS 'Schema owning the source table';
|
||||
|
||||
@@ -18,34 +18,104 @@ AS
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Deletes export file from OCI bucket if it exists (used for cleanup before retry)
|
||||
* Silently ignores if file doesn't exist (ORA-20404)
|
||||
* Deletes ALL files matching specific file pattern before retry export
|
||||
* Critical for preventing data duplication when DBMS_CLOUD.EXPORT_DATA fails mid-process
|
||||
*
|
||||
* Problem: Export fails after creating partial file(s), retry creates new _2, _3 suffixed files
|
||||
* Solution: Delete ALL files matching the base filename pattern before retry
|
||||
*
|
||||
* Pattern matching strategy:
|
||||
* - Parquet: folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/*.parquet (folder-level safe - each chunk has own partition folder)
|
||||
* - CSV: folder/TABLENAME_202411*.csv (file-level pattern - multiple chunks share same folder!)
|
||||
*
|
||||
* CRITICAL for parallel processing:
|
||||
* - Parquet chunks are isolated by partition folder structure (safe to delete folder/*)
|
||||
* - CSV chunks share flat folder structure - MUST use file-specific pattern (TABLENAME_YYYYMM*)
|
||||
* to avoid deleting files from other parallel chunks in same folder
|
||||
**/
|
||||
PROCEDURE DELETE_FAILED_EXPORT_FILE(
|
||||
pFileUri IN VARCHAR2,
|
||||
pCredentialName IN VARCHAR2,
|
||||
pParameters IN VARCHAR2
|
||||
) IS
|
||||
vBucketUri VARCHAR2(4000);
|
||||
vFolderPath VARCHAR2(4000);
|
||||
vFileName VARCHAR2(1000);
|
||||
vFileNamePattern VARCHAR2(1000);
|
||||
vSlashPos NUMBER;
|
||||
vDotPos NUMBER;
|
||||
vFilesDeleted NUMBER := 0;
|
||||
BEGIN
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to delete potentially corrupted file: ' || pFileUri, 'DEBUG', pParameters);
|
||||
-- Extract components from URI
|
||||
-- Example Parquet: https://.../bucket/folder/PARTITION_YEAR=2024/PARTITION_MONTH=11/202411.parquet
|
||||
-- Example CSV: https://.../bucket/folder/TABLENAME_202411.csv
|
||||
|
||||
-- Find last slash before filename
|
||||
vSlashPos := INSTR(pFileUri, '/', -1);
|
||||
|
||||
IF vSlashPos > 0 THEN
|
||||
-- Extract filename from URI (after last slash)
|
||||
vFileName := SUBSTR(pFileUri, vSlashPos + 1);
|
||||
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => pCredentialName,
|
||||
object_uri => pFileUri
|
||||
);
|
||||
-- Extract folder path (before last slash)
|
||||
vFolderPath := SUBSTR(pFileUri, 1, vSlashPos - 1);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted existing file (cleanup before retry): ' || pFileUri, 'INFO', pParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
-- Object not found is OK (file doesn't exist)
|
||||
IF SQLCODE = -20404 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File does not exist (OK): ' || pFileUri, 'DEBUG', pParameters);
|
||||
ELSE
|
||||
-- Log but don't fail - export will attempt anyway
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete file (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
|
||||
END IF;
|
||||
END;
|
||||
-- Find bucket URI (protocol + namespace + bucket name)
|
||||
-- Bucket URI ends after /o/ in OCI Object Storage URLs
|
||||
vBucketUri := SUBSTR(pFileUri, 1, INSTR(pFileUri, '/o/') + 2);
|
||||
|
||||
-- Extract relative folder path (after bucket)
|
||||
vFolderPath := SUBSTR(vFolderPath, LENGTH(vBucketUri) + 1);
|
||||
|
||||
-- Create file pattern by removing extension
|
||||
-- Oracle adds suffixes BEFORE extension: file.csv -> file_1_timestamp.csv
|
||||
-- Pattern: file* matches file_1_timestamp.csv, file_2_timestamp.csv
|
||||
vDotPos := INSTR(vFileName, '.', -1);
|
||||
IF vDotPos > 0 THEN
|
||||
vFileNamePattern := SUBSTR(vFileName, 1, vDotPos - 1) || '%';
|
||||
ELSE
|
||||
vFileNamePattern := vFileName || '%';
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup before retry - Pattern: ' || vFolderPath || '/' || vFileNamePattern, 'DEBUG', pParameters);
|
||||
|
||||
-- List and delete ALL files matching pattern
|
||||
-- CRITICAL: Uses file-specific pattern for CSV chunk isolation in shared folder
|
||||
FOR rec IN (
|
||||
SELECT object_name
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => pCredentialName,
|
||||
location_uri => vBucketUri
|
||||
))
|
||||
WHERE object_name LIKE vFolderPath || '/' || vFileNamePattern
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => pCredentialName,
|
||||
object_uri => vBucketUri || rec.object_name
|
||||
);
|
||||
|
||||
vFilesDeleted := vFilesDeleted + 1;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted partial file ' || vFilesDeleted || ': ' || rec.object_name, 'DEBUG', pParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
-- Log but continue - don't fail entire cleanup
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not delete ' || rec.object_name || ': ' || SQLERRM, 'WARNING', pParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
IF vFilesDeleted > 0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Cleanup completed: Deleted ' || vFilesDeleted || ' partial file(s) from previous failed export', 'INFO', pParameters);
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('No existing files to clean up (pattern match: ' || vFileNamePattern || ')', 'DEBUG', pParameters);
|
||||
END IF;
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cannot parse file URI for cleanup: ' || pFileUri, 'WARNING', pParameters);
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
-- Don't fail export if cleanup fails - log and continue
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Cleanup failed (will retry export anyway): ' || SQLERRM, 'WARNING', pParameters);
|
||||
END DELETE_FAILED_EXPORT_FILE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
@@ -415,6 +485,8 @@ AS
|
||||
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
|
||||
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: '||pFormat||')', 'DEBUG', pParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
|
||||
-- Construct the URI based on format
|
||||
IF pFormat = 'PARQUET' THEN
|
||||
-- Parquet: Use Hive-style partitioning
|
||||
@@ -425,6 +497,7 @@ AS
|
||||
'PARTITION_MONTH=' || sanitizeFilename(pMonth) || '/' ||
|
||||
sanitizeFilename(pYear) || sanitizeFilename(pMonth) || '.parquet';
|
||||
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', pParameters);
|
||||
|
||||
-- Delete potentially corrupted file from previous failed attempt
|
||||
@@ -445,6 +518,7 @@ AS
|
||||
sanitizeFilename(vFileName);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('CSV export URI: ' || vUri, 'DEBUG', pParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('CSV maxfilesize: ' || pMaxFileSize || ' bytes (' || ROUND(pMaxFileSize/1048576, 2) || ' MB)', 'DEBUG', pParameters);
|
||||
|
||||
-- Delete potentially corrupted file from previous failed attempt
|
||||
-- This prevents Oracle from creating _1 suffixed files on retry
|
||||
@@ -472,8 +546,7 @@ AS
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Unsupported format: ' || pFormat || '. Use PARQUET or CSV.');
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || pYear || '/' || pMonth || ' (Format: ' || pFormat || ')', 'DEBUG', pParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', pParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || pYear || '/' || pMonth, 'DEBUG', pParameters);
|
||||
END EXPORT_SINGLE_PARTITION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
@@ -485,7 +558,8 @@ AS
|
||||
**/
|
||||
PROCEDURE EXPORT_PARTITION_PARALLEL (
|
||||
pStartId IN NUMBER,
|
||||
pEndId IN NUMBER
|
||||
pEndId IN NUMBER,
|
||||
pTaskName IN VARCHAR2 DEFAULT NULL
|
||||
) IS
|
||||
vYear VARCHAR2(4);
|
||||
vMonth VARCHAR2(2);
|
||||
@@ -502,9 +576,12 @@ AS
|
||||
vFileBaseName VARCHAR2(1000);
|
||||
vMaxFileSize NUMBER;
|
||||
vJobClass VARCHAR2(128);
|
||||
vTaskName VARCHAR2(128);
|
||||
vParameters VARCHAR2(4000);
|
||||
BEGIN
|
||||
-- Retrieve chunk context from global temporary table
|
||||
-- Retrieve chunk context from A_PARALLEL_EXPORT_CHUNKS table
|
||||
-- CRITICAL: Filter by CHUNK_ID and TASK_NAME for precise session isolation
|
||||
-- pTaskName parameter passed from RUN_TASK ensures deterministic single-row retrieval
|
||||
SELECT
|
||||
YEAR_VALUE,
|
||||
MONTH_VALUE,
|
||||
@@ -520,7 +597,8 @@ AS
|
||||
FORMAT_TYPE,
|
||||
FILE_BASE_NAME,
|
||||
MAX_FILE_SIZE,
|
||||
JOB_CLASS
|
||||
JOB_CLASS,
|
||||
TASK_NAME
|
||||
INTO
|
||||
vYear,
|
||||
vMonth,
|
||||
@@ -536,18 +614,22 @@ AS
|
||||
vFormat,
|
||||
vFileBaseName,
|
||||
vMaxFileSize,
|
||||
vJobClass
|
||||
vJobClass,
|
||||
vTaskName
|
||||
FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
|
||||
WHERE CHUNK_ID = pStartId;
|
||||
WHERE CHUNK_ID = pStartId
|
||||
AND TASK_NAME = pTaskName;
|
||||
|
||||
vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId;
|
||||
vParameters := 'Parallel task - Year: ' || vYear || ', Month: ' || vMonth || ', ChunkID: ' || pStartId || ', TaskName: ' || vTaskName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Starting parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
|
||||
|
||||
-- Mark chunk as PROCESSING
|
||||
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
|
||||
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
|
||||
SET STATUS = 'PROCESSING',
|
||||
ERROR_MESSAGE = NULL
|
||||
WHERE CHUNK_ID = pStartId;
|
||||
WHERE CHUNK_ID = pStartId
|
||||
AND TASK_NAME = vTaskName;
|
||||
COMMIT;
|
||||
|
||||
-- Call the worker procedure
|
||||
@@ -570,26 +652,30 @@ AS
|
||||
);
|
||||
|
||||
-- Mark chunk as COMPLETED
|
||||
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
|
||||
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
|
||||
SET STATUS = 'COMPLETED',
|
||||
EXPORT_TIMESTAMP = SYSTIMESTAMP,
|
||||
ERROR_MESSAGE = NULL
|
||||
WHERE CHUNK_ID = pStartId;
|
||||
WHERE CHUNK_ID = pStartId
|
||||
AND TASK_NAME = vTaskName;
|
||||
COMMIT;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Completed parallel export for partition ' || vYear || '/' || vMonth, 'DEBUG', vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
-- Capture error details in variable (SQLERRM cannot be used directly in SQL)
|
||||
vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
|
||||
vgMsgTmp := 'Parallel task error for partition ' || vYear || '/' || vMonth || ' (ChunkID: ' || pStartId || ', TaskName: ' || vTaskName || '): ' || SQLERRM || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
|
||||
|
||||
-- Mark chunk as FAILED with error message
|
||||
-- CRITICAL: Use both CHUNK_ID AND TASK_NAME for session isolation
|
||||
-- Use vgMsgTmp variable instead of SQLERRM directly (Oracle limitation in SQL context)
|
||||
UPDATE CT_MRDS.A_PARALLEL_EXPORT_CHUNKS
|
||||
SET STATUS = 'FAILED',
|
||||
ERROR_MESSAGE = SUBSTR(vgMsgTmp, 1, 4000)
|
||||
WHERE CHUNK_ID = pStartId;
|
||||
WHERE CHUNK_ID = pStartId
|
||||
AND TASK_NAME = vTaskName;
|
||||
COMMIT;
|
||||
|
||||
RAISE;
|
||||
@@ -1056,8 +1142,8 @@ AS
|
||||
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
|
||||
FOR i IN 1 .. vPartitions.COUNT LOOP
|
||||
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
|
||||
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
|
||||
ON (t.CHUNK_ID = s.chunk_id)
|
||||
USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
|
||||
ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
|
||||
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
|
||||
@@ -1066,33 +1152,34 @@ AS
|
||||
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
|
||||
pCredentialName, 'PARQUET', NULL, pTemplateTableName, 104857600, pJobClass, 'PENDING')
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET TASK_NAME = vTaskName,
|
||||
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
|
||||
-- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
|
||||
-- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
|
||||
UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
|
||||
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
|
||||
-- Log chunk statistics
|
||||
-- Log chunk statistics (session-safe: only count chunks for THIS task)
|
||||
DECLARE
|
||||
vPendingCount NUMBER;
|
||||
vFailedCount NUMBER;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
|
||||
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
|
||||
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
|
||||
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
|
||||
END;
|
||||
|
||||
-- Create parallel task
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
|
||||
|
||||
-- Define chunks by number range (1 to partition count)
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
|
||||
-- Define chunks using SQL query to ensure TASK_NAME isolation
|
||||
-- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
|
||||
-- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
|
||||
task_name => vTaskName,
|
||||
table_owner => 'CT_MRDS',
|
||||
table_name => 'A_PARALLEL_EXPORT_CHUNKS',
|
||||
table_column => 'CHUNK_ID',
|
||||
chunk_size => 1 -- Each partition is one chunk
|
||||
sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
|
||||
by_rowid => FALSE
|
||||
);
|
||||
|
||||
-- Execute task in parallel
|
||||
@@ -1101,7 +1188,7 @@ AS
|
||||
IF pJobClass IS NOT NULL THEN
|
||||
DBMS_PARALLEL_EXECUTE.RUN_TASK(
|
||||
task_name => vTaskName,
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
|
||||
language_flag => DBMS_SQL.NATIVE,
|
||||
parallel_level => pParallelDegree,
|
||||
job_class => pJobClass
|
||||
@@ -1109,7 +1196,7 @@ AS
|
||||
ELSE
|
||||
DBMS_PARALLEL_EXECUTE.RUN_TASK(
|
||||
task_name => vTaskName,
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
|
||||
language_flag => DBMS_SQL.NATIVE,
|
||||
parallel_level => pParallelDegree
|
||||
);
|
||||
@@ -1360,8 +1447,8 @@ AS
|
||||
-- Populate chunks table (insert new chunks, preserve FAILED chunks for retry)
|
||||
FOR i IN 1 .. vPartitions.COUNT LOOP
|
||||
MERGE INTO CT_MRDS.A_PARALLEL_EXPORT_CHUNKS t
|
||||
USING (SELECT i AS chunk_id, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
|
||||
ON (t.CHUNK_ID = s.chunk_id)
|
||||
USING (SELECT i AS chunk_id, vTaskName AS task_name, vPartitions(i).year AS yr, vPartitions(i).month AS mn FROM DUAL) s
|
||||
ON (t.CHUNK_ID = s.chunk_id AND t.TASK_NAME = s.task_name)
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (CHUNK_ID, TASK_NAME, YEAR_VALUE, MONTH_VALUE, SCHEMA_NAME, TABLE_NAME, KEY_COLUMN_NAME,
|
||||
BUCKET_URI, FOLDER_NAME, PROCESSED_COLUMNS, MIN_DATE, MAX_DATE,
|
||||
@@ -1370,33 +1457,34 @@ AS
|
||||
vBucketUri, pFolderName, vProcessedColumnList, pMinDate, pMaxDate,
|
||||
pCredentialName, 'CSV', vFileBaseName, pTemplateTableName, pMaxFileSize, pJobClass, 'PENDING')
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET TASK_NAME = vTaskName,
|
||||
STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
|
||||
-- Match found: chunk exists for SAME task (composite PK: TASK_NAME, CHUNK_ID)
|
||||
-- This handles retry scenario: reset FAILED chunks to PENDING for re-processing
|
||||
UPDATE SET STATUS = CASE WHEN t.STATUS = 'FAILED' THEN 'PENDING' ELSE t.STATUS END,
|
||||
ERROR_MESSAGE = CASE WHEN t.STATUS = 'FAILED' THEN NULL ELSE t.ERROR_MESSAGE END;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
|
||||
-- Log chunk statistics
|
||||
-- Log chunk statistics (session-safe: only count chunks for THIS task)
|
||||
DECLARE
|
||||
vPendingCount NUMBER;
|
||||
vFailedCount NUMBER;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING';
|
||||
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED';
|
||||
SELECT COUNT(*) INTO vPendingCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'PENDING' AND TASK_NAME = vTaskName;
|
||||
SELECT COUNT(*) INTO vFailedCount FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE STATUS = 'FAILED' AND TASK_NAME = vTaskName;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics: PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Chunk statistics for task ' || vTaskName || ': PENDING=' || vPendingCount || ', FAILED (retry)=' || vFailedCount, 'INFO', vParameters);
|
||||
END;
|
||||
|
||||
-- Create parallel task
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => vTaskName);
|
||||
|
||||
-- Define chunks by number range (1 to partition count)
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_NUMBER_COL(
|
||||
-- Define chunks using SQL query to ensure TASK_NAME isolation
|
||||
-- CRITICAL: Filter by TASK_NAME to avoid selecting chunks from other concurrent sessions
|
||||
-- CRITICAL: Use START_ID and END_ID aliases to avoid ORA-00960 ambiguous column naming
|
||||
DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
|
||||
task_name => vTaskName,
|
||||
table_owner => 'CT_MRDS',
|
||||
table_name => 'A_PARALLEL_EXPORT_CHUNKS',
|
||||
table_column => 'CHUNK_ID',
|
||||
chunk_size => 1 -- Each partition is one chunk
|
||||
sql_stmt => 'SELECT CHUNK_ID AS START_ID, CHUNK_ID AS END_ID FROM CT_MRDS.A_PARALLEL_EXPORT_CHUNKS WHERE TASK_NAME = ''' || vTaskName || ''' ORDER BY CHUNK_ID',
|
||||
by_rowid => FALSE
|
||||
);
|
||||
|
||||
-- Execute task in parallel
|
||||
@@ -1405,7 +1493,7 @@ AS
|
||||
IF pJobClass IS NOT NULL THEN
|
||||
DBMS_PARALLEL_EXECUTE.RUN_TASK(
|
||||
task_name => vTaskName,
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
|
||||
language_flag => DBMS_SQL.NATIVE,
|
||||
parallel_level => pParallelDegree,
|
||||
job_class => pJobClass
|
||||
@@ -1413,7 +1501,7 @@ AS
|
||||
ELSE
|
||||
DBMS_PARALLEL_EXECUTE.RUN_TASK(
|
||||
task_name => vTaskName,
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id); END;',
|
||||
sql_stmt => 'BEGIN CT_MRDS.DATA_EXPORTER.EXPORT_PARTITION_PARALLEL(:start_id, :end_id, ''' || vTaskName || '''); END;',
|
||||
language_flag => DBMS_SQL.NATIVE,
|
||||
parallel_level => pParallelDegree
|
||||
);
|
||||
|
||||
@@ -9,17 +9,17 @@ AS
|
||||
**/
|
||||
|
||||
-- Package Version Information
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.11.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 10:00:00';
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.14.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-25 09:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (last 3-5 changes)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10) ||
|
||||
'v2.10.1 (2026-02-17): CRITICAL FIX - Remove redundant COMPLETED chunks deletion before parallel export that caused ORA-01403 errors (phantom chunks created by CREATE_CHUNKS_BY_NUMBER_COL).' || CHR(10) ||
|
||||
'v2.10.0 (2026-02-13): CRITICAL FIX - Register ALL files created by DBMS_CLOUD.EXPORT_DATA (multi-file support due to Oracle parallel processing on large instances). Prevents orphaned files in rollback.' || CHR(10) ||
|
||||
'v2.9.0 (2026-02-13): Added pProcessName parameter to EXPORT_TABLE_DATA and EXPORT_TABLE_DATA_TO_CSV_BY_DATE procedures for process tracking in A_SOURCE_FILE_RECEIVED table.' || CHR(10) ||
|
||||
'v2.8.1 (2026-02-12): FIX query in EXPORT_TABLE_DATA - removed A_LOAD_HISTORY join to ensure single file output (simple SELECT).' || CHR(10);
|
||||
'v2.14.0 (2026-02-25): OPTIMIZATION - Added pTaskName parameter to EXPORT_PARTITION_PARALLEL for deterministic filtering. Replaced FETCH FIRST 1 ROW ONLY safeguard with precise WHERE CHUNK_ID AND TASK_NAME filter. Eliminates ORDER BY overhead and provides cleaner session isolation.' || CHR(10) ||
|
||||
'v2.13.1 (2026-02-25): CRITICAL FIX - Added START_ID and END_ID aliasses in CREATE_CHUNKS_BY_SQL to avoid ORA-00960 ambiguous column naming error.' || CHR(10) ||
|
||||
'v2.13.0 (2026-02-25): CRITICAL SESSION ISOLATION FIX - Changed CREATE_CHUNKS_BY_NUMBER_COL to CREATE_CHUNKS_BY_SQL with TASK_NAME filter (fixes ORA-01422 in concurrent sessions). Added ORDER BY CREATED_DATE DESC FETCH FIRST 1 ROW safeguard to EXPORT_PARTITION_PARALLEL SELECT. Composite PK (TASK_NAME, CHUNK_ID) now fully functional.' || CHR(10) ||
|
||||
'v2.12.0 (2026-02-24): CRITICAL FIX - Rewritten DELETE_FAILED_EXPORT_FILE to use file-specific pattern matching (prevents deleting parallel CSV chunks in shared folder). Added vQuery logging before DBMS_CLOUD calls. Added CSV maxfilesize logging.' || CHR(10) ||
|
||||
'v2.11.0 (2026-02-18): Added pJobClass parameter to EXPORT_TABLE_DATA_BY_DATE and EXPORT_TABLE_DATA_TO_CSV_BY_DATE for Oracle Scheduler job class support (resource/priority management).' || CHR(10);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
@@ -54,10 +54,12 @@ AS
|
||||
* but should NOT be called directly by external code.
|
||||
* @param pStartId - Chunk start ID (CHUNK_ID from A_PARALLEL_EXPORT_CHUNKS table)
|
||||
* @param pEndId - Chunk end ID (same as pStartId for single-row chunks)
|
||||
* @param pTaskName - Task name for session isolation (optional, DEFAULT NULL for backward compatibility)
|
||||
**/
|
||||
PROCEDURE EXPORT_PARTITION_PARALLEL (
|
||||
pStartId IN NUMBER,
|
||||
pEndId IN NUMBER
|
||||
pEndId IN NUMBER,
|
||||
pTaskName IN VARCHAR2 DEFAULT NULL
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -1,125 +1,31 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY)
|
||||
-- MARS-835: Export Group 1 - HIST Only (DEBT, DEBT_DAILY)
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet)
|
||||
-- Purpose: Export ALL data to HIST bucket (Parquet with Hive-style partitioning)
|
||||
-- Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
|
||||
-- Excludes legacy columns not required in new structure
|
||||
-- USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats
|
||||
-- USES: DATA_EXPORTER v2.12.0 with pTemplateTableName for column order and date formats
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-17
|
||||
-- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName)
|
||||
-- Updated: 2026-02-24 (Changed to HIST-only export, no DATA bucket split)
|
||||
-- Related: MARS-835 - CSDB Data Export
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET TIMING ON
|
||||
|
||||
DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Exporting CSDB.DEBT - Split DATA + HIST
|
||||
PROMPT Exporting CSDB.DEBT - HIST Only
|
||||
PROMPT ========================================================================
|
||||
PROMPT Last 6 months to DATA bucket (CSV format)
|
||||
PROMPT Older data to HIST bucket (Parquet with partitioning)
|
||||
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
|
||||
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
|
||||
PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- PRE-EXPORT CHECK: List existing files and count records
|
||||
DECLARE
|
||||
vFileCount NUMBER := 0;
|
||||
vRecordCount NUMBER := 0;
|
||||
vLocationUri VARCHAR2(1000);
|
||||
-- Export ALL data to HIST bucket (Parquet)
|
||||
-- NEW v2.12.0: Per-column date format handling with template table, full data range
|
||||
BEGIN
|
||||
-- Get bucket URI for DATA bucket
|
||||
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/';
|
||||
|
||||
-- Count existing files
|
||||
SELECT COUNT(*)
|
||||
INTO vFileCount
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vLocationUri
|
||||
))
|
||||
WHERE object_name NOT LIKE '%/'; -- Exclude directories
|
||||
|
||||
IF vFileCount > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
|
||||
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- List existing files
|
||||
DBMS_OUTPUT.PUT_LINE('Existing files:');
|
||||
FOR rec IN (
|
||||
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vLocationUri
|
||||
))
|
||||
WHERE object_name NOT LIKE '%/'
|
||||
ORDER BY object_name
|
||||
) LOOP
|
||||
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
|
||||
END LOOP;
|
||||
|
||||
-- Count records in external table
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
|
||||
DBMS_OUTPUT.PUT_LINE('>>>');
|
||||
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
|
||||
DBMS_OUTPUT.PUT_LINE('>>>');
|
||||
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
|
||||
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Export recent data to DATA bucket (CSV)
|
||||
-- NEW v2.4.0: Per-column date format handling with template table for column order
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
|
||||
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
pSchemaName => 'OU_CSDB',
|
||||
pTableName => 'LEGACY_DEBT',
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'DATA',
|
||||
pFolderName => 'ODS/CSDB/CSDB_DEBT',
|
||||
pMinDate => &cutoff_date,
|
||||
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
|
||||
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
||||
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
||||
pProcessName => 'MARS-835', -- Process identifier for tracking
|
||||
pJobClass => 'high' -- Oracle Scheduler job class for resource management
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
|
||||
END;
|
||||
/
|
||||
|
||||
-- Export historical data to HIST bucket (Parquet)
|
||||
-- NEW v2.4.0: Per-column date format handling with template table
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (ALL data)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
|
||||
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
|
||||
@@ -128,7 +34,8 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
|
||||
pMaxDate => &cutoff_date,
|
||||
pMinDate => DATE '1900-01-01', -- Include all historical data
|
||||
pMaxDate => DATE '9999-12-31', -- Include all future dates
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
|
||||
pJobClass => 'high' -- Oracle Scheduler job class for resource management
|
||||
@@ -139,110 +46,18 @@ END;
|
||||
/
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST
|
||||
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - HIST Only
|
||||
PROMPT ========================================================================
|
||||
PROMPT Last 6 months to DATA bucket (CSV format)
|
||||
PROMPT Older data to HIST bucket (Parquet with partitioning)
|
||||
PROMPT ALL data to HIST bucket (Parquet with Hive-style partitioning)
|
||||
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
|
||||
PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS,
|
||||
PROMPT ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS
|
||||
PROMPT ========================================================================
|
||||
|
||||
-- PRE-EXPORT CHECK: List existing files and count records
|
||||
DECLARE
|
||||
vFileCount NUMBER := 0;
|
||||
vRecordCount NUMBER := 0;
|
||||
vLocationUri VARCHAR2(1000);
|
||||
-- Export ALL data to HIST bucket (Parquet)
|
||||
-- NEW v2.12.0: Per-column date format handling with template table, full data range
|
||||
BEGIN
|
||||
-- Get bucket URI for DATA bucket
|
||||
vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/';
|
||||
|
||||
-- Count existing files
|
||||
SELECT COUNT(*)
|
||||
INTO vFileCount
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vLocationUri
|
||||
))
|
||||
WHERE object_name NOT LIKE '%/'; -- Exclude directories
|
||||
|
||||
IF vFileCount > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
|
||||
DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- List existing files
|
||||
DBMS_OUTPUT.PUT_LINE('Existing files:');
|
||||
FOR rec IN (
|
||||
SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vLocationUri
|
||||
))
|
||||
WHERE object_name NOT LIKE '%/'
|
||||
ORDER BY object_name
|
||||
) LOOP
|
||||
DBMS_OUTPUT.PUT_LINE(' - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
|
||||
END LOOP;
|
||||
|
||||
-- Count records in external table
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
|
||||
DBMS_OUTPUT.PUT_LINE('>>>');
|
||||
DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
|
||||
DBMS_OUTPUT.PUT_LINE('>>>');
|
||||
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
|
||||
DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('===============================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Export recent data to DATA bucket (CSV)
|
||||
-- NEW v2.4.0: Per-column date format handling with template table for column order
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
|
||||
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
|
||||
pSchemaName => 'OU_CSDB',
|
||||
pTableName => 'LEGACY_DEBT_DAILY',
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'DATA',
|
||||
pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
|
||||
pMinDate => &cutoff_date,
|
||||
pMaxDate => DATE '9999-12-31', -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
|
||||
pMaxFileSize => 104857600, -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
|
||||
pRegisterExport => TRUE, -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
|
||||
pProcessName => 'MARS-835', -- Process identifier for tracking
|
||||
pJobClass => 'high' -- Oracle Scheduler job class for resource management
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
|
||||
END;
|
||||
/
|
||||
|
||||
-- Export historical data to HIST bucket (Parquet)
|
||||
-- NEW v2.4.0: Per-column date format handling with template table
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (ALL data)...');
|
||||
DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
|
||||
|
||||
CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
|
||||
@@ -251,7 +66,8 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
|
||||
pMaxDate => &cutoff_date,
|
||||
pMinDate => DATE '1900-01-01', -- Include all historical data
|
||||
pMaxDate => DATE '9999-12-31', -- Include all future dates
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
|
||||
pJobClass => 'high' -- Oracle Scheduler job class for resource management
|
||||
@@ -264,8 +80,8 @@ END;
|
||||
PROMPT ========================================================================
|
||||
PROMPT Group 1 Export Completed
|
||||
PROMPT ========================================================================
|
||||
PROMPT - LEGACY_DEBT: DATA + HIST exported
|
||||
PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported
|
||||
PROMPT - LEGACY_DEBT: HIST exported (ALL data)
|
||||
PROMPT - LEGACY_DEBT_DAILY: HIST exported (ALL data)
|
||||
PROMPT ========================================================================
|
||||
|
||||
--=============================================================================================================================
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
-- =====================================================================================
|
||||
-- Script: 03_MARS_835_verify_exports.sql
|
||||
-- Purpose: Verify exported files exist in DATA and HIST buckets after export
|
||||
-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Created: 2025-12-17
|
||||
-- Updated: 2026-02-24 (Changed to HIST-only verification)
|
||||
-- MARS Issue: MARS-835
|
||||
-- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/
|
||||
-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/
|
||||
-- =====================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED;
|
||||
@@ -13,17 +14,14 @@ SET VERIFY OFF;
|
||||
SET LINESIZE 200;
|
||||
|
||||
PROMPT =====================================================================================
|
||||
PROMPT MARS-835 Verification: Listing exported files in DATA and HIST buckets
|
||||
PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy)
|
||||
PROMPT =====================================================================================
|
||||
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vHistBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFileCount NUMBER := 0;
|
||||
vTotalDataFiles NUMBER := 0;
|
||||
vTotalHistFiles NUMBER := 0;
|
||||
vTotalDataSize NUMBER := 0;
|
||||
vTotalHistSize NUMBER := 0;
|
||||
|
||||
TYPE t_folder_info IS RECORD (
|
||||
@@ -33,25 +31,18 @@ DECLARE
|
||||
);
|
||||
TYPE t_folder_list IS TABLE OF t_folder_info;
|
||||
|
||||
vDataFolders t_folder_list;
|
||||
vHistFolders t_folder_list;
|
||||
BEGIN
|
||||
-- Get bucket URIs and credential from FILE_MANAGER
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
-- Get bucket URI and credential from FILE_MANAGER
|
||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
|
||||
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
|
||||
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- Initialize folder lists
|
||||
vDataFolders := t_folder_list(
|
||||
t_folder_info('ODS/CSDB/CSDB_DEBT/', 'DEBT', 'CSV'),
|
||||
t_folder_info('ODS/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'CSV')
|
||||
);
|
||||
|
||||
-- Initialize folder list (all tables in HIST)
|
||||
-- Initialize folder list (all 6 tables in HIST)
|
||||
vHistFolders := t_folder_list(
|
||||
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'),
|
||||
t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'),
|
||||
@@ -62,49 +53,7 @@ BEGIN
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket Exports (CSV format - last 6 months)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
|
||||
-- Check DATA bucket exports
|
||||
FOR i IN 1..vDataFolders.COUNT LOOP
|
||||
vFileCount := 0;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Table: ' || vDataFolders(i).table_name || ' (' || vDataFolders(i).expected_format || ')');
|
||||
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i).folder_name);
|
||||
DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
|
||||
|
||||
BEGIN
|
||||
FOR rec IN (
|
||||
SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri || vDataFolders(i).folder_name
|
||||
))
|
||||
WHERE object_name LIKE '%.csv'
|
||||
ORDER BY created DESC
|
||||
) LOOP
|
||||
vFileCount := vFileCount + 1;
|
||||
vTotalDataFiles := vTotalDataFiles + 1;
|
||||
vTotalDataSize := vTotalDataSize + rec.bytes;
|
||||
DBMS_OUTPUT.PUT_LINE(' [' || vFileCount || '] ' || rec.object_name ||
|
||||
' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
|
||||
END LOOP;
|
||||
|
||||
IF vFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' [ERROR] No CSV files found - Export may have failed!');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE(' [SUCCESS] Found ' || vFileCount || ' CSV file(s)');
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' [ERROR] Cannot access folder - ' || SQLERRM);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning)');
|
||||
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
|
||||
-- Check HIST bucket exports
|
||||
@@ -155,24 +104,19 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Export Verification Summary');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('DATA Bucket (CSV):');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalDataFiles);
|
||||
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalDataSize/1024/1024/1024, 2) || ' GB');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 2 (DEBT, DEBT_DAILY - last 6 months)');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet):');
|
||||
DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Total files: ' || vTotalHistFiles || '+');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables with historical data)');
|
||||
DBMS_OUTPUT.PUT_LINE(' - Expected tables: 6 (all CSDB tables exported to HIST)');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
IF vTotalDataFiles >= 2 AND vTotalHistFiles >= 6 THEN
|
||||
IF vTotalHistFiles >= 6 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL');
|
||||
DBMS_OUTPUT.PUT_LINE(' Files found in both DATA and HIST buckets');
|
||||
DBMS_OUTPUT.PUT_LINE(' Files found in HIST bucket for all tables');
|
||||
DBMS_OUTPUT.PUT_LINE(' Proceed to record count verification (Step 4)');
|
||||
ELSIF vTotalDataFiles = 0 AND vTotalHistFiles = 0 THEN
|
||||
ELSIF vTotalHistFiles = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED');
|
||||
DBMS_OUTPUT.PUT_LINE(' No files found in either bucket');
|
||||
DBMS_OUTPUT.PUT_LINE(' No files found in HIST bucket');
|
||||
DBMS_OUTPUT.PUT_LINE(' Review export logs for errors');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected');
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
-- =====================================================================================
|
||||
-- Script: 04_MARS_835_verify_record_counts.sql
|
||||
-- Purpose: Verify record counts match between source tables and exported data
|
||||
-- Purpose: Verify record counts match between source tables and exported data (HIST-only)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Created: 2025-12-17
|
||||
-- Updated: 2026-02-24 (Changed to HIST-only verification)
|
||||
-- MARS Issue: MARS-835
|
||||
-- Verification: Compare OU_CSDB source tables with ODS external tables
|
||||
-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only)
|
||||
-- =====================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED;
|
||||
@@ -13,28 +14,23 @@ SET VERIFY OFF;
|
||||
SET LINESIZE 200;
|
||||
|
||||
PROMPT =====================================================================================
|
||||
PROMPT MARS-835 Record Count Verification
|
||||
PROMPT MARS-835 Record Count Verification (HIST-only strategy)
|
||||
PROMPT =====================================================================================
|
||||
PROMPT Comparing source table counts with exported external table counts
|
||||
PROMPT Comparing source table counts with HIST external table counts
|
||||
PROMPT =====================================================================================
|
||||
|
||||
DECLARE
|
||||
TYPE t_table_info IS RECORD (
|
||||
source_schema VARCHAR2(50),
|
||||
source_table VARCHAR2(100),
|
||||
data_external_table VARCHAR2(100),
|
||||
hist_external_table VARCHAR2(100),
|
||||
has_data_export BOOLEAN,
|
||||
has_hist_export BOOLEAN
|
||||
hist_external_table VARCHAR2(100)
|
||||
);
|
||||
TYPE t_table_list IS TABLE OF t_table_info;
|
||||
|
||||
vTables t_table_list;
|
||||
vSourceCount NUMBER;
|
||||
vDataCount NUMBER;
|
||||
vHistCount NUMBER;
|
||||
vTotalSourceCount NUMBER := 0;
|
||||
vTotalDataCount NUMBER := 0;
|
||||
vTotalHistCount NUMBER := 0;
|
||||
vMismatchCount NUMBER := 0;
|
||||
vSql VARCHAR2(4000);
|
||||
@@ -42,18 +38,18 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- Initialize table list with export configuration
|
||||
-- Initialize table list (all tables HIST-only)
|
||||
vTables := t_table_list(
|
||||
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ODS', 'ODS.CSDB_DEBT_ARCHIVE', TRUE, TRUE),
|
||||
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ODS', 'ODS.CSDB_DEBT_DAILY_ARCHIVE', TRUE, TRUE),
|
||||
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', NULL, 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE', FALSE, TRUE),
|
||||
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', NULL, 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE', FALSE, TRUE),
|
||||
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', NULL, 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE', FALSE, TRUE),
|
||||
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', NULL, 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE', FALSE, TRUE)
|
||||
t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'),
|
||||
t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'),
|
||||
t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'),
|
||||
t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'),
|
||||
t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'),
|
||||
t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE')
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
|
||||
DBMS_OUTPUT.PUT_LINE('Table Name Source Count DATA Count HIST Count Status');
|
||||
DBMS_OUTPUT.PUT_LINE('Table Name Source Count HIST Count Status');
|
||||
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
|
||||
|
||||
FOR i IN 1..vTables.COUNT LOOP
|
||||
@@ -70,31 +66,6 @@ BEGIN
|
||||
CONTINUE;
|
||||
END;
|
||||
|
||||
-- Get DATA external table count (if applicable)
|
||||
IF vTables(i).has_data_export THEN
|
||||
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).data_external_table;
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE vSql INTO vDataCount;
|
||||
vTotalDataCount := vTotalDataCount + vDataCount;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
-- If source table is empty (0 records), no files were exported
|
||||
-- External table returns error, treat as 0
|
||||
-- Acceptable error codes:
|
||||
-- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
|
||||
-- ORA-29400: data cartridge error
|
||||
-- KUP-13023: nothing matched wildcard query (no files in bucket)
|
||||
-- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
|
||||
IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
|
||||
vDataCount := 0;
|
||||
ELSE
|
||||
vDataCount := -1;
|
||||
END IF;
|
||||
END;
|
||||
ELSE
|
||||
vDataCount := NULL;
|
||||
END IF;
|
||||
|
||||
-- Get HIST external table count
|
||||
vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table;
|
||||
BEGIN
|
||||
@@ -119,18 +90,8 @@ BEGIN
|
||||
-- Display results
|
||||
DECLARE
|
||||
vStatus VARCHAR2(20);
|
||||
vDataDisplay VARCHAR2(17);
|
||||
vHistDisplay VARCHAR2(17);
|
||||
BEGIN
|
||||
-- Format DATA count display
|
||||
IF vDataCount IS NULL THEN
|
||||
vDataDisplay := 'N/A';
|
||||
ELSIF vDataCount = -1 THEN
|
||||
vDataDisplay := 'ERROR';
|
||||
ELSE
|
||||
vDataDisplay := TO_CHAR(vDataCount, '9,999,999,999');
|
||||
END IF;
|
||||
|
||||
-- Format HIST count display
|
||||
IF vHistCount = -1 THEN
|
||||
vHistDisplay := 'ERROR';
|
||||
@@ -138,35 +99,20 @@ BEGIN
|
||||
vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
|
||||
END IF;
|
||||
|
||||
-- Determine status
|
||||
IF vTables(i).has_data_export THEN
|
||||
-- Split export: check DATA + HIST = SOURCE
|
||||
IF (vDataCount + vHistCount) = vSourceCount THEN
|
||||
vStatus := 'PASS';
|
||||
ELSIF vDataCount = -1 OR vHistCount = -1 THEN
|
||||
vStatus := 'ERROR';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
ELSE
|
||||
vStatus := 'MISMATCH';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
END IF;
|
||||
-- Determine status (HIST only: check HIST = SOURCE)
|
||||
IF vHistCount = vSourceCount THEN
|
||||
vStatus := 'PASS';
|
||||
ELSIF vHistCount = -1 THEN
|
||||
vStatus := 'ERROR';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
ELSE
|
||||
-- HIST only: check HIST = SOURCE
|
||||
IF vHistCount = vSourceCount THEN
|
||||
vStatus := 'PASS';
|
||||
ELSIF vHistCount = -1 THEN
|
||||
vStatus := 'ERROR';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
ELSE
|
||||
vStatus := 'MISMATCH';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
END IF;
|
||||
vStatus := 'MISMATCH';
|
||||
vMismatchCount := vMismatchCount + 1;
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
RPAD(vTables(i).source_table, 24) ||
|
||||
LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
|
||||
LPAD(vDataDisplay, 15) ||
|
||||
LPAD(vHistDisplay, 15) || ' ' ||
|
||||
vStatus
|
||||
);
|
||||
@@ -177,18 +123,16 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE(
|
||||
RPAD('TOTALS', 24) ||
|
||||
LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
|
||||
LPAD(TO_CHAR(vTotalDataCount, '9,999,999,999'), 15) ||
|
||||
LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
|
||||
DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
|
||||
DBMS_OUTPUT.PUT_LINE('Total DATA records: ' || TO_CHAR(vTotalDataCount, '9,999,999,999') || ' (last 6 months)');
|
||||
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (historical + full exports)');
|
||||
DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
IF vMismatchCount = 0 THEN
|
||||
@@ -209,7 +153,6 @@ BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE(' MISMATCH - Record counts differ (may be pre-existing files or export issue)');
|
||||
DBMS_OUTPUT.PUT_LINE(' Check pre-check results to identify pre-existing files');
|
||||
DBMS_OUTPUT.PUT_LINE(' ERROR - Cannot access table (may not exist yet)');
|
||||
DBMS_OUTPUT.PUT_LINE(' N/A - Not applicable (table not exported to DATA)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
|
||||
EXCEPTION
|
||||
|
||||
@@ -1,68 +1,34 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-835 ROLLBACK: Delete Group 1 Exported Files (DEBT, DEBT_DAILY)
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Delete exported CSV and Parquet files from DATA and HIST buckets
|
||||
-- Purpose: Delete exported Parquet files from HIST bucket (ARCHIVE only)
|
||||
-- WARNING: This will permanently delete exported data files!
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-17
|
||||
-- Updated: 2026-02-24 (Changed to HIST-only rollback, no DATA bucket)
|
||||
-- Related: MARS-835 - CSDB Data Export Rollback
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT ROLLBACK: Deleting DEBT exported files
|
||||
PROMPT ROLLBACK: Deleting DEBT exported files from HIST
|
||||
PROMPT ========================================================================
|
||||
PROMPT WARNING: This will delete files from:
|
||||
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT/
|
||||
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT/
|
||||
PROMPT ========================================================================
|
||||
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vHistBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Get bucket URIs and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
-- Get bucket URI and credential
|
||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT CSV files from DATA bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
|
||||
|
||||
-- Delete CSV files for DEBT from DATA bucket using LIST_OBJECTS
|
||||
FOR rec IN (
|
||||
SELECT object_name
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/'
|
||||
))
|
||||
WHERE object_name LIKE 'LEGACY_DEBT%'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => vCredentialName,
|
||||
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT/' || rec.object_name
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT Parquet files from ARCHIVE bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
|
||||
vFileCount := 0;
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
|
||||
|
||||
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
|
||||
FOR rec IN (
|
||||
@@ -99,58 +65,23 @@ END;
|
||||
/
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files
|
||||
PROMPT ROLLBACK: Deleting DEBT_DAILY exported files from HIST
|
||||
PROMPT ========================================================================
|
||||
PROMPT WARNING: This will delete files from:
|
||||
PROMPT - DATA bucket: mrds_data_dev/ODS/CSDB/CSDB_DEBT_DAILY/
|
||||
PROMPT - HIST bucket: mrds_hist_dev/ARCHIVE/CSDB/CSDB_DEBT_DAILY/
|
||||
PROMPT ========================================================================
|
||||
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vHistBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vFileCount NUMBER := 0;
|
||||
BEGIN
|
||||
-- Get bucket URIs and credential
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
-- Get bucket URI and credential
|
||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY CSV files from DATA bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS to scan bucket');
|
||||
|
||||
-- Delete CSV files for DEBT_DAILY from DATA bucket using LIST_OBJECTS
|
||||
FOR rec IN (
|
||||
SELECT object_name
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/'
|
||||
))
|
||||
WHERE object_name LIKE 'LEGACY_DEBT_DAILY%'
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => vCredentialName,
|
||||
object_uri => vDataBucketUri || 'ODS/CSDB/CSDB_DEBT_DAILY/' || rec.object_name
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE(' Deleted: ' || rec.object_name);
|
||||
vFileCount := vFileCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' Skipped (not found): ' || rec.object_name);
|
||||
ELSE
|
||||
RAISE;
|
||||
END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: DEBT_DAILY CSV files deleted from DATA bucket (' || vFileCount || ' file(s))');
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Deleting DEBT_DAILY Parquet files from ARCHIVE bucket...');
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS (Parquet files not registered)');
|
||||
vFileCount := 0;
|
||||
DBMS_OUTPUT.PUT_LINE(' Using DBMS_CLOUD.LIST_OBJECTS');
|
||||
|
||||
-- Delete Parquet files from ARCHIVE bucket using DBMS_CLOUD.LIST_OBJECTS
|
||||
FOR rec IN (
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
-- =====================================================================================
|
||||
-- Script: 99_MARS_835_verify_rollback.sql
|
||||
-- Purpose: Verify all exported files have been deleted from DATA and HIST buckets
|
||||
-- Purpose: Verify all exported files have been deleted from HIST bucket (HIST-only strategy)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Created: 2025-12-17
|
||||
-- Updated: 2026-02-24 (Changed to HIST-only verification)
|
||||
-- MARS Issue: MARS-835
|
||||
-- Verification: Confirm complete rollback (no CSDB files remaining)
|
||||
-- Verification: Confirm complete rollback (no CSDB files remaining in HIST)
|
||||
-- =====================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED;
|
||||
@@ -19,33 +20,23 @@ PROMPT Checking that all CSDB export files have been deleted
|
||||
PROMPT =====================================================================================
|
||||
|
||||
DECLARE
|
||||
vDataBucketUri VARCHAR2(500);
|
||||
vHistBucketUri VARCHAR2(500);
|
||||
vCredentialName VARCHAR2(100);
|
||||
vDataFileCount NUMBER := 0;
|
||||
vHistFileCount NUMBER := 0;
|
||||
vTotalFiles NUMBER := 0;
|
||||
|
||||
TYPE t_folder_list IS TABLE OF VARCHAR2(200);
|
||||
vDataFolders t_folder_list;
|
||||
vHistFolders t_folder_list;
|
||||
BEGIN
|
||||
-- Get bucket URIs
|
||||
vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
|
||||
-- Get bucket URI
|
||||
vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
|
||||
vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('ROLLBACK VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
|
||||
DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
|
||||
DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
-- Initialize folder lists
|
||||
vDataFolders := t_folder_list(
|
||||
'ODS/CSDB/CSDB_DEBT/',
|
||||
'ODS/CSDB/CSDB_DEBT_DAILY/'
|
||||
);
|
||||
|
||||
-- Initialize folder list (all 6 tables in HIST)
|
||||
-- Initialize folder list (all 6 tables in HIST)
|
||||
vHistFolders := t_folder_list(
|
||||
'ARCHIVE/CSDB/CSDB_DEBT/',
|
||||
'ARCHIVE/CSDB/CSDB_DEBT_DAILY/',
|
||||
@@ -55,47 +46,6 @@ BEGIN
|
||||
'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/'
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (should be empty)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
|
||||
-- Check DATA bucket
|
||||
FOR i IN 1..vDataFolders.COUNT LOOP
|
||||
DECLARE
|
||||
vCount NUMBER := 0;
|
||||
BEGIN
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i));
|
||||
|
||||
FOR rec IN (
|
||||
SELECT object_name
|
||||
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => vCredentialName,
|
||||
location_uri => vDataBucketUri || vDataFolders(i)
|
||||
))
|
||||
WHERE object_name LIKE '%.csv'
|
||||
) LOOP
|
||||
vCount := vCount + 1;
|
||||
vDataFileCount := vDataFileCount + 1;
|
||||
DBMS_OUTPUT.PUT_LINE(' [FOUND] ' || rec.object_name);
|
||||
END LOOP;
|
||||
|
||||
IF vCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' [OK] No CSV files found');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE(' [INFO] Found ' || vCount || ' file(s) - may be pre-existing files from before installation');
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20404 THEN
|
||||
DBMS_OUTPUT.PUT_LINE(' [OK] Folder does not exist or is empty');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE(' [ERROR] ' || SQLERRM);
|
||||
END IF;
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (should be empty)');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
@@ -139,24 +89,21 @@ BEGIN
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
vTotalFiles := vDataFileCount + vHistFileCount;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('Rollback Verification Summary');
|
||||
DBMS_OUTPUT.PUT_LINE('=====================================================================================');
|
||||
DBMS_OUTPUT.PUT_LINE('DATA bucket files remaining: ' || vDataFileCount);
|
||||
DBMS_OUTPUT.PUT_LINE('HIST bucket files remaining: ' || vHistFileCount || '+');
|
||||
DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles || '+');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
|
||||
IF vTotalFiles = 0 THEN
|
||||
IF vHistFileCount = 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('[PASSED] ROLLBACK VERIFICATION PASSED');
|
||||
DBMS_OUTPUT.PUT_LINE(' All CSDB export files have been deleted or were not created');
|
||||
DBMS_OUTPUT.PUT_LINE(' Buckets are clean and ready for re-export if needed');
|
||||
DBMS_OUTPUT.PUT_LINE(' HIST bucket is clean and ready for re-export if needed');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('[INFO] ROLLBACK VERIFICATION COMPLETED');
|
||||
DBMS_OUTPUT.PUT_LINE(' Found ' || vTotalFiles || '+ file(s) remaining in buckets');
|
||||
DBMS_OUTPUT.PUT_LINE(' Found ' || vHistFileCount || '+ file(s) remaining in HIST bucket');
|
||||
DBMS_OUTPUT.PUT_LINE(' NOTE: These may be pre-existing files from before installation.');
|
||||
DBMS_OUTPUT.PUT_LINE(' Rollback only deletes files created during this export operation.');
|
||||
DBMS_OUTPUT.PUT_LINE(' If needed, manually verify and clean up remaining files.');
|
||||
|
||||
@@ -1324,7 +1324,8 @@ AS
|
||||
rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN
|
||||
-- Other TIMESTAMP types (without timezone)
|
||||
rec.quoted_column_name || ' TIMESTAMP ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
-- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format')
|
||||
rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN
|
||||
-- For CSV field definitions, use data_length for CHAR() specification
|
||||
rec.quoted_column_name || ' CHAR(' || rec.data_length || ')'
|
||||
@@ -1680,10 +1681,10 @@ AS
|
||||
||cgBL||pLevel||'TEMPLATE_TABLE_NAME = '||pSourceFileConfig.TEMPLATE_TABLE_NAME
|
||||
||cgBL||pLevel||'CONTAINER_FILE_KEY = '||pSourceFileConfig.CONTAINER_FILE_KEY
|
||||
||cgBL||pLevel||'ODS_SCHEMA_NAME = '||pSourceFileConfig.ODS_SCHEMA_NAME
|
||||
||cgBL||pLevel||'DAYS_FOR_ARCHIVE_THRESHOLD = '||pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
||cgBL||pLevel||'FILES_COUNT_OVER_ARCHIVE_THRESHOLD = '||pSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD
|
||||
||cgBL||pLevel||'BYTES_SUM_OVER_ARCHIVE_THRESHOLD = '||pSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD
|
||||
||cgBL||pLevel||'ROWS_COUNT_OVER_ARCHIVE_THRESHOLD = '||pSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD
|
||||
||cgBL||pLevel||'ARCHIVE_THRESHOLD_DAYS = '||pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS
|
||||
||cgBL||pLevel||'ARCHIVE_THRESHOLD_FILES_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT
|
||||
||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM
|
||||
||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT
|
||||
||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
|
||||
|
||||
||cgBL||pLevel||''||'--------------------------------'
|
||||
|
||||
@@ -17,14 +17,16 @@ AS
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00';
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) ||
|
||||
'3.5.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) ||
|
||||
'3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) ||
|
||||
'3.3.2 (2026-02-20): MARS-828 - Fixed threshold column names in GET_DET_SOURCE_FILE_CONFIG_INFO for MARS-828 compatibility' || CHR(13)||CHR(10) ||
|
||||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
-- ====================================================================
|
||||
-- T2_PEAK_LIQUIDITY_NEED Template Table
|
||||
-- ====================================================================
|
||||
-- Purpose: Template table for T2 Peak Liquidity Need data files
|
||||
-- Schema: CT_ET_TEMPLATES
|
||||
-- Created: 2026-02-24
|
||||
-- ====================================================================
|
||||
|
||||
CREATE TABLE CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED (
|
||||
A_KEY NUMBER(38,0) NOT NULL,
|
||||
A_WORKFLOW_HISTORY_KEY NUMBER(38,0) NOT NULL,
|
||||
BUSINESS_DATE DATE,
|
||||
CURRENCY VARCHAR2(3 CHAR),
|
||||
SYSTEM_ENTITY VARCHAR2(4 CHAR),
|
||||
PARTY_RIAD VARCHAR2(256 CHAR),
|
||||
PARTY_BIC VARCHAR2(256 CHAR),
|
||||
MRG_LEND_INDIC VARCHAR2(256 CHAR),
|
||||
PEAK_LIQUIDITY_NEED NUMBER(28,10),
|
||||
TIMESTAMP TIMESTAMP(6)
|
||||
);
|
||||
|
||||
-- Column comments
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.A_KEY IS 'Primary key';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.A_WORKFLOW_HISTORY_KEY IS 'Workflow history reference';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.BUSINESS_DATE IS 'Business date of the record';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.CURRENCY IS 'Currency code (ISO 4217)';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.SYSTEM_ENTITY IS 'System entity identifier';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.PARTY_RIAD IS 'Party RIAD code';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.PARTY_BIC IS 'Party BIC code';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.MRG_LEND_INDIC IS 'Marginal lending indicator';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.PEAK_LIQUIDITY_NEED IS 'Peak liquidity need amount';
|
||||
COMMENT ON COLUMN CT_ET_TEMPLATES.T2_PEAK_LIQUIDITY_NEED.TIMESTAMP IS 'Record timestamp';
|
||||
|
||||
/
|
||||
@@ -1324,7 +1324,8 @@ AS
|
||||
rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN
|
||||
-- Other TIMESTAMP types (without timezone)
|
||||
rec.quoted_column_name || ' TIMESTAMP ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
-- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format')
|
||||
rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
|
||||
WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN
|
||||
-- For CSV field definitions, use data_length for CHAR() specification
|
||||
rec.quoted_column_name || ' CHAR(' || rec.data_length || ')'
|
||||
@@ -1968,9 +1969,23 @@ AS
|
||||
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
|
||||
pSourceFileConfigKey IN NUMBER,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
-- Type for storing grant information
|
||||
TYPE tGrantRecord IS RECORD (
|
||||
grantee VARCHAR2(128),
|
||||
privilege VARCHAR2(40),
|
||||
grantable VARCHAR2(3)
|
||||
);
|
||||
TYPE tGrantList IS TABLE OF tGrantRecord;
|
||||
|
||||
vInboxGrants tGrantList;
|
||||
vOdsGrants tGrantList;
|
||||
vArchiveGrants tGrantList;
|
||||
|
||||
vSourceKey VARCHAR2(50);
|
||||
vSourceFileId VARCHAR2(100);
|
||||
vTableId VARCHAR2(100);
|
||||
@@ -1986,28 +2001,124 @@ AS
|
||||
vOdsPrefix VARCHAR2(500);
|
||||
vArchivePrefix VARCHAR2(500);
|
||||
|
||||
vTableExists NUMBER;
|
||||
vParameters VARCHAR2(4000);
|
||||
vAreaUpper VARCHAR2(20);
|
||||
|
||||
-- Nested procedure to save table grants before DROP
|
||||
PROCEDURE SAVE_GRANTS(pTableName VARCHAR2, pGrantList OUT tGrantList) IS
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants for table: ' || pTableName, 'DEBUG');
|
||||
|
||||
SELECT grantee, privilege, grantable
|
||||
BULK COLLECT INTO pGrantList
|
||||
FROM ALL_TAB_PRIVS
|
||||
WHERE table_schema = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
|
||||
AND table_name = pTableName
|
||||
AND grantee NOT IN ('SYS', 'SYSTEM', 'PUBLIC') -- Exclude system accounts
|
||||
ORDER BY grantee, privilege;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Saved ' || pGrantList.COUNT || ' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName,
|
||||
'INFO'
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
pGrantList := tGrantList(); -- Empty list
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('No grants found for table: ' || pTableName, 'INFO');
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Warning: Could not save grants for ' || pTableName || ': ' || SQLERRM,
|
||||
'WARNING'
|
||||
);
|
||||
pGrantList := tGrantList(); -- Empty list on error
|
||||
END SAVE_GRANTS;
|
||||
|
||||
-- Nested procedure to restore table grants after CREATE
|
||||
PROCEDURE RESTORE_GRANTS(pTableName VARCHAR2, pGrantList tGrantList) IS
|
||||
vGrantSQL VARCHAR2(500);
|
||||
vGrantCount NUMBER := 0;
|
||||
vFailCount NUMBER := 0;
|
||||
BEGIN
|
||||
IF pGrantList IS NULL OR pGrantList.COUNT = 0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'No grants to restore for table: ' || pTableName,
|
||||
'INFO'
|
||||
);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restoring ' || pGrantList.COUNT || ' grants for table: ' || pTableName,
|
||||
'DEBUG'
|
||||
);
|
||||
|
||||
FOR i IN 1..pGrantList.COUNT LOOP
|
||||
BEGIN
|
||||
vGrantSQL := 'GRANT ' || pGrantList(i).privilege ||
|
||||
' ON ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
|
||||
' TO ' || pGrantList(i).grantee;
|
||||
|
||||
IF pGrantList(i).grantable = 'YES' THEN
|
||||
vGrantSQL := vGrantSQL || ' WITH GRANT OPTION';
|
||||
END IF;
|
||||
|
||||
EXECUTE IMMEDIATE vGrantSQL;
|
||||
vGrantCount := vGrantCount + 1;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restored grant: ' || pGrantList(i).privilege ||
|
||||
' TO ' || pGrantList(i).grantee ||
|
||||
CASE WHEN pGrantList(i).grantable = 'YES' THEN ' WITH GRANT OPTION' ELSE '' END,
|
||||
'DEBUG'
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vFailCount := vFailCount + 1;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Warning: Could not restore grant (' || pGrantList(i).privilege ||
|
||||
' TO ' || pGrantList(i).grantee || ') on ' || pTableName || ': ' || SQLERRM,
|
||||
'WARNING'
|
||||
);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Restored ' || vGrantCount || ' of ' || pGrantList.COUNT ||
|
||||
' grants for table: ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName ||
|
||||
CASE WHEN vFailCount > 0 THEN ' (' || vFailCount || ' failed)' ELSE '' END,
|
||||
'INFO'
|
||||
);
|
||||
END RESTORE_GRANTS;
|
||||
|
||||
PROCEDURE DROP_IF_EXISTS(pTableName VARCHAR2) IS
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO vTableExists
|
||||
FROM ALL_TABLES
|
||||
WHERE OWNER = 'ODS' AND TABLE_NAME = pTableName;
|
||||
|
||||
IF vTableExists > 0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Dropping existing table: ' || pTableName, 'INFO');
|
||||
EXECUTE IMMEDIATE 'DROP TABLE ODS.' || pTableName;
|
||||
END IF;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Attempting to drop table: ' || pTableName, 'DEBUG');
|
||||
EXECUTE IMMEDIATE 'DROP TABLE ' || SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') || '.' || pTableName;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Table dropped successfully: ' || pTableName, 'INFO');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING');
|
||||
IF SQLCODE = -942 THEN -- ORA-00942: table or view does not exist
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Table does not exist, skipping drop: ' || pTableName, 'INFO');
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Error dropping table ' || pTableName || ': ' || SQLERRM, 'WARNING');
|
||||
RAISE; -- Re-raise if not "table not exists" error
|
||||
END IF;
|
||||
END DROP_IF_EXISTS;
|
||||
|
||||
BEGIN
|
||||
-- Validate and normalize pArea parameter
|
||||
vAreaUpper := UPPER(TRIM(pArea));
|
||||
|
||||
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
|
||||
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
|
||||
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
|
||||
END IF;
|
||||
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileConfigKey => ' || NVL(TO_CHAR(pSourceFileConfigKey), 'NULL'),
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pArea => ''' || vAreaUpper || ''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
|
||||
|
||||
@@ -2017,7 +2128,7 @@ AS
|
||||
NVL(ENCODING, 'UTF8')
|
||||
INTO vSourceKey, vSourceFileId, vTableId, vTemplateTableName,
|
||||
vEncoding
|
||||
FROM A_SOURCE_FILE_CONFIG
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
-- Set default delimiter (column DELIMITER does not exist in A_SOURCE_FILE_CONFIG)
|
||||
@@ -2030,9 +2141,9 @@ AS
|
||||
END;
|
||||
|
||||
-- 2. Generate table names
|
||||
vInboxTableName := vSourceKey || '_' || vTableId || '_INBOX';
|
||||
vOdsTableName := vSourceKey || '_' || vTableId || '_ODS';
|
||||
vArchiveTableName := vSourceKey || '_' || vTableId || '_ARCHIVE';
|
||||
vInboxTableName := vTableId || '_INBOX';
|
||||
vOdsTableName := vTableId || '_ODS';
|
||||
vArchiveTableName := vTableId || '_ARCHIVE';
|
||||
|
||||
-- 3. Generate paths (OFFICIAL PATH PATTERNS)
|
||||
vInboxPrefix := 'INBOX/' || vSourceKey || '/' || vSourceFileId || '/' || vTableId;
|
||||
@@ -2040,52 +2151,103 @@ AS
|
||||
vArchivePrefix := 'ARCHIVE/' || vSourceKey || '/' || vTableId;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId,
|
||||
'Creating external tables for: ' || vSourceKey || '/' || vSourceFileId || '/' || vTableId ||
|
||||
' (Area: ' || vAreaUpper || ')',
|
||||
'INFO'
|
||||
);
|
||||
|
||||
-- 4. DROP existing tables if pRecreate = TRUE
|
||||
IF pRecreate THEN
|
||||
DROP_IF_EXISTS(vInboxTableName);
|
||||
DROP_IF_EXISTS(vOdsTableName);
|
||||
DROP_IF_EXISTS(vArchiveTableName);
|
||||
-- Save grants before dropping tables (if pRestoreGrants = TRUE)
|
||||
IF pRestoreGrants THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Saving grants before dropping tables...', 'INFO');
|
||||
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
SAVE_GRANTS(vInboxTableName, vInboxGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
SAVE_GRANTS(vOdsTableName, vOdsGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
SAVE_GRANTS(vArchiveTableName, vArchiveGrants);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Drop existing tables based on pArea
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vInboxTableName);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vOdsTableName);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
DROP_IF_EXISTS(vArchiveTableName);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- 5. Create INBOX external table
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
|
||||
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vInboxTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vInboxPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
-- 5. Create INBOX external table (if requested)
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating INBOX external table: ' || vInboxTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vInboxTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vInboxPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 6. Create ODS external table
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
|
||||
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vOdsTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vOdsPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvDataBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
-- 6. Create ODS external table (if requested)
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ODS external table: ' || vOdsTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vOdsTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vOdsPrefix,
|
||||
pBucketUri => ENV_MANAGER.gvDataBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 7. Create ARCHIVE external table
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
|
||||
ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vArchiveTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vArchivePrefix,
|
||||
pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
-- 7. Create ARCHIVE external table (if requested)
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Creating ARCHIVE external table: ' || vArchiveTableName, 'INFO');
|
||||
CREATE_EXTERNAL_TABLE(
|
||||
pTableName => vArchiveTableName,
|
||||
pTemplateTableName => vTemplateTableName,
|
||||
pPrefix => vArchivePrefix,
|
||||
pBucketUri => ENV_MANAGER.gvArchiveBucketUri,
|
||||
pDelimiter => vDelimiter,
|
||||
pEncoding => vEncoding
|
||||
);
|
||||
END IF;
|
||||
|
||||
-- 8. Restore grants after creating tables (if pRecreate = TRUE and pRestoreGrants = TRUE)
|
||||
IF pRecreate AND pRestoreGrants THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Restoring grants after creating tables...', 'INFO');
|
||||
|
||||
IF vAreaUpper IN ('INBOX', 'ALL') THEN
|
||||
RESTORE_GRANTS(vInboxTableName, vInboxGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ODS', 'ALL') THEN
|
||||
RESTORE_GRANTS(vOdsTableName, vOdsGrants);
|
||||
END IF;
|
||||
|
||||
IF vAreaUpper IN ('ARCHIVE', 'ALL') THEN
|
||||
RESTORE_GRANTS(vArchiveTableName, vArchiveGrants);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'End CREATE_EXTERNAL_TABLES_SET - Successfully created all 3 external tables for config: ' || pSourceFileConfigKey,
|
||||
'End CREATE_EXTERNAL_TABLES_SET - Successfully created external tables for config: ' ||
|
||||
pSourceFileConfigKey || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO',
|
||||
vParameters
|
||||
);
|
||||
@@ -2103,19 +2265,32 @@ AS
|
||||
pSourceKey IN VARCHAR2 DEFAULT NULL,
|
||||
pSourceFileId IN VARCHAR2 DEFAULT NULL,
|
||||
pTableId IN VARCHAR2 DEFAULT NULL,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
vCount NUMBER := 0;
|
||||
vProcessed NUMBER := 0;
|
||||
vFailed NUMBER := 0;
|
||||
vParameters VARCHAR2(4000);
|
||||
vAreaUpper VARCHAR2(20);
|
||||
BEGIN
|
||||
-- Validate and normalize pArea parameter
|
||||
vAreaUpper := UPPER(TRIM(pArea));
|
||||
|
||||
IF vAreaUpper NOT IN ('INBOX', 'ODS', 'ARCHIVE', 'ALL') THEN
|
||||
vgMsgTmp := 'Invalid pArea parameter: ''' || pArea || '''. Must be one of: INBOX, ODS, ARCHIVE, ALL';
|
||||
RAISE_APPLICATION_ERROR(-20010, vgMsgTmp);
|
||||
END IF;
|
||||
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
|
||||
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
|
||||
'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END
|
||||
'pRecreate => ' || CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pRestoreGrants => ' || CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END,
|
||||
'pArea => ''' || vAreaUpper || ''''
|
||||
));
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
|
||||
@@ -2123,7 +2298,7 @@ AS
|
||||
-- Iterate over configurations matching criteria (only INPUT files)
|
||||
FOR rec IN (
|
||||
SELECT A_SOURCE_FILE_CONFIG_KEY, A_SOURCE_KEY, SOURCE_FILE_ID, TABLE_ID
|
||||
FROM A_SOURCE_FILE_CONFIG
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND (pSourceKey IS NULL OR A_SOURCE_KEY = pSourceKey)
|
||||
AND (pSourceFileId IS NULL OR SOURCE_FILE_ID = pSourceFileId)
|
||||
@@ -2135,14 +2310,16 @@ AS
|
||||
BEGIN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Creating external tables set for: ' || rec.A_SOURCE_KEY || '/' ||
|
||||
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID,
|
||||
rec.SOURCE_FILE_ID || '/' || rec.TABLE_ID || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO'
|
||||
);
|
||||
|
||||
-- Call procedure to create set of 3 tables
|
||||
-- Call procedure to create set of tables (based on pArea)
|
||||
CREATE_EXTERNAL_TABLES_SET(
|
||||
pSourceFileConfigKey => rec.A_SOURCE_FILE_CONFIG_KEY,
|
||||
pRecreate => pRecreate
|
||||
pRecreate => pRecreate,
|
||||
pRestoreGrants => pRestoreGrants,
|
||||
pArea => vAreaUpper
|
||||
);
|
||||
|
||||
vProcessed := vProcessed + 1;
|
||||
@@ -2161,7 +2338,7 @@ AS
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'End CREATE_EXTERNAL_TABLES_BATCH - Total: ' || vCount ||
|
||||
', Processed: ' || vProcessed || ', Failed: ' || vFailed,
|
||||
', Processed: ' || vProcessed || ', Failed: ' || vFailed || ' (Area: ' || vAreaUpper || ')',
|
||||
'INFO',
|
||||
vParameters
|
||||
);
|
||||
|
||||
@@ -17,12 +17,14 @@ AS
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00';
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) ||
|
||||
'3.5.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) ||
|
||||
'3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) ||
|
||||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
|
||||
@@ -602,48 +604,72 @@ AS
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLES_SET
|
||||
* @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration
|
||||
* from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following
|
||||
* official path patterns. Optionally drops and recreates existing tables.
|
||||
* @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table.
|
||||
* Automatically generates table names and paths following official path patterns.
|
||||
* Optionally drops and recreates existing tables. If pRestoreGrants is TRUE, saves and restores table grants.
|
||||
* The pArea parameter allows selective table creation.
|
||||
* @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table
|
||||
* @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist
|
||||
* @example BEGIN
|
||||
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
|
||||
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
|
||||
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
|
||||
* 'INBOX' - creates only INBOX table
|
||||
* 'ODS' - creates only ODS table
|
||||
* 'ARCHIVE' - creates only ARCHIVE table
|
||||
* 'ALL' - creates all three tables (default)
|
||||
* @example -- Create only INBOX table
|
||||
* BEGIN
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
|
||||
* pSourceFileConfigKey => 123,
|
||||
* pRecreate => FALSE
|
||||
* pArea => 'INBOX'
|
||||
* );
|
||||
* END;
|
||||
* @ex_rslt Creates three external tables in ODS schema:
|
||||
* - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX
|
||||
* - C2D_A_UC_DISSEM_METADATA_LOADS_ODS
|
||||
* - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE
|
||||
*
|
||||
* -- Create all tables with grant preservation
|
||||
* BEGIN
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
|
||||
* pSourceFileConfigKey => 123,
|
||||
* pRecreate => TRUE,
|
||||
* pRestoreGrants => TRUE,
|
||||
* pArea => 'ALL'
|
||||
* );
|
||||
* END;
|
||||
* @ex_rslt Creates external table(s) in ODS schema based on pArea parameter
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
|
||||
pSourceFileConfigKey IN NUMBER,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
);
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLES_BATCH
|
||||
* @desc Creates external table sets for multiple configurations based on filter criteria.
|
||||
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration
|
||||
* (INBOX, ODS, ARCHIVE). Continues processing even if individual sets fail.
|
||||
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter
|
||||
* (INBOX, ODS, ARCHIVE, or ALL). Continues processing even if individual sets fail.
|
||||
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operations.
|
||||
* @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources)
|
||||
* @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types)
|
||||
* @param pTableId - Filter by TABLE_ID (NULL = all tables)
|
||||
* @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist
|
||||
* @example -- Create all external tables for C2D source
|
||||
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
|
||||
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
|
||||
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
|
||||
* @example -- Create only INBOX tables for C2D source
|
||||
* BEGIN
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
|
||||
* pSourceKey => 'C2D',
|
||||
* pRecreate => FALSE
|
||||
* pArea => 'INBOX'
|
||||
* );
|
||||
* END;
|
||||
*
|
||||
* -- Recreate all external tables for all sources
|
||||
* -- Create all external tables for all sources with grant preservation
|
||||
* BEGIN
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
|
||||
* pRecreate => TRUE
|
||||
* pRecreate => TRUE,
|
||||
* pRestoreGrants => TRUE,
|
||||
* pArea => 'ALL'
|
||||
* );
|
||||
* END;
|
||||
* @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1
|
||||
@@ -652,7 +678,9 @@ AS
|
||||
pSourceKey IN VARCHAR2 DEFAULT NULL,
|
||||
pSourceFileId IN VARCHAR2 DEFAULT NULL,
|
||||
pTableId IN VARCHAR2 DEFAULT NULL,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -56,6 +56,120 @@ AS
|
||||
RAISE;
|
||||
END CREATE_EXTERNAL_TABLE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* CREATE_EXTERNAL_TABLES_SET - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET
|
||||
*/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
|
||||
pSourceFileConfigKey IN NUMBER,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vRecreateStr VARCHAR2(10);
|
||||
vRestoreGrantsStr VARCHAR2(10);
|
||||
BEGIN
|
||||
-- Convert BOOLEAN to VARCHAR2 for logging
|
||||
vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END;
|
||||
vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END;
|
||||
|
||||
-- Log the start of the procedure
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(
|
||||
SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileConfigKey => ' || pSourceFileConfigKey,
|
||||
'pRecreate => ' || vRecreateStr,
|
||||
'pRestoreGrants => ' || vRestoreGrantsStr,
|
||||
'pArea => ''' || pArea || ''''
|
||||
)
|
||||
);
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
|
||||
|
||||
-- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET procedure
|
||||
-- This ensures all logic remains centralized in the original package
|
||||
-- and ODS wrapper simply delegates execution with DEFINER rights
|
||||
CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
|
||||
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||
pRecreate => pRecreate,
|
||||
pRestoreGrants => pRestoreGrants,
|
||||
pArea => pArea
|
||||
);
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET', 'INFO', vParameters);
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET: ' || SQLERRM,
|
||||
'ERROR',
|
||||
vParameters
|
||||
);
|
||||
RAISE;
|
||||
END CREATE_EXTERNAL_TABLES_SET;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* CREATE_EXTERNAL_TABLES_BATCH - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH
|
||||
*/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
|
||||
pSourceKey IN VARCHAR2 DEFAULT NULL,
|
||||
pSourceFileId IN VARCHAR2 DEFAULT NULL,
|
||||
pTableId IN VARCHAR2 DEFAULT NULL,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
)
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vRecreateStr VARCHAR2(10);
|
||||
vRestoreGrantsStr VARCHAR2(10);
|
||||
BEGIN
|
||||
-- Convert BOOLEAN to VARCHAR2 for logging
|
||||
vRecreateStr := CASE WHEN pRecreate THEN 'TRUE' ELSE 'FALSE' END;
|
||||
vRestoreGrantsStr := CASE WHEN pRestoreGrants THEN 'TRUE' ELSE 'FALSE' END;
|
||||
|
||||
-- Log the start of the procedure
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(
|
||||
SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceKey => ''' || NVL(pSourceKey, 'NULL') || '''',
|
||||
'pSourceFileId => ''' || NVL(pSourceFileId, 'NULL') || '''',
|
||||
'pTableId => ''' || NVL(pTableId, 'NULL') || '''',
|
||||
'pRecreate => ' || vRecreateStr,
|
||||
'pRestoreGrants => ' || vRestoreGrantsStr,
|
||||
'pArea => ''' || pArea || ''''
|
||||
)
|
||||
);
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
|
||||
|
||||
-- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH procedure
|
||||
-- This ensures all logic remains centralized in the original package
|
||||
-- and ODS wrapper simply delegates execution with DEFINER rights
|
||||
CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
|
||||
pSourceKey => pSourceKey,
|
||||
pSourceFileId => pSourceFileId,
|
||||
pTableId => pTableId,
|
||||
pRecreate => pRecreate,
|
||||
pRestoreGrants => pRestoreGrants,
|
||||
pArea => pArea
|
||||
);
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH', 'INFO', vParameters);
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
|
||||
'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH: ' || SQLERRM,
|
||||
'ERROR',
|
||||
vParameters
|
||||
);
|
||||
RAISE;
|
||||
END CREATE_EXTERNAL_TABLES_BATCH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -16,12 +16,15 @@ AS
|
||||
*/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.1.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-24 11:50:00';
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.4.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-18 16:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'2.4.0 (2026-02-18): MARS-1057 - Added pArea parameter for selective table creation (INBOX/ODS/ARCHIVE/ALL)' || CHR(13)||CHR(10) ||
|
||||
'2.3.0 (2026-02-18): MARS-1057 - Added pRestoreGrants parameter support for grant preservation during table recreate' || CHR(13)||CHR(10) ||
|
||||
'2.2.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH wrappers for batch external table creation' || CHR(13)||CHR(10) ||
|
||||
'2.1.0 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'1.5.0 (2025-10-12): Enhanced external table creation with official path patterns support' || CHR(13)||CHR(10) ||
|
||||
@@ -60,6 +63,76 @@ AS
|
||||
pEncoding IN VARCHAR2 DEFAULT NULL
|
||||
);
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLES_SET
|
||||
* @desc Creates a complete set of external tables for a single configuration from A_SOURCE_FILE_CONFIG table.
|
||||
* Automatically generates table names and paths following official path patterns.
|
||||
* Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET with DEFINER rights.
|
||||
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operation.
|
||||
* The pArea parameter allows selective table creation.
|
||||
* @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table
|
||||
* @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist
|
||||
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
|
||||
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
|
||||
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
|
||||
* @example -- Create only INBOX table
|
||||
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET(
|
||||
* pSourceFileConfigKey => 123,
|
||||
* pArea => 'INBOX'
|
||||
* );
|
||||
*
|
||||
* -- Create all tables with grant preservation
|
||||
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_SET(
|
||||
* pSourceFileConfigKey => 123,
|
||||
* pRecreate => TRUE,
|
||||
* pRestoreGrants => TRUE,
|
||||
* pArea => 'ALL'
|
||||
* );
|
||||
* @ex_rslt Creates external table(s) in ODS schema based on pArea parameter
|
||||
*/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
|
||||
pSourceFileConfigKey IN NUMBER,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
);
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLES_BATCH
|
||||
* @desc Creates external table sets for multiple configurations based on filter criteria.
|
||||
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates tables based on pArea parameter.
|
||||
* Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH with DEFINER rights.
|
||||
* If pRestoreGrants is TRUE, saves and restores table grants during recreate operations.
|
||||
* @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources)
|
||||
* @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types)
|
||||
* @param pTableId - Filter by TABLE_ID (NULL = all tables)
|
||||
* @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist
|
||||
* @param pRestoreGrants - If TRUE, saves grants before DROP and restores after CREATE (only when pRecreate=TRUE)
|
||||
* Uses DBA_TAB_PRIVS - requires SELECT ANY DICTIONARY or SELECT ON DBA_TAB_PRIVS privilege
|
||||
* @param pArea - Specifies which tables to create: 'INBOX', 'ODS', 'ARCHIVE', or 'ALL' (default)
|
||||
* @example -- Create only INBOX tables for C2D source
|
||||
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH(
|
||||
* pSourceKey => 'C2D',
|
||||
* pArea => 'INBOX'
|
||||
* );
|
||||
*
|
||||
* -- Create all tables with grant preservation
|
||||
* EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLES_BATCH(
|
||||
* pRecreate => TRUE,
|
||||
* pRestoreGrants => TRUE,
|
||||
* pArea => 'ALL'
|
||||
* );
|
||||
* @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1
|
||||
*/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
|
||||
pSourceKey IN VARCHAR2 DEFAULT NULL,
|
||||
pSourceFileId IN VARCHAR2 DEFAULT NULL,
|
||||
pTableId IN VARCHAR2 DEFAULT NULL,
|
||||
pRecreate IN BOOLEAN DEFAULT FALSE,
|
||||
pRestoreGrants IN BOOLEAN DEFAULT TRUE,
|
||||
pArea IN VARCHAR2 DEFAULT 'ALL'
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
@@ -95,8 +168,8 @@ AS
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_MANAGER_ODS.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_MANAGER_ODS
|
||||
* Version: 2.1.0
|
||||
* Build Date: 2025-11-24 11:50:00
|
||||
* Version: 2.2.0
|
||||
* Build Date: 2025-11-27 15:00:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
Reference in New Issue
Block a user