Compare commits
156 Commits
aac908ebba
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a35e28042b | ||
|
|
92feb95ae0 | ||
|
|
74b8857096 | ||
|
|
24997b1583 | ||
|
|
eb9b2bc38b | ||
|
|
2ea708a694 | ||
|
|
12c58f32a3 | ||
|
|
811df6e8b1 | ||
|
|
c2e9409e55 | ||
|
|
c96bf2051f | ||
|
|
5d0e03d7ad | ||
|
|
ffd6c7eeae | ||
|
|
bbdf008125 | ||
|
|
396e7416f6 | ||
|
|
0ed75875ac | ||
|
|
a7db9b67bc | ||
|
|
ce9b6eeff6 | ||
|
|
0725119b45 | ||
|
|
896e67bcb9 | ||
|
|
ad5a6f393a | ||
|
|
a4ac132b76 | ||
|
|
6468d12349 | ||
|
|
fe0f7bce18 | ||
|
|
6b2f60f413 | ||
|
|
ca11debd93 | ||
|
|
24e6bce18c | ||
|
|
aa03dd1616 | ||
|
|
9190681051 | ||
|
|
096994d514 | ||
|
|
1385bfb9e7 | ||
|
|
7d2fb34ad9 | ||
|
|
202b535f9f | ||
|
|
5ba6c30fda | ||
|
|
64a4b9a2f0 | ||
|
|
dec3e7137e | ||
|
|
0ecc119ee9 | ||
|
|
182e6240d3 | ||
|
|
b81e524351 | ||
|
|
73e99b6e76 | ||
|
|
113ea0a618 | ||
|
|
59e18d9b35 | ||
|
|
a58a5ae82a | ||
|
|
b537719b64 | ||
|
|
4de14b64fb | ||
|
|
36a04dde04 | ||
|
|
cad6e63479 | ||
|
|
7db10725a0 | ||
|
|
a13a9d415f | ||
|
|
1c6f552df9 | ||
|
|
e9d4056451 | ||
|
|
60b218d211 | ||
|
|
819b6f7880 | ||
|
|
c68d5bfe2c | ||
|
|
c607bbe26e | ||
|
|
1569237306 | ||
|
|
472a724fe0 | ||
|
|
04d4f6ac02 | ||
|
|
ca5d8b320c | ||
|
|
2605896469 | ||
|
|
b588b0bb72 | ||
|
|
6060f93fde | ||
|
|
99aca3af40 | ||
|
|
1089184367 | ||
|
|
e538706896 | ||
|
|
ff034fcd68 | ||
|
|
b85172ae84 | ||
|
|
577c94f363 | ||
|
|
11723f6c88 | ||
|
|
b63be15f5d | ||
|
|
28972e7428 | ||
|
|
866b886c70 | ||
|
|
3848a9471b | ||
|
|
cafa30f8f3 | ||
|
|
a0f4146a24 | ||
|
|
2b116c0256 | ||
|
|
204616252a | ||
|
|
93d2e537a9 | ||
|
|
2143a81aa0 | ||
|
|
b6cf54f103 | ||
|
|
6ade77c3c0 | ||
|
|
8ce07c3360 | ||
|
|
64dc830a2b | ||
|
|
020dacb571 | ||
|
|
057a4e7ce3 | ||
|
|
a9a51f19be | ||
|
|
fc6304c60b | ||
|
|
912b7a6466 | ||
|
|
b6f6446232 | ||
|
|
aa2f2f13f9 | ||
|
|
5c77d42d9a | ||
|
|
d370b9c9ef | ||
|
|
c5d39fc01d | ||
|
|
80916ea302 | ||
|
|
a4ab56e96d | ||
|
|
92c5261215 | ||
|
|
b34c942a8f | ||
|
|
d175179ddc | ||
|
|
d8a5edc3b1 | ||
|
|
d6c34085f7 | ||
|
|
2c40f091e0 | ||
|
|
255044f23b | ||
|
|
0053ecc556 | ||
|
|
3b7ce7de97 | ||
|
|
a3ec31341b | ||
|
|
6039d6bce4 | ||
|
|
a7d286b1e6 | ||
|
|
f2bcdb3a76 | ||
|
|
15dafbff2a | ||
|
|
3261aa1a6d | ||
|
|
1327410880 | ||
|
|
f4e9fb9cb2 | ||
|
|
f8213b76ab | ||
|
|
641af7415f | ||
|
|
c1c3890a1a | ||
|
|
63ed05930e | ||
|
|
5320db627b | ||
|
|
3171ff2ddf | ||
|
|
26aba08759 | ||
|
|
d237c2d7aa | ||
|
|
96e3e2f845 | ||
|
|
3eb39091de | ||
|
|
113b9f55e3 | ||
|
|
b96becd8ef | ||
|
|
b1bb24e487 | ||
|
|
86535048fc | ||
|
|
9ffad718fe | ||
|
|
6c8b22eac9 | ||
|
|
d443c4f07d | ||
|
|
cdd9dff32d | ||
|
|
70909ba8c4 | ||
|
|
88325f0b8c | ||
|
|
dc8031262c | ||
|
|
fc95b12c9e | ||
|
|
23b9962128 | ||
|
|
802c102f29 | ||
|
|
fa1f179be9 | ||
|
|
293f2873b7 | ||
|
|
5e9233649e | ||
|
|
974f6eca25 | ||
|
|
c4f5ba48bc | ||
|
|
e6ab189dc9 | ||
|
|
e93140e962 | ||
|
|
f42777f480 | ||
|
|
19925603fa | ||
|
|
569c647b6c | ||
|
|
8a162ce9bf | ||
|
|
7864c568d7 | ||
|
|
c508e1e4e2 | ||
|
|
d6fd0b3dbd | ||
|
|
32c7a5dcee | ||
|
|
76c8ce6cc7 | ||
|
|
64965fb8c8 | ||
|
|
d17d71031a | ||
|
|
e3ff1618ce | ||
|
|
b353fb38f5 | ||
|
|
98af9fa737 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -19,6 +19,8 @@ issues/
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/export/*
|
||||
|
||||
MARS_Packages/REL01/MARS-1056/confluence/
|
||||
MARS_Packages/REL01/MARS-1056/log/
|
||||
MARS_Packages/REL01/MARS-1046/confluence/
|
||||
|
||||
7
.vscode/settings.json
vendored
7
.vscode/settings.json
vendored
@@ -121,6 +121,13 @@
|
||||
"password": "Cloudpass#34",
|
||||
"connectionString": "ggmichalski_high",
|
||||
"walletLocation": "c:\\_git\\OracleAI\\oracledb1\\Wallet_ggmichalski"
|
||||
},
|
||||
{
|
||||
"name": "OU_C2D@ggmichalski_high",
|
||||
"username": "OU_C2D",
|
||||
"password": "Cloudpass#34",
|
||||
"connectionString": "ggmichalski_high",
|
||||
"walletLocation": "c:\\_git\\OracleAI\\oracledb1\\Wallet_ggmichalski"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -24,7 +24,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -44,7 +46,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -64,7 +68,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_ADHOC_ADJ_ITEM_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -29,7 +29,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_HEADER',
|
||||
pParallelDegree => 4
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -49,7 +51,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_BALANCESHEET_ITEM',
|
||||
pParallelDegree => 16
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_BALANCESHEET_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_BALANCESHEET_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -24,7 +24,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -44,7 +46,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 2
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -64,7 +68,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 2
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_CSM_ADJ_ITEM_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -29,7 +29,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES',
|
||||
pParallelDegree => 8
|
||||
pParallelDegree => 8,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY exported');
|
||||
EXCEPTION
|
||||
@@ -49,7 +51,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_STANDING_FACILITIES_HEADER',
|
||||
pParallelDegree => 2
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_STANDING_FACILITY_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -25,7 +25,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_HEADER',
|
||||
pParallelDegree => 2
|
||||
pParallelDegree => 2,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -45,7 +47,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_CURRENT_ACCOUNTS_ITEM',
|
||||
pParallelDegree => 16
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_CURRENT_ACCOUNTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_MRR_IND_CURRENT_ACCOUNT_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -29,7 +29,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_FORECAST_HEADER',
|
||||
pParallelDegree => 4
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -49,7 +51,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_FORECAST_ITEM',
|
||||
pParallelDegree => 16
|
||||
pParallelDegree => 16,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_FORECAST_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_FORECAST_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -24,7 +24,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_HEADER',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -44,7 +46,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM',
|
||||
pParallelDegree => 4
|
||||
pParallelDegree => 4,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM exported');
|
||||
EXCEPTION
|
||||
@@ -64,7 +68,7 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_QRE_ADJUSTMENTS_ITEM_HEADER',
|
||||
pParallelDegree => 2
|
||||
pParallelDegree => 2, pTemplateTableName => 'CT_ET_TEMPLATES.LM_QRE_ADJUSTMENTS_ITEM_HEADER', pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_QR_ADJ_ITEM_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -24,7 +24,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_TTS_HEADER',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_HEADER',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_HEADER exported');
|
||||
EXCEPTION
|
||||
@@ -44,7 +46,9 @@ BEGIN
|
||||
pKeyColumnName => 'A_ETL_LOAD_SET_FK',
|
||||
pBucketArea => 'ARCHIVE',
|
||||
pFolderName => 'ARCHIVE/LM/LM_TTS_ITEM',
|
||||
pParallelDegree => 1
|
||||
pParallelDegree => 1,
|
||||
pTemplateTableName => 'CT_ET_TEMPLATES.LM_TTS_ITEM',
|
||||
pJobClass => 'high'
|
||||
);
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_TTS_ITEM exported');
|
||||
EXCEPTION
|
||||
24
MARS_Packages/REL01_ADDITIONS/MARS-828/.gitignore
vendored
Normal file
24
MARS_Packages/REL01_ADDITIONS/MARS-828/.gitignore
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
# MARS-828 Package - Git Ignore Rules
|
||||
# Standard exclusions for MARS deployment packages
|
||||
|
||||
# Confluence documentation (generated, not source)
|
||||
confluence/
|
||||
|
||||
patches/
|
||||
# Log files from SPOOL operations
|
||||
log/
|
||||
|
||||
# Test directories and files
|
||||
test/
|
||||
|
||||
# Mock data scripts (development only)
|
||||
mock_data/
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.bak
|
||||
*~
|
||||
|
||||
# OS-specific files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,58 @@
|
||||
-- MARS-828: Add archival strategy columns to A_SOURCE_FILE_CONFIG
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Description: Adds ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns to support flexible archival strategies
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Adding archival strategy columns
|
||||
PROMPT ========================================
|
||||
|
||||
-- Add new columns
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD (
|
||||
ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL,
|
||||
MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL,
|
||||
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'Y' NOT NULL
|
||||
);
|
||||
|
||||
-- Add check constraints
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
|
||||
CHK_ARCHIVAL_STRATEGY CHECK (
|
||||
ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'MINIMUM_AGE_MONTHS', 'HYBRID')
|
||||
);
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
|
||||
CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N'));
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
|
||||
CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N'));
|
||||
|
||||
-- Add comments
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
|
||||
'Archival strategy: THRESHOLD_BASED (days), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
|
||||
'Minimum age in months for archival (used with MINIMUM_AGE_MONTHS or HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0';
|
||||
|
||||
-- Verify columns added
|
||||
SELECT
|
||||
column_name,
|
||||
data_type,
|
||||
data_length,
|
||||
nullable,
|
||||
data_default
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'IS_ARCHIVE_ENABLED', 'IS_KEEP_IN_TRASH')
|
||||
ORDER BY column_id;
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Archival strategy columns added successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,49 @@
|
||||
-- MARS-828: Rename threshold columns for consistency
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-28
|
||||
-- Description: Renames threshold columns to use consistent ARCHIVE_THRESHOLD_* prefix pattern
|
||||
-- Old naming was inconsistent (DAYS_FOR vs FILES_COUNT_OVER)
|
||||
-- New naming groups all threshold columns with common prefix
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Renaming threshold columns for consistency
|
||||
PROMPT ========================================
|
||||
|
||||
-- Rename threshold columns to consistent ARCHIVE_THRESHOLD_* pattern
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN DAYS_FOR_ARCHIVE_THRESHOLD TO ARCHIVE_THRESHOLD_DAYS;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN FILES_COUNT_OVER_ARCHIVE_THRESHOLD TO ARCHIVE_THRESHOLD_FILES_COUNT;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN BYTES_SUM_OVER_ARCHIVE_THRESHOLD TO ARCHIVE_THRESHOLD_BYTES_SUM;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD TO ARCHIVE_THRESHOLD_ROWS_COUNT;
|
||||
|
||||
-- Verify column renames
|
||||
PROMPT ========================================
|
||||
PROMPT Verifying threshold column renames...
|
||||
PROMPT ========================================
|
||||
|
||||
SELECT
|
||||
column_name,
|
||||
data_type,
|
||||
data_length
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND column_name LIKE 'ARCHIVE_THRESHOLD%'
|
||||
ORDER BY column_id;
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Expected columns:
|
||||
PROMPT ARCHIVE_THRESHOLD_DAYS
|
||||
PROMPT ARCHIVE_THRESHOLD_FILES_COUNT
|
||||
PROMPT ARCHIVE_THRESHOLD_BYTES_SUM
|
||||
PROMPT ARCHIVE_THRESHOLD_ROWS_COUNT
|
||||
PROMPT ========================================
|
||||
|
||||
PROMPT Threshold columns renamed successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,160 @@
|
||||
-- =====================================================================
|
||||
-- Script: 01b_MARS_828_add_column_comments.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Purpose: Add comprehensive column comments for A_SOURCE_FILE_CONFIG and A_SOURCE_FILE_RECEIVED tables
|
||||
-- Description: Documents all columns to improve database maintainability and user understanding
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Adding comprehensive column comments
|
||||
PROMPT ========================================
|
||||
|
||||
-- =====================================================================
|
||||
-- A_SOURCE_FILE_CONFIG Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Adding column comments for A_SOURCE_FILE_CONFIG...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Primary key - unique identifier for source file configuration record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
|
||||
'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
|
||||
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
|
||||
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
|
||||
'Human-readable description of the source file and its purpose';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
|
||||
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
|
||||
'Identifier for the target table where data will be loaded (without schema prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
|
||||
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
|
||||
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
|
||||
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
|
||||
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
|
||||
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
|
||||
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
|
||||
'Schema name where ODS external tables are created (typically ODS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
|
||||
'Number of hours before table statistics expire and need to be recalculated';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
|
||||
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
|
||||
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
-- =====================================================================
|
||||
-- A_SOURCE_FILE_RECEIVED Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Adding column comments for A_SOURCE_FILE_RECEIVED...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS
|
||||
'Primary key - unique identifier for received file record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS
|
||||
'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS
|
||||
'MD5 checksum of file content for integrity verification and duplicate detection';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS
|
||||
'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS
|
||||
'File size in bytes';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS
|
||||
'Date when file was registered in the system (extracted from CREATED timestamp)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS
|
||||
'Name of temporary external table created for file validation (dropped after validation)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS
|
||||
'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS
|
||||
'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS
|
||||
'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)';
|
||||
|
||||
-- =====================================================================
|
||||
-- Verification
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT Verifying column comments...
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
COUNT(*) as total_columns,
|
||||
COUNT(comments) as documented_columns,
|
||||
COUNT(*) - COUNT(comments) as undocumented_columns
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
GROUP BY table_name
|
||||
ORDER BY table_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT Detailed column documentation status:
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
CASE WHEN comments IS NULL THEN 'MISSING' ELSE 'OK' END as comment_status
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
ORDER BY table_name, column_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Column comments added successfully
|
||||
PROMPT ========================================
|
||||
PROMPT A_SOURCE_FILE_CONFIG: All 20 columns documented
|
||||
PROMPT A_SOURCE_FILE_RECEIVED: All 12 columns documented
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,24 @@
|
||||
-- MARS-828: Create validation trigger for archival strategy
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Description: Validates archival strategy configuration consistency
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Creating archival strategy validation trigger
|
||||
PROMPT ========================================
|
||||
|
||||
@@new_version/TRG_BI_A_SRC_FILE_CFG_ARCH_VAL.sql
|
||||
|
||||
-- Verify trigger created
|
||||
SELECT
|
||||
trigger_name,
|
||||
status,
|
||||
trigger_type,
|
||||
triggering_event
|
||||
FROM all_triggers
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL';
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Archival strategy validation trigger created successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,29 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_ARCHIVER Package Specification with FN_ function names
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-11
|
||||
-- Related: MARS-828 Archival Strategy Implementation
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Specification v3.3.0
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_ARCHIVER.pkg
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_ARCHIVER'
|
||||
AND OBJECT_TYPE = 'PACKAGE';
|
||||
|
||||
PROMPT SUCCESS: FILE_ARCHIVER Package Specification v3.3.0 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -0,0 +1,38 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_ARCHIVER Package Body v3.3.0
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_ARCHIVER Package Body with config-based archival and FN_ function names
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-11
|
||||
-- Related: MARS-828 Archival Strategy Implementation
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_ARCHIVER Package Body v3.3.0
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_ARCHIVER.pkb
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_ARCHIVER'
|
||||
AND OBJECT_TYPE IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY OBJECT_TYPE;
|
||||
|
||||
-- Check for any compilation errors
|
||||
SELECT 'COMPILATION ERRORS FOUND' AS WARNING
|
||||
FROM ALL_ERRORS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND NAME = 'FILE_ARCHIVER'
|
||||
AND TYPE = 'PACKAGE BODY'
|
||||
AND ROWNUM = 1;
|
||||
|
||||
PROMPT SUCCESS: FILE_ARCHIVER Package Body v3.3.0 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -0,0 +1,157 @@
|
||||
-- MARS-828: Verify installation
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Verification Script
|
||||
PROMPT ========================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
|
||||
-- 1. Verify columns added
|
||||
PROMPT
|
||||
PROMPT 1. Verifying A_SOURCE_FILE_CONFIG columns...
|
||||
SELECT
|
||||
column_name,
|
||||
data_type,
|
||||
nullable,
|
||||
data_default
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS')
|
||||
ORDER BY column_id;
|
||||
|
||||
-- 2. Verify constraint
|
||||
PROMPT
|
||||
PROMPT 2. Verifying check constraint...
|
||||
SELECT
|
||||
constraint_name,
|
||||
constraint_type,
|
||||
search_condition
|
||||
FROM all_constraints
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND constraint_name = 'CHK_ARCHIVAL_STRATEGY';
|
||||
|
||||
-- 3. Verify trigger
|
||||
PROMPT
|
||||
PROMPT 3. Verifying validation trigger...
|
||||
SELECT
|
||||
trigger_name,
|
||||
status,
|
||||
trigger_type
|
||||
FROM all_triggers
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL';
|
||||
|
||||
-- 4. Check package compilation status
|
||||
PROMPT
|
||||
PROMPT 4. Checking FILE_ARCHIVER package status...
|
||||
SELECT
|
||||
object_name,
|
||||
object_type,
|
||||
status,
|
||||
TO_CHAR(last_ddl_time, 'YYYY-MM-DD HH24:MI:SS') as last_ddl_time
|
||||
FROM all_objects
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name = 'FILE_ARCHIVER'
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_type;
|
||||
|
||||
-- 5. Check FILE_MANAGER package compilation status
|
||||
PROMPT
|
||||
PROMPT 5. Checking FILE_MANAGER package status...
|
||||
SELECT
|
||||
object_name,
|
||||
object_type,
|
||||
status,
|
||||
TO_CHAR(last_ddl_time, 'YYYY-MM-DD HH24:MI:SS') as last_ddl_time
|
||||
FROM all_objects
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name = 'FILE_MANAGER'
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_type;
|
||||
|
||||
-- 6. Check for compilation errors
|
||||
PROMPT
|
||||
PROMPT 6. Checking for compilation errors (FILE_ARCHIVER)...
|
||||
SELECT
|
||||
name,
|
||||
type,
|
||||
line,
|
||||
position,
|
||||
text
|
||||
FROM all_errors
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND name = 'FILE_ARCHIVER'
|
||||
ORDER BY type, sequence;
|
||||
|
||||
-- 7. Check for compilation errors (FILE_MANAGER)
|
||||
PROMPT
|
||||
PROMPT 7. Checking for compilation errors (FILE_MANAGER)...
|
||||
SELECT
|
||||
name,
|
||||
type,
|
||||
line,
|
||||
position,
|
||||
text
|
||||
FROM all_errors
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND name = 'FILE_MANAGER'
|
||||
ORDER BY type, sequence;
|
||||
|
||||
-- 8. Verify package versions
|
||||
PROMPT
|
||||
PROMPT 8. Verifying package versions...
|
||||
PROMPT FILE_ARCHIVER version:
|
||||
SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() as package_version FROM DUAL;
|
||||
PROMPT FILE_MANAGER version:
|
||||
SELECT CT_MRDS.FILE_MANAGER.GET_VERSION() as package_version FROM DUAL;
|
||||
|
||||
-- 9. Test trigger validation
|
||||
PROMPT
|
||||
PROMPT 9. Testing trigger validation (should fail)...
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
SET SERVEROUTPUT ON
|
||||
DECLARE
|
||||
vTestPassed BOOLEAN := FALSE;
|
||||
BEGIN
|
||||
-- This should fail - MINIMUM_AGE_MONTHS strategy without value
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
) VALUES (
|
||||
-999,
|
||||
'TEST',
|
||||
'INPUT',
|
||||
'TEST',
|
||||
'MINIMUM_AGE_MONTHS',
|
||||
NULL -- Should trigger error
|
||||
);
|
||||
|
||||
ROLLBACK;
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: Trigger validation did not fire!');
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20999 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Trigger validation working correctly');
|
||||
DBMS_OUTPUT.PUT_LINE('Trigger correctly rejected MINIMUM_AGE_MONTHS strategy without required value');
|
||||
vTestPassed := TRUE;
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('ERROR: Unexpected error occurred during trigger validation');
|
||||
DBMS_OUTPUT.PUT_LINE('Error code: ' || SQLCODE);
|
||||
END IF;
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR EXIT FAILURE
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Verification Complete
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,350 @@
|
||||
-- =====================================================================
|
||||
-- Script: 06_MARS_828_configure_release01_tables.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Purpose: Configure COMPLETE archival parameters for Release 01 tables
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-05
|
||||
--
|
||||
-- Description:
|
||||
-- Configures ALL archival parameters for 25 tables:
|
||||
-- - ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS
|
||||
-- - Archival triggers (FILES/ROWS/BYTES thresholds)
|
||||
-- - Statistics expiration (HOURS_TO_EXPIRE_STATISTICS)
|
||||
--
|
||||
-- Configuration by group:
|
||||
-- - 19 LM tables: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 100K rows OR 1GB, 24h stats
|
||||
-- - 2 CSDB DEBT: MINIMUM_AGE_MONTHS=0 (current month only), 5 files OR 50K rows OR 512MB, 48h stats
|
||||
-- - 4 CSDB ratings: MINIMUM_AGE_MONTHS=0 (current month only), 10 files OR 20K rows OR 256MB, 72h stats
|
||||
--
|
||||
-- Dependencies:
|
||||
-- - A_SOURCE_FILE_CONFIG table with all archival columns
|
||||
-- - TRG_BI_A_SRC_FILE_CFG_ARCH_VAL trigger
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT MARS-828: Enhanced Configuration - Complete Archival Parameters
|
||||
PROMPT =====================================================================
|
||||
PROMPT
|
||||
PROMPT This script configures COMPLETE archival parameters for 25 tables
|
||||
PROMPT
|
||||
PROMPT LM Tables (19):
|
||||
PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only)
|
||||
PROMPT - Triggers: 10 files OR 100,000 rows OR 1 GB
|
||||
PROMPT - Stats Expiration: 24 hours
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT Tables (2):
|
||||
PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only)
|
||||
PROMPT - Triggers: 5 files OR 50,000 rows OR 512 MB
|
||||
PROMPT - Stats Expiration: 48 hours
|
||||
PROMPT
|
||||
PROMPT CSDB Rating/Description Tables (4):
|
||||
PROMPT - Strategy: MINIMUM_AGE_MONTHS = 0 (current month only)
|
||||
PROMPT - Triggers: 10 files OR 20,000 rows OR 256 MB
|
||||
PROMPT - Stats Expiration: 72 hours
|
||||
PROMPT
|
||||
PROMPT Current timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS CURRENT_TIME FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 1: LM Tables Configuration (MINIMUM_AGE_MONTHS = 0)
|
||||
PROMPT =====================================================================
|
||||
PROMPT Thresholds: 10 files OR 100K rows OR 1GB
|
||||
PROMPT Stats expire: 24 hours
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Update all 19 LM tables in a single statement
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
|
||||
MINIMUM_AGE_MONTHS = 0, -- 0 = current month only
|
||||
ODS_SCHEMA_NAME = 'ODS',
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = 10,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = 100000,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = 1073741824, -- 1 GB
|
||||
HOURS_TO_EXPIRE_STATISTICS = 24,
|
||||
IS_ARCHIVE_ENABLED = 'Y', -- Enable archival for all LM tables
|
||||
IS_KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'LM'
|
||||
AND TABLE_ID IN (
|
||||
'LM_STANDING_FACILITIES',
|
||||
'LM_STANDING_FACILITIES_HEADER',
|
||||
'LM_TTS_HEADER',
|
||||
'LM_TTS_ITEM',
|
||||
'LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
'LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
'LM_BALANCESHEET_HEADER',
|
||||
'LM_BALANCESHEET_ITEM',
|
||||
'LM_CSM_ADJUSTMENTS_HEADER',
|
||||
'LM_CSM_ADJUSTMENTS_ITEM',
|
||||
'LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
'LM_CURRENT_ACCOUNTS_HEADER',
|
||||
'LM_CURRENT_ACCOUNTS_ITEM',
|
||||
'LM_FORECAST_HEADER',
|
||||
'LM_FORECAST_ITEM',
|
||||
'LM_QRE_ADJUSTMENTS_HEADER',
|
||||
'LM_QRE_ADJUSTMENTS_ITEM',
|
||||
'LM_QRE_ADJUSTMENTS_ITEM_HEADER'
|
||||
);
|
||||
|
||||
PROMPT
|
||||
PROMPT LM tables configuration completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 2: CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 0)
|
||||
PROMPT =====================================================================
|
||||
PROMPT Thresholds: 5 files OR 50K rows OR 512MB
|
||||
PROMPT Stats expire: 48 hours
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Update CSDB DEBT tables (current month only)
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
|
||||
MINIMUM_AGE_MONTHS = 0,
|
||||
ODS_SCHEMA_NAME = 'ODS',
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = 5,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = 50000,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = 536870912, -- 512 MB
|
||||
HOURS_TO_EXPIRE_STATISTICS = 48,
|
||||
IS_ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB DEBT tables
|
||||
IS_KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'CSDB'
|
||||
AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY');
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT tables configuration completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 3: CSDB Rating/Description Tables (MINIMUM_AGE_MONTHS = 0)
|
||||
PROMPT =====================================================================
|
||||
PROMPT Thresholds: 10 files OR 20K rows OR 256MB
|
||||
PROMPT Stats expire: 72 hours
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Update CSDB rating/description tables
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS',
|
||||
MINIMUM_AGE_MONTHS = 0, -- 0 = current month only
|
||||
ODS_SCHEMA_NAME = 'ODS',
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = 10,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = 20000,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = 268435456, -- 256 MB
|
||||
HOURS_TO_EXPIRE_STATISTICS = 72,
|
||||
IS_ARCHIVE_ENABLED = 'Y', -- Enable archival for CSDB rating/description tables
|
||||
IS_KEEP_IN_TRASH = 'N' -- Delete files immediately after archival (no TRASH retention)
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'CSDB'
|
||||
AND TABLE_ID IN (
|
||||
'CSDB_INSTR_RAT_FULL',
|
||||
'CSDB_INSTR_DESC_FULL',
|
||||
'CSDB_ISSUER_RAT_FULL',
|
||||
'CSDB_ISSUER_DESC_FULL'
|
||||
);
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB rating/description tables configuration completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT Committing configuration changes...
|
||||
PROMPT =====================================================================
|
||||
|
||||
COMMIT;
|
||||
|
||||
PROMPT
|
||||
PROMPT Configuration committed successfully.
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT VERIFICATION: Complete Archival Configuration
|
||||
PROMPT =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT LM Tables (MINIMUM_AGE_MONTHS = 0):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
|
||||
AND MINIMUM_AGE_MONTHS = 0
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT = 10
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT = 100000
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM = 1073741824
|
||||
AND HOURS_TO_EXPIRE_STATISTICS = 24
|
||||
AND IS_ARCHIVE_ENABLED = 'Y'
|
||||
AND IS_KEEP_IN_TRASH = 'N'
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'LM'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID LIKE 'LM_%'
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT Tables (MINIMUM_AGE_MONTHS = 0):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
|
||||
AND MINIMUM_AGE_MONTHS = 0
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT = 5
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT = 50000
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM = 536870912
|
||||
AND HOURS_TO_EXPIRE_STATISTICS = 48
|
||||
AND IS_ARCHIVE_ENABLED = 'Y'
|
||||
AND IS_KEEP_IN_TRASH = 'N'
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'CSDB'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY')
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB Rating/Description Tables (MINIMUM_AGE_MONTHS = 0):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS'
|
||||
AND MINIMUM_AGE_MONTHS = 0
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT = 10
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT = 20000
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM = 268435456
|
||||
AND HOURS_TO_EXPIRE_STATISTICS = 72
|
||||
AND IS_ARCHIVE_ENABLED = 'Y'
|
||||
AND IS_KEEP_IN_TRASH = 'N'
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'CSDB'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID IN ('CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL')
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT =====================================================================
|
||||
PROMPT Summary: Total Configured Tables with Full Parameters
|
||||
PROMPT =====================================================================
|
||||
|
||||
SELECT
|
||||
COUNT(*) AS TOTAL_CONFIGURED,
|
||||
SUM(CASE WHEN MINIMUM_AGE_MONTHS = 0 THEN 1 ELSE 0 END) AS CURRENT_MONTH_ONLY,
|
||||
SUM(CASE WHEN MINIMUM_AGE_MONTHS > 0 THEN 1 ELSE 0 END) AS MULTI_MONTH_RETENTION,
|
||||
SUM(CASE WHEN ARCHIVE_THRESHOLD_FILES_COUNT IS NOT NULL THEN 1 ELSE 0 END) AS WITH_FILE_THRESHOLD,
|
||||
SUM(CASE WHEN ARCHIVE_THRESHOLD_ROWS_COUNT IS NOT NULL THEN 1 ELSE 0 END) AS WITH_ROWS_THRESHOLD,
|
||||
SUM(CASE WHEN ARCHIVE_THRESHOLD_BYTES_SUM IS NOT NULL THEN 1 ELSE 0 END) AS WITH_BYTES_THRESHOLD,
|
||||
SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NOT NULL THEN 1 ELSE 0 END) AS WITH_STATS_EXPIRY,
|
||||
SUM(CASE WHEN IS_ARCHIVE_ENABLED = 'Y' THEN 1 ELSE 0 END) AS ARCHIVAL_ENABLED,
|
||||
SUM(CASE WHEN IS_KEEP_IN_TRASH = 'N' THEN 1 ELSE 0 END) AS IMMEDIATE_DELETE
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%')
|
||||
OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN (
|
||||
'CSDB_DEBT', 'CSDB_DEBT_DAILY',
|
||||
'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL',
|
||||
'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL'
|
||||
)));
|
||||
|
||||
PROMPT
|
||||
PROMPT Expected:
|
||||
PROMPT - TOTAL_CONFIGURED = 25
|
||||
PROMPT - WITH_FILE_THRESHOLD = 25
|
||||
PROMPT - WITH_ROWS_THRESHOLD = 25
|
||||
PROMPT - WITH_BYTES_THRESHOLD = 25
|
||||
PROMPT - WITH_STATS_EXPIRY = 25
|
||||
PROMPT - ARCHIVAL_ENABLED = 25 (all tables enabled for archival)
|
||||
PROMPT - IMMEDIATE_DELETE = 25 (all tables delete files immediately, no TRASH retention)
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT Parameter Configuration Summary by Group
|
||||
PROMPT =====================================================================
|
||||
|
||||
SELECT
|
||||
CASE
|
||||
WHEN A_SOURCE_KEY = 'LM' THEN 'LM Tables'
|
||||
WHEN TABLE_ID LIKE 'CSDB_DEBT%' THEN 'CSDB DEBT'
|
||||
ELSE 'CSDB Ratings'
|
||||
END AS GROUP_NAME,
|
||||
COUNT(*) AS TABLE_COUNT,
|
||||
MAX(ARCHIVAL_STRATEGY) AS STRATEGY,
|
||||
MAX(MINIMUM_AGE_MONTHS) AS MIN_AGE,
|
||||
MAX(ARCHIVE_THRESHOLD_FILES_COUNT) AS FILES_THRESHOLD,
|
||||
MAX(ARCHIVE_THRESHOLD_ROWS_COUNT) AS ROWS_THRESHOLD,
|
||||
ROUND(MAX(ARCHIVE_THRESHOLD_BYTES_SUM)/1048576, 0) || ' MB' AS BYTES_THRESHOLD,
|
||||
MAX(HOURS_TO_EXPIRE_STATISTICS) AS STATS_HOURS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%')
|
||||
OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN (
|
||||
'CSDB_DEBT', 'CSDB_DEBT_DAILY',
|
||||
'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL',
|
||||
'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL'
|
||||
)))
|
||||
GROUP BY
|
||||
CASE
|
||||
WHEN A_SOURCE_KEY = 'LM' THEN 'LM Tables'
|
||||
WHEN TABLE_ID LIKE 'CSDB_DEBT%' THEN 'CSDB DEBT'
|
||||
ELSE 'CSDB Ratings'
|
||||
END
|
||||
ORDER BY GROUP_NAME;
|
||||
|
||||
PROMPT
|
||||
PROMPT =====================================================================
|
||||
PROMPT Configuration Complete
|
||||
PROMPT =====================================================================
|
||||
PROMPT
|
||||
PROMPT Next Steps:
|
||||
PROMPT 1. Review verification results above
|
||||
PROMPT 2. All 25 tables should show STATUS = 'OK'
|
||||
PROMPT 3. Verify parameter configuration summary matches expectations
|
||||
PROMPT 4. Monitor archival process with new thresholds
|
||||
PROMPT
|
||||
PROMPT Archival Trigger Logic (OR condition):
|
||||
PROMPT - Archival starts when ANY threshold is exceeded
|
||||
PROMPT - Example: 10 files OR 100K rows OR 1GB triggers archival
|
||||
PROMPT
|
||||
PROMPT Statistics Refresh:
|
||||
PROMPT - LM: Every 24 hours (daily updates for frequent changes)
|
||||
PROMPT - CSDB DEBT: Every 48 hours (bi-daily for larger datasets)
|
||||
PROMPT - CSDB Ratings: Every 72 hours (every 3 days for stable data)
|
||||
PROMPT
|
||||
PROMPT Log file: 06_MARS_828_configure_release01_tables.log
|
||||
PROMPT =====================================================================
|
||||
@@ -0,0 +1,46 @@
|
||||
-- MARS-828: Add TRASH retention statuses to A_SOURCE_FILE_RECEIVED
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-09
|
||||
-- Description: Adds ARCHIVED_AND_TRASHED and ARCHIVED_AND_PURGED statuses to support TRASH retention feature
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Adding TRASH retention statuses
|
||||
PROMPT ========================================
|
||||
|
||||
-- Drop old constraint
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK;
|
||||
|
||||
-- Add new constraint with TRASH retention statuses
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK
|
||||
CHECK (PROCESSING_STATUS IN (
|
||||
'RECEIVED',
|
||||
'VALIDATED',
|
||||
'VALIDATION_FAILED',
|
||||
'READY_FOR_INGESTION',
|
||||
'INGESTED',
|
||||
'ARCHIVED', -- Legacy status (backward compatibility)
|
||||
'ARCHIVED_AND_TRASHED', -- Files archived to Parquet and kept in TRASH folder (DATA bucket subfolder)
|
||||
'ARCHIVED_AND_PURGED' -- Files archived to Parquet and deleted from TRASH folder
|
||||
));
|
||||
|
||||
-- Add comment
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)';
|
||||
|
||||
-- Verify constraint
|
||||
SELECT constraint_name, constraint_type, search_condition
|
||||
FROM user_constraints
|
||||
WHERE table_name = 'A_SOURCE_FILE_RECEIVED'
|
||||
AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK';
|
||||
|
||||
-- Show current status distribution
|
||||
SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
GROUP BY PROCESSING_STATUS
|
||||
ORDER BY PROCESSING_STATUS;
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT TRASH retention statuses added successfully
|
||||
PROMPT Status flow: INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED (optional)
|
||||
PROMPT Legacy ARCHIVED status maintained for backward compatibility
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,23 @@
|
||||
-- MARS-828: Grant EXECUTE privilege on T_FILENAME to MRDS_LOADER
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-17
|
||||
-- Description: Grants EXECUTE privilege on CT_MRDS.T_FILENAME type to MRDS_LOADER user for file processing operations
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Granting EXECUTE on T_FILENAME
|
||||
PROMPT ========================================
|
||||
|
||||
-- Grant EXECUTE privilege
|
||||
GRANT EXECUTE ON CT_MRDS.T_FILENAME TO MRDS_LOADER;
|
||||
|
||||
PROMPT EXECUTE privilege on CT_MRDS.T_FILENAME granted to MRDS_LOADER
|
||||
|
||||
-- Verify grant
|
||||
SELECT GRANTEE, PRIVILEGE, GRANTABLE
|
||||
FROM USER_TAB_PRIVS
|
||||
WHERE TABLE_NAME = 'T_FILENAME'
|
||||
AND GRANTEE = 'MRDS_LOADER';
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT T_FILENAME privilege grant completed
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,29 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_MANAGER Package Specification v3.3.2
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_MANAGER Package Specification with MARS-828 column compatibility
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Related: MARS-828 Threshold Column Rename Compatibility
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_MANAGER Package Specification v3.3.2
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_MANAGER.pkg
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_MANAGER'
|
||||
AND OBJECT_TYPE = 'PACKAGE';
|
||||
|
||||
PROMPT SUCCESS: FILE_MANAGER Package Specification v3.3.2 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -0,0 +1,38 @@
|
||||
--=============================================================================================================================
|
||||
-- MARS-828: Install CT_MRDS.FILE_MANAGER Package Body v3.3.2
|
||||
--=============================================================================================================================
|
||||
-- Purpose: Deploy FILE_MANAGER Package Body with MARS-828 threshold column compatibility
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Related: MARS-828 Threshold Column Rename Compatibility
|
||||
--=============================================================================================================================
|
||||
|
||||
SET SERVEROUTPUT ON
|
||||
|
||||
PROMPT ========================================================================
|
||||
PROMPT Installing CT_MRDS.FILE_MANAGER Package Body v3.3.2
|
||||
PROMPT ========================================================================
|
||||
|
||||
@@new_version/FILE_MANAGER.pkb
|
||||
|
||||
-- Verify package compilation (check specific schema when installing as ADMIN)
|
||||
SELECT OBJECT_NAME, OBJECT_TYPE, STATUS
|
||||
FROM ALL_OBJECTS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND OBJECT_NAME = 'FILE_MANAGER'
|
||||
AND OBJECT_TYPE IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY OBJECT_TYPE;
|
||||
|
||||
-- Check for any compilation errors
|
||||
SELECT 'COMPILATION ERRORS FOUND' AS WARNING
|
||||
FROM ALL_ERRORS
|
||||
WHERE OWNER = 'CT_MRDS'
|
||||
AND NAME = 'FILE_MANAGER'
|
||||
AND TYPE = 'PACKAGE BODY'
|
||||
AND ROWNUM = 1;
|
||||
|
||||
PROMPT SUCCESS: FILE_MANAGER Package Body v3.3.2 installed
|
||||
|
||||
--=============================================================================================================================
|
||||
-- End of Script
|
||||
--=============================================================================================================================
|
||||
@@ -0,0 +1,78 @@
|
||||
-- MARS-828 ROLLBACK: Remove TRASH retention statuses from A_SOURCE_FILE_RECEIVED
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-09
|
||||
-- Description: Rollback TRASH retention statuses to original constraint
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828 ROLLBACK: Removing TRASH retention statuses
|
||||
PROMPT ========================================
|
||||
|
||||
-- Check if any files have new statuses
|
||||
DECLARE
|
||||
vTrashCount NUMBER;
|
||||
vPurgedCount NUMBER;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO vTrashCount
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
|
||||
SELECT COUNT(*) INTO vPurgedCount
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_PURGED';
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_TRASHED status: ' || vTrashCount);
|
||||
DBMS_OUTPUT.PUT_LINE('Files with ARCHIVED_AND_PURGED status: ' || vPurgedCount);
|
||||
|
||||
IF vTrashCount > 0 OR vPurgedCount > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('WARNING: Files exist with new statuses!');
|
||||
DBMS_OUTPUT.PUT_LINE('Migrating statuses to ARCHIVED before rollback...');
|
||||
|
||||
-- Migrate new statuses back to ARCHIVED
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED'
|
||||
WHERE PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED');
|
||||
|
||||
COMMIT;
|
||||
DBMS_OUTPUT.PUT_LINE('Migrated ' || SQL%ROWCOUNT || ' records to ARCHIVED status');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('No files with new statuses - safe to rollback');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Drop new constraint
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED DROP CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK;
|
||||
|
||||
-- Restore original constraint
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK
|
||||
CHECK (PROCESSING_STATUS IN (
|
||||
'RECEIVED',
|
||||
'VALIDATED',
|
||||
'VALIDATION_FAILED',
|
||||
'READY_FOR_INGESTION',
|
||||
'INGESTED',
|
||||
'ARCHIVED'
|
||||
));
|
||||
|
||||
-- Remove comment
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'File processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED';
|
||||
|
||||
-- Verify constraint restored
|
||||
SELECT constraint_name, constraint_type, search_condition
|
||||
FROM user_constraints
|
||||
WHERE table_name = 'A_SOURCE_FILE_RECEIVED'
|
||||
AND constraint_name = 'A_SOURCE_FILE_RECEIVED_CHK';
|
||||
|
||||
-- Show current status distribution
|
||||
SELECT PROCESSING_STATUS, COUNT(*) as FILE_COUNT
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
GROUP BY PROCESSING_STATUS
|
||||
ORDER BY PROCESSING_STATUS;
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT TRASH retention statuses rollback completed
|
||||
PROMPT Original constraint restored
|
||||
PROMPT All ARCHIVED_AND_TRASHED/ARCHIVED_AND_PURGED migrated to ARCHIVED
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,11 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_ARCHIVER Package Specification to v2.0.0
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package specification version (pre-MARS-828)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- WARNING: This removes all MARS-828 version information updates
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/v2.0.0/FILE_ARCHIVER.pkg
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_ARCHIVER Package Body to v2.0.0
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package body version (pre-MARS-828)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- WARNING: This removes all MARS-828 archival strategy enhancements
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/v2.0.0/FILE_ARCHIVER.pkb
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- MARS-828: Rollback validation trigger
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Description: Drop archival strategy validation trigger
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Dropping archival strategy validation trigger
|
||||
PROMPT ========================================
|
||||
|
||||
DROP TRIGGER CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL;
|
||||
|
||||
-- Verify trigger dropped
|
||||
SELECT COUNT(*) as trigger_count
|
||||
FROM all_triggers
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND trigger_name = 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL';
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Validation trigger dropped successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,41 @@
|
||||
-- MARS-828: Rollback archival strategy columns
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Description: Remove ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, and IS_KEEP_IN_TRASH columns
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Removing archival strategy and config columns
|
||||
PROMPT ========================================
|
||||
|
||||
-- Drop check constraints first
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
DROP CONSTRAINT CHK_IS_ARCHIVE_ENABLED;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
DROP CONSTRAINT CHK_IS_KEEP_IN_TRASH;
|
||||
|
||||
-- Drop columns
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
);
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
|
||||
IS_ARCHIVE_ENABLED,
|
||||
IS_KEEP_IN_TRASH
|
||||
);
|
||||
|
||||
-- Verify columns dropped
|
||||
SELECT
|
||||
column_name
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND column_name IN ('ARCHIVAL_STRATEGY', 'MINIMUM_AGE_MONTHS', 'IS_ARCHIVE_ENABLED', 'IS_KEEP_IN_TRASH');
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Archival strategy and config columns removed successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,47 @@
|
||||
-- MARS-828: Rollback threshold column renames
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-28
|
||||
-- Description: Reverts threshold columns back to original naming
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Rolling back threshold column renames
|
||||
PROMPT ========================================
|
||||
|
||||
-- Revert threshold columns to original names
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN ARCHIVE_THRESHOLD_DAYS TO DAYS_FOR_ARCHIVE_THRESHOLD;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN ARCHIVE_THRESHOLD_FILES_COUNT TO FILES_COUNT_OVER_ARCHIVE_THRESHOLD;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN ARCHIVE_THRESHOLD_BYTES_SUM TO BYTES_SUM_OVER_ARCHIVE_THRESHOLD;
|
||||
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
RENAME COLUMN ARCHIVE_THRESHOLD_ROWS_COUNT TO ROWS_COUNT_OVER_ARCHIVE_THRESHOLD;
|
||||
|
||||
-- Verify rollback
|
||||
PROMPT ========================================
|
||||
PROMPT Verifying threshold column rollback...
|
||||
PROMPT ========================================
|
||||
|
||||
SELECT
|
||||
column_name,
|
||||
data_type,
|
||||
data_length
|
||||
FROM all_tab_columns
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name = 'A_SOURCE_FILE_CONFIG'
|
||||
AND (column_name LIKE '%ARCHIVE_THRESHOLD%' OR column_name LIKE 'DAYS_FOR%')
|
||||
ORDER BY column_id;
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT Expected original columns:
|
||||
PROMPT DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
PROMPT FILES_COUNT_OVER_ARCHIVE_THRESHOLD
|
||||
PROMPT BYTES_SUM_OVER_ARCHIVE_THRESHOLD
|
||||
PROMPT ROWS_COUNT_OVER_ARCHIVE_THRESHOLD
|
||||
PROMPT ========================================
|
||||
|
||||
PROMPT Threshold column renames rolled back successfully
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,84 @@
|
||||
-- =====================================================================
|
||||
-- Script: 94b_MARS_828_rollback_column_comments.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- Purpose: Remove column comments added by 01b_MARS_828_add_column_comments.sql
|
||||
-- Description: Optional rollback - removes documentation but does not affect functionality
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Removing column comments (optional)
|
||||
PROMPT ========================================
|
||||
|
||||
-- =====================================================================
|
||||
-- Remove A_SOURCE_FILE_CONFIG Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Removing column comments from A_SOURCE_FILE_CONFIG...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS '';
|
||||
|
||||
-- =====================================================================
|
||||
-- Remove A_SOURCE_FILE_RECEIVED Column Comments
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT Removing column comments from A_SOURCE_FILE_RECEIVED...
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS '';
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS '';
|
||||
|
||||
-- =====================================================================
|
||||
-- Verification
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT Verifying column comments removed...
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
table_name,
|
||||
COUNT(*) as total_columns,
|
||||
COUNT(CASE WHEN comments IS NOT NULL AND LENGTH(comments) > 0 THEN 1 END) as documented_columns
|
||||
FROM all_col_comments
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED')
|
||||
GROUP BY table_name
|
||||
ORDER BY table_name;
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT Column comments removed successfully
|
||||
PROMPT ========================================
|
||||
PROMPT NOTE: This is an optional rollback step
|
||||
PROMPT Database functionality is not affected
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,23 @@
|
||||
-- MARS-828 ROLLBACK: Revoke EXECUTE privilege on T_FILENAME from MRDS_LOADER
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-17
|
||||
-- Description: Revokes EXECUTE privilege on CT_MRDS.T_FILENAME type from MRDS_LOADER user
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Revoking EXECUTE on T_FILENAME
|
||||
PROMPT ========================================
|
||||
|
||||
-- Revoke EXECUTE privilege
|
||||
REVOKE EXECUTE ON CT_MRDS.T_FILENAME FROM MRDS_LOADER;
|
||||
|
||||
PROMPT EXECUTE privilege on CT_MRDS.T_FILENAME revoked from MRDS_LOADER
|
||||
|
||||
-- Verify revocation
|
||||
SELECT GRANTEE, PRIVILEGE, GRANTABLE
|
||||
FROM USER_TAB_PRIVS
|
||||
WHERE TABLE_NAME = 'T_FILENAME'
|
||||
AND GRANTEE = 'MRDS_LOADER';
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT T_FILENAME privilege revocation completed
|
||||
PROMPT ========================================
|
||||
@@ -0,0 +1,274 @@
|
||||
-- =====================================================================
|
||||
-- Script: 96_MARS_828_rollback_release01_configuration.sql
|
||||
-- MARS Issue: MARS-828
|
||||
-- Purpose: ROLLBACK archival configuration for Release 01 tables
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-05
|
||||
--
|
||||
-- Description:
|
||||
-- Rolls back archival configuration for 25 tables by setting all
|
||||
-- archival parameters back to NULL (unconfigured state):
|
||||
-- - ARCHIVAL_STRATEGY
|
||||
-- - MINIMUM_AGE_MONTHS
|
||||
-- - ARCHIVE_THRESHOLD_FILES_COUNT
|
||||
-- - ARCHIVE_THRESHOLD_ROWS_COUNT
|
||||
-- - ARCHIVE_THRESHOLD_BYTES_SUM
|
||||
-- - HOURS_TO_EXPIRE_STATISTICS
|
||||
--
|
||||
-- This script reverts changes made by:
|
||||
-- - 06_MARS_828_configure_release01_tables.sql
|
||||
--
|
||||
-- Dependencies:
|
||||
-- - A_SOURCE_FILE_CONFIG table with archival columns
|
||||
-- =====================================================================
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT MARS-828: ROLLBACK - Remove Archival Configuration
|
||||
PROMPT =====================================================================
|
||||
PROMPT
|
||||
PROMPT This script will REMOVE archival configuration from 25 tables
|
||||
PROMPT
|
||||
PROMPT WARNING: This will reset ALL archival parameters to NULL
|
||||
PROMPT
|
||||
PROMPT Tables affected:
|
||||
PROMPT - 19 LM tables
|
||||
PROMPT - 2 CSDB DEBT tables
|
||||
PROMPT - 4 CSDB rating/description tables
|
||||
PROMPT
|
||||
PROMPT Current timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS CURRENT_TIME FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 1: Rollback LM Tables Configuration
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Rollback all 19 LM tables in a single statement
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = NULL,
|
||||
MINIMUM_AGE_MONTHS = NULL,
|
||||
ODS_SCHEMA_NAME = NULL,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = NULL,
|
||||
HOURS_TO_EXPIRE_STATISTICS = NULL
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'LM'
|
||||
AND TABLE_ID IN (
|
||||
'LM_STANDING_FACILITIES',
|
||||
'LM_STANDING_FACILITIES_HEADER',
|
||||
'LM_TTS_HEADER',
|
||||
'LM_TTS_ITEM',
|
||||
'LM_ADHOC_ADJUSTMENTS_HEADER',
|
||||
'LM_ADHOC_ADJUSTMENTS_ITEM',
|
||||
'LM_ADHOC_ADJUSTMENTS_ITEM_HEADER',
|
||||
'LM_BALANCESHEET_HEADER',
|
||||
'LM_BALANCESHEET_ITEM',
|
||||
'LM_CSM_ADJUSTMENTS_HEADER',
|
||||
'LM_CSM_ADJUSTMENTS_ITEM',
|
||||
'LM_CSM_ADJUSTMENTS_ITEM_HEADER',
|
||||
'LM_CURRENT_ACCOUNTS_HEADER',
|
||||
'LM_CURRENT_ACCOUNTS_ITEM',
|
||||
'LM_FORECAST_HEADER',
|
||||
'LM_FORECAST_ITEM',
|
||||
'LM_QRE_ADJUSTMENTS_HEADER',
|
||||
'LM_QRE_ADJUSTMENTS_ITEM',
|
||||
'LM_QRE_ADJUSTMENTS_ITEM_HEADER'
|
||||
);
|
||||
|
||||
PROMPT
|
||||
PROMPT LM tables rollback completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 2: Rollback CSDB DEBT Tables Configuration
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Rollback CSDB DEBT tables
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = NULL,
|
||||
MINIMUM_AGE_MONTHS = NULL,
|
||||
ODS_SCHEMA_NAME = NULL,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = NULL,
|
||||
HOURS_TO_EXPIRE_STATISTICS = NULL
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'CSDB'
|
||||
AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY');
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT tables rollback completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT SECTION 3: Rollback CSDB Rating/Description Tables Configuration
|
||||
PROMPT =====================================================================
|
||||
|
||||
-- Rollback CSDB rating/description tables
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
SET ARCHIVAL_STRATEGY = NULL,
|
||||
MINIMUM_AGE_MONTHS = NULL,
|
||||
ODS_SCHEMA_NAME = NULL,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT = NULL,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM = NULL,
|
||||
HOURS_TO_EXPIRE_STATISTICS = NULL
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND A_SOURCE_KEY = 'CSDB'
|
||||
AND TABLE_ID IN (
|
||||
'CSDB_INSTR_RAT_FULL',
|
||||
'CSDB_INSTR_DESC_FULL',
|
||||
'CSDB_ISSUER_RAT_FULL',
|
||||
'CSDB_ISSUER_DESC_FULL'
|
||||
);
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB rating/description tables rollback completed
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT Committing rollback changes...
|
||||
PROMPT =====================================================================
|
||||
|
||||
COMMIT;
|
||||
|
||||
PROMPT
|
||||
PROMPT Rollback committed successfully.
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT VERIFICATION: Archival Configuration Removed
|
||||
PROMPT =====================================================================
|
||||
|
||||
PROMPT
|
||||
PROMPT LM Tables (should all be NULL):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY IS NULL
|
||||
AND MINIMUM_AGE_MONTHS IS NULL
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM IS NULL
|
||||
AND HOURS_TO_EXPIRE_STATISTICS IS NULL
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR - Still configured'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'LM'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID LIKE 'LM_%'
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB DEBT Tables (should all be NULL):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY IS NULL
|
||||
AND MINIMUM_AGE_MONTHS IS NULL
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM IS NULL
|
||||
AND HOURS_TO_EXPIRE_STATISTICS IS NULL
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR - Still configured'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'CSDB'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID IN ('CSDB_DEBT', 'CSDB_DEBT_DAILY')
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT CSDB Rating/Description Tables (should all be NULL):
|
||||
PROMPT
|
||||
|
||||
SELECT
|
||||
TABLE_ID,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS,
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT AS FILE_THR,
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT AS ROW_THR,
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM AS BYTE_THR,
|
||||
HOURS_TO_EXPIRE_STATISTICS AS STATS_HRS,
|
||||
CASE
|
||||
WHEN ARCHIVAL_STRATEGY IS NULL
|
||||
AND MINIMUM_AGE_MONTHS IS NULL
|
||||
AND ARCHIVE_THRESHOLD_FILES_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_ROWS_COUNT IS NULL
|
||||
AND ARCHIVE_THRESHOLD_BYTES_SUM IS NULL
|
||||
AND HOURS_TO_EXPIRE_STATISTICS IS NULL
|
||||
THEN 'OK'
|
||||
ELSE 'ERROR - Still configured'
|
||||
END AS STATUS
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE A_SOURCE_KEY = 'CSDB'
|
||||
AND SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND TABLE_ID IN ('CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL', 'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL')
|
||||
ORDER BY TABLE_ID;
|
||||
|
||||
PROMPT
|
||||
PROMPT =====================================================================
|
||||
PROMPT Summary: Total Tables with Rollback Applied
|
||||
PROMPT =====================================================================
|
||||
|
||||
SELECT
|
||||
COUNT(*) AS TOTAL_TABLES,
|
||||
SUM(CASE WHEN ARCHIVAL_STRATEGY IS NULL THEN 1 ELSE 0 END) AS STRATEGY_NULL,
|
||||
SUM(CASE WHEN MINIMUM_AGE_MONTHS IS NULL THEN 1 ELSE 0 END) AS MIN_AGE_NULL,
|
||||
SUM(CASE WHEN FILES_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS FILE_THR_NULL,
|
||||
SUM(CASE WHEN ROWS_COUNT_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS ROWS_THR_NULL,
|
||||
SUM(CASE WHEN BYTES_SUM_OVER_ARCHIVE_THRESHOLD IS NULL THEN 1 ELSE 0 END) AS BYTES_THR_NULL,
|
||||
SUM(CASE WHEN HOURS_TO_EXPIRE_STATISTICS IS NULL THEN 1 ELSE 0 END) AS STATS_NULL
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
WHERE SOURCE_FILE_TYPE = 'INPUT'
|
||||
AND ((A_SOURCE_KEY = 'LM' AND TABLE_ID LIKE 'LM_%')
|
||||
OR (A_SOURCE_KEY = 'CSDB' AND TABLE_ID IN (
|
||||
'CSDB_DEBT', 'CSDB_DEBT_DAILY',
|
||||
'CSDB_INSTR_RAT_FULL', 'CSDB_INSTR_DESC_FULL',
|
||||
'CSDB_ISSUER_RAT_FULL', 'CSDB_ISSUER_DESC_FULL'
|
||||
)));
|
||||
|
||||
PROMPT
|
||||
PROMPT Expected: All counts should be 25 (all parameters NULL for all tables)
|
||||
PROMPT - TOTAL_TABLES = 25
|
||||
PROMPT - STRATEGY_NULL = 25
|
||||
PROMPT - MIN_AGE_NULL = 25
|
||||
PROMPT - FILE_THR_NULL = 25
|
||||
PROMPT - ROWS_THR_NULL = 25
|
||||
PROMPT - BYTES_THR_NULL = 25
|
||||
PROMPT - STATS_NULL = 25
|
||||
PROMPT
|
||||
|
||||
PROMPT =====================================================================
|
||||
PROMPT Rollback Complete
|
||||
PROMPT =====================================================================
|
||||
PROMPT
|
||||
PROMPT Next Steps:
|
||||
PROMPT 1. Review verification results above
|
||||
PROMPT 2. All 25 tables should show STATUS = 'OK'
|
||||
PROMPT 3. All archival parameters should be NULL
|
||||
PROMPT 4. Tables are now in unconfigured state (no archival strategy)
|
||||
PROMPT
|
||||
PROMPT To restore configuration, re-run:
|
||||
PROMPT @06_MARS_828_configure_release01_tables.sql
|
||||
PROMPT
|
||||
PROMPT Log file: 96_MARS_828_rollback_release01_configuration.log
|
||||
PROMPT =====================================================================
|
||||
@@ -0,0 +1,10 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_MANAGER Package Specification to v3.3.1
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package specification version (pre-threshold column rename compatibility)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- WARNING: This removes MARS-828 threshold column compatibility from FILE_MANAGER
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/FILE_MANAGER.pkg
|
||||
@@ -0,0 +1,10 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Rollback FILE_MANAGER Package Body to v3.3.1
|
||||
-- ===================================================================
|
||||
-- Purpose: Restore previous package body version (pre-threshold column rename compatibility)
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-02-20
|
||||
-- WARNING: This removes MARS-828 threshold column compatibility from FILE_MANAGER
|
||||
-- ===================================================================
|
||||
|
||||
@@rollback_version/FILE_MANAGER.pkb
|
||||
159
MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql
Normal file
159
MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql
Normal file
@@ -0,0 +1,159 @@
|
||||
-- ============================================================================
|
||||
-- MARS-828 Master Installation Script
|
||||
-- ============================================================================
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Purpose: Deploy enhanced archival strategies for FILE_ARCHIVER package
|
||||
-- Target Schema: CT_MRDS
|
||||
-- Estimated Time: 2-3 minutes
|
||||
-- Prerequisites: FILE_ARCHIVER v2.0.0, ENV_MANAGER v3.x, ADMIN privileges
|
||||
-- ============================================================================
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET VERIFY OFF
|
||||
SET FEEDBACK ON
|
||||
SET ECHO OFF
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET VERIFY OFF
|
||||
SET FEEDBACK ON
|
||||
SET ECHO OFF
|
||||
|
||||
-- Create log directory if it doesn't exist
|
||||
host mkdir log 2>nul
|
||||
|
||||
-- Generate dynamic SPOOL filename with timestamp
|
||||
var filename VARCHAR2(100)
|
||||
BEGIN
|
||||
:filename := 'log/INSTALL_MARS_828_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
|
||||
END;
|
||||
/
|
||||
column filename new_value _filename
|
||||
select :filename filename from dual;
|
||||
spool &_filename
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT MARS-828 Installation Starting
|
||||
PROMPT ============================================================================
|
||||
PROMPT Package: CT_MRDS.FILE_ARCHIVER v3.3.0 + CT_MRDS.FILE_MANAGER v3.3.2
|
||||
PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving + FILE_MANAGER compatibility
|
||||
PROMPT Purpose: Flexible archival policies per data source with file retention and config-based control
|
||||
PROMPT Steps: 14 (DDL, Rename, Comments, Trigger, Statuses, Grants, Packages, Verify, Track, Configure)
|
||||
PROMPT Timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL;
|
||||
PROMPT ============================================================================
|
||||
|
||||
-- Confirm installation with user
|
||||
ACCEPT continue CHAR PROMPT 'Type YES to continue with installation, or Ctrl+C to abort: '
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
BEGIN
|
||||
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Installation aborted by user');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
-- Installation steps
|
||||
PROMPT
|
||||
PROMPT Step 1/12: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG
|
||||
PROMPT ==============================================================================
|
||||
@@01_MARS_828_install_add_archival_strategy_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 2/12: Renaming threshold columns for consistent naming
|
||||
PROMPT ==========================================================
|
||||
@@01a_MARS_828_rename_threshold_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 3/12: Adding comprehensive column comments
|
||||
PROMPT ===============================================
|
||||
@@01b_MARS_828_add_column_comments.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 4/12: Creating validation trigger
|
||||
PROMPT ======================================
|
||||
@@02_MARS_828_install_archival_strategy_trigger.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 5/12: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED
|
||||
PROMPT ===================================================================
|
||||
@@07_MARS_828_install_add_trash_retention_statuses.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 6/12: Granting privileges on T_FILENAME to MRDS_LOADER
|
||||
PROMPT ==========================================================
|
||||
@@08_MARS_828_install_grant_t_filename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 7/12: Deploying FILE_ARCHIVER Package Specification v3.3.0
|
||||
PROMPT ==============================================================
|
||||
@@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 8/14: Deploying FILE_ARCHIVER Package Body v3.3.0
|
||||
PROMPT ====================================================
|
||||
@@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 9/14: Deploying FILE_MANAGER Package Specification v3.3.2
|
||||
PROMPT =============================================================
|
||||
@@09_MARS_828_install_CT_MRDS_FILE_MANAGER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 10/14: Deploying FILE_MANAGER Package Body v3.3.2
|
||||
PROMPT ===================================================
|
||||
@@10_MARS_828_install_CT_MRDS_FILE_MANAGER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 11/14: Verifying installation
|
||||
PROMPT ==================================
|
||||
@@05_MARS_828_verify_installation.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 12/14: Tracking package versions
|
||||
PROMPT =====================================
|
||||
@@track_package_versions.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 13/14: Verifying tracked packages
|
||||
PROMPT ======================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 14/14: Configuring Release 01 tables archival strategies
|
||||
PROMPT ============================================================
|
||||
@@06_MARS_828_configure_release01_tables.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT MARS-828 Installation Completed
|
||||
PROMPT ============================================================================
|
||||
PROMPT Completion Time:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_end FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT Installation Summary:
|
||||
PROMPT - Packages Installed:
|
||||
PROMPT * CT_MRDS.FILE_ARCHIVER v3.3.0 (includes selective archiving and config-based TRASH policy)
|
||||
PROMPT * CT_MRDS.FILE_MANAGER v3.3.2 (compatible with MARS-828 threshold column renames)
|
||||
PROMPT - Strategies: THRESHOLD_BASED (default), MINIMUM_AGE_MONTHS (0=current month), HYBRID
|
||||
PROMPT - Selective Archiving: IS_ARCHIVE_ENABLED column (Y=archive, N=skip)
|
||||
PROMPT - TRASH Policy: IS_KEEP_IN_TRASH column (Y=keep files, N=delete immediately)
|
||||
PROMPT * Default: IS_ARCHIVE_ENABLED='Y', IS_KEEP_IN_TRASH='N' (archiving enabled, immediate deletion)
|
||||
PROMPT * TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME)
|
||||
PROMPT * No more pKeepInTrash parameter - policy from config only
|
||||
PROMPT - New Procedure: ARCHIVE_ALL_FOR_SOURCE(pSourceKey) for batch processing
|
||||
PROMPT - TRASH Management: RESTORE_FILE_FROM_TRASH, PURGE_TRASH_FOLDER (3-level granularity)
|
||||
PROMPT - New Statuses: ARCHIVED_AND_TRASHED, ARCHIVED_AND_PURGED
|
||||
PROMPT - Backward Compatible: Yes (default THRESHOLD_BASED, existing behavior preserved)
|
||||
PROMPT - Configured Tables: 25 Release 01 tables (19 LM + 6 CSDB)
|
||||
PROMPT - Includes All Fixes from v3.0.0 through v3.2.1
|
||||
PROMPT
|
||||
PROMPT Note: Incremental patches available in patches/ directory
|
||||
PROMPT
|
||||
PROMPT Log file: &_filename
|
||||
PROMPT ============================================================================
|
||||
|
||||
spool off
|
||||
|
||||
quit;
|
||||
@@ -1,310 +0,0 @@
|
||||
|
||||
============================================================================
|
||||
MARS-828 Installation Starting (AUTO MODE)
|
||||
============================================================================
|
||||
|
||||
INSTALL_START
|
||||
______________________
|
||||
2026-01-28 06:49:11
|
||||
|
||||
1 row selected.
|
||||
|
||||
============================================================================
|
||||
|
||||
Step 1/7: Adding archival strategy columns
|
||||
===================================================================
|
||||
========================================
|
||||
MARS-828: Adding archival strategy columns
|
||||
========================================
|
||||
|
||||
Error starting at line : 11 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\01_MARS_828_install_add_archival_strategy_columns.sql
|
||||
In command -
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD (
|
||||
ARCHIVAL_STRATEGY VARCHAR2(30) DEFAULT 'THRESHOLD_BASED' NOT NULL,
|
||||
MINIMUM_AGE_MONTHS NUMBER(3) DEFAULT NULL
|
||||
)
|
||||
Error report -
|
||||
ORA-01430: column being added already exists in table
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-01430/
|
||||
01430. 00000 - "column being added already exists in table"
|
||||
*Cause: An ALTER TABLE ADD statement specified the name of a
|
||||
column that was already in the table. All column names must be
|
||||
unique within a table.
|
||||
*Action: Specify a unique name for the new column, then
|
||||
re-execute the statement.
|
||||
|
||||
Error starting at line : 17 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\01_MARS_828_install_add_archival_strategy_columns.sql
|
||||
In command -
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG ADD CONSTRAINT
|
||||
CHK_ARCHIVAL_STRATEGY CHECK (
|
||||
ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID')
|
||||
)
|
||||
Error report -
|
||||
ORA-02264: name already used by an existing constraint
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-02264/
|
||||
02264. 00000 - "name already used by an existing constraint"
|
||||
*Cause: The specified constraint name has to be unique.
|
||||
*Action: Specify a unique constraint name for the constraint.
|
||||
|
||||
Comment created.
|
||||
|
||||
|
||||
Comment created.
|
||||
|
||||
|
||||
COLUMN_NAME DATA_TYPE DATA_LENGTH NULLABLE DATA_DEFAULT
|
||||
_____________________ ____________ ______________ ___________ _____________________
|
||||
ARCHIVAL_STRATEGY VARCHAR2 30 N 'THRESHOLD_BASED'
|
||||
MINIMUM_AGE_MONTHS NUMBER 22 Y NULL
|
||||
|
||||
2 rows selected.
|
||||
|
||||
========================================
|
||||
Archival strategy columns added successfully
|
||||
========================================
|
||||
|
||||
Step 2/7: Creating validation trigger
|
||||
======================================
|
||||
========================================
|
||||
MARS-828: Creating archival strategy validation trigger
|
||||
========================================
|
||||
|
||||
Trigger CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL compiled
|
||||
|
||||
|
||||
TRIGGER_NAME STATUS TRIGGER_TYPE TRIGGERING_EVENT
|
||||
_________________________________ __________ __________________ ___________________
|
||||
TRG_BI_A_SRC_FILE_CFG_ARCH_VAL ENABLED BEFORE EACH ROW INSERT OR UPDATE
|
||||
|
||||
1 row selected.
|
||||
|
||||
========================================
|
||||
Archival strategy validation trigger created successfully
|
||||
========================================
|
||||
|
||||
Step 3/7: Deploying FILE_ARCHIVER Spec v3.0.0
|
||||
===============================================================
|
||||
|
||||
Package CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
========================================
|
||||
FILE_ARCHIVER Specification v3.0.0 ready for installation
|
||||
========================================
|
||||
|
||||
Step 4/7: Deploying FILE_ARCHIVER Body v3.0.0
|
||||
======================================================
|
||||
|
||||
Package Body CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
LINE/COL ERROR
|
||||
--------- -------------------------------------------------------------
|
||||
36/10 PLS-00103: Encountered the symbol "\" when expecting one of the following: ( begin case declare else end exit for goto if loop mod null pragma raise return select update when while with <an identifier> <a double-quoted delimited-identifier> <a bind variable> << continue close current delete fetch lock insert open rollback savepoint set sql execute commit forall merge pipe purge json_object The symbol "case was inserted before "\" to continue.
|
||||
49/8 PLS-00103: Encountered the symbol "GET_ARCHIVAL_WHERE_CLAUSE" when expecting one of the following: case
|
||||
67/9 PLS-00103: Encountered the symbol "JOIN" when expecting one of the following: , ; for group having intersect minus order start union where connect
|
||||
Errors: check compiler log
|
||||
|
||||
Step 5/7: Verifying installation
|
||||
=================================
|
||||
========================================
|
||||
MARS-828: Verification Script
|
||||
========================================
|
||||
|
||||
1. Verifying A_SOURCE_FILE_CONFIG columns...
|
||||
|
||||
COLUMN_NAME DATA_TYPE NULLABLE DATA_DEFAULT
|
||||
_____________________ ____________ ___________ _____________________
|
||||
ARCHIVAL_STRATEGY VARCHAR2 N 'THRESHOLD_BASED'
|
||||
MINIMUM_AGE_MONTHS NUMBER Y NULL
|
||||
|
||||
2 rows selected.
|
||||
|
||||
|
||||
2. Verifying check constraint...
|
||||
|
||||
CONSTRAINT_NAME CONSTRAINT_TYPE SEARCH_CONDITION
|
||||
________________________ __________________ _________________________________________________________________________________________________
|
||||
CHK_ARCHIVAL_STRATEGY C ARCHIVAL_STRATEGY IN ('THRESHOLD_BASED', 'CURRENT_MONTH_ONLY', 'MINIMUM_AGE_MONTHS', 'HYBRID')
|
||||
|
||||
1 row selected.
|
||||
|
||||
|
||||
3. Verifying validation trigger...
|
||||
|
||||
TRIGGER_NAME STATUS TRIGGER_TYPE
|
||||
_________________________________ __________ __________________
|
||||
TRG_BI_A_SRC_FILE_CFG_ARCH_VAL ENABLED BEFORE EACH ROW
|
||||
|
||||
1 row selected.
|
||||
|
||||
|
||||
4. Checking FILE_ARCHIVER package status...
|
||||
|
||||
OBJECT_NAME OBJECT_TYPE STATUS LAST_DDL_TIME
|
||||
________________ _______________ __________ ______________________
|
||||
FILE_ARCHIVER PACKAGE VALID 2026-01-28 06:49:14
|
||||
FILE_ARCHIVER PACKAGE BODY INVALID 2026-01-28 06:49:15
|
||||
|
||||
2 rows selected.
|
||||
|
||||
|
||||
5. Checking for compilation errors...
|
||||
|
||||
NAME TYPE LINE POSITION TEXT
|
||||
________________ _______________ _______ ___________ _____________________________________________________________________________
|
||||
FILE_ARCHIVER PACKAGE BODY 36 10 PLS-00103: Encountered the symbol "\" when expecting one of the following:
|
||||
|
||||
( begin case declare else end exit for goto if loop mod null
|
||||
pragma raise return select update when while with
|
||||
<an identifier> <a double-quoted delimited-identifier>
|
||||
<a bind variable> << continue close current delete fetch lock
|
||||
insert open rollback savepoint set sql execute commit forall
|
||||
merge pipe purge json_object
|
||||
The symbol "case was inserted before "\" to continue.
|
||||
|
||||
FILE_ARCHIVER PACKAGE BODY 49 8 PLS-00103: Encountered the symbol "GET_ARCHIVAL_WHERE_CLAUSE" when expecting one of the following:
|
||||
|
||||
case
|
||||
|
||||
FILE_ARCHIVER PACKAGE BODY 67 9 PLS-00103: Encountered the symbol "JOIN" when expecting one of the following:
|
||||
|
||||
, ; for group having intersect minus order start union where
|
||||
connect
|
||||
|
||||
|
||||
3 rows selected.
|
||||
|
||||
|
||||
6. Verifying FILE_ARCHIVER version...
|
||||
|
||||
Error starting at line : 79 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\05_MARS_828_verify_installation.sql
|
||||
In command -
|
||||
SELECT CT_MRDS.FILE_ARCHIVER.GET_VERSION() as package_version FROM DUAL
|
||||
Error at Command Line : 79 Column : 68 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\05_MARS_828_verify_installation.sql
|
||||
Error report -
|
||||
SQL Error: ORA-04063: package body "CT_MRDS.FILE_ARCHIVER" has errors
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-04063/04063. 00000 - "%s has errors"
|
||||
*Cause: Attempt to execute a stored procedure or use a view that has
|
||||
errors. For stored procedures, the problem could be syntax errors
|
||||
or references to other, non-existent procedures. For views,
|
||||
the problem could be a reference in the view's defining query to
|
||||
a non-existent table.
|
||||
Can also be a table which has references to non-existent or
|
||||
inaccessible types.
|
||||
*Action: Fix the errors and/or create referenced objects as necessary.
|
||||
|
||||
More Details :
|
||||
https://docs.oracle.com/error-help/db/ora-04063/
|
||||
|
||||
7. Testing trigger validation (should fail)...
|
||||
SUCCESS: Trigger validation working correctly
|
||||
Expected error: ORA-20999: MINIMUM_AGE_MONTHS is required for MINIMUM_AGE_MONTHS strategy
|
||||
ORA-06512: at "CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL", line 26
|
||||
ORA-06512: at "CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL", line 8
|
||||
ORA-04088: error during execution of trigger 'CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL'
|
||||
|
||||
|
||||
PL/SQL procedure successfully completed.
|
||||
|
||||
|
||||
========================================
|
||||
MARS-828: Verification Complete
|
||||
========================================
|
||||
|
||||
Step 6/7: Tracking package versions
|
||||
====================================
|
||||
========================================
|
||||
Package Version Tracking
|
||||
========================================
|
||||
|
||||
Summary:
|
||||
--------
|
||||
Packages tracked: 0/1
|
||||
========================================
|
||||
|
||||
|
||||
PL/SQL procedure successfully completed.
|
||||
|
||||
|
||||
Step 7/7: Verifying tracked packages
|
||||
=====================================
|
||||
|
||||
========================================
|
||||
Package Version Verification
|
||||
========================================
|
||||
|
||||
|
||||
PACKAGE_OWNER PACKAGE_NAME VERSION STATUS
|
||||
________________ ___________________ __________ _______________________________________________________
|
||||
CT_MRDS DATA_EXPORTER 2.5.0 OK: Package CT_MRDS.DATA_EXPORTER has not changed.
|
||||
Last Tracked: 2026-01-26 17:15:41
|
||||
Version: 2.5.0
|
||||
CT_MRDS ENV_MANAGER 3.2.0 OK: Package CT_MRDS.ENV_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-26 17:15:41
|
||||
Version: 3.2.0
|
||||
CT_MRDS FILE_ARCHIVER 2.0.0 WARNING: Package CT_MRDS.FILE_ARCHIVER has changed!
|
||||
========================================
|
||||
Last Tracked Version: 2.0.0
|
||||
Last Tracked Date: 2025-11-25 16:00:36
|
||||
|
||||
SPECIFICATION Changed:
|
||||
Current Hash: 71a835e531971ca7...
|
||||
Last Hash: 836641155e237fc5...
|
||||
|
||||
BODY Changed:
|
||||
Current Hash: 6f87dff6b0394529...
|
||||
Last Hash: 9bf9b3c0e059493c...
|
||||
|
||||
RECOMMENDATION:
|
||||
1. Update PACKAGE_VERSION constant
|
||||
2. Update PACKAGE_BUILD_DATE constant
|
||||
3. Add entry to VERSION_HISTORY
|
||||
4. Call TRACK_PACKAGE_VERSION to update tracking
|
||||
CT_MRDS FILE_MANAGER 3.4.0 OK: Package CT_MRDS.FILE_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-26 11:42:32
|
||||
Version: 3.4.0
|
||||
CT_MRDS WORKFLOW_MANAGER 1.7.1 OK: Package CT_MRDS.WORKFLOW_MANAGER has not changed.
|
||||
Last Tracked: 2025-11-25 16:00:36
|
||||
Version: 1.7.1
|
||||
ODS FILE_MANAGER_ODS 2.1.0 WARNING: Package ODS.FILE_MANAGER_ODS has changed!
|
||||
========================================
|
||||
Last Tracked Version: 2.1.0
|
||||
Last Tracked Date: 2025-11-26 08:58:57
|
||||
|
||||
BODY Changed:
|
||||
Current Hash: 1d167a53256c10dd...
|
||||
Last Hash: NULL...
|
||||
|
||||
RECOMMENDATION:
|
||||
1. Update PACKAGE_VERSION constant
|
||||
2. Update PACKAGE_BUILD_DATE constant
|
||||
3. Add entry to VERSION_HISTORY
|
||||
4. Call TRACK_PACKAGE_VERSION to update tracking
|
||||
EnvironmentID set to: dev
|
||||
|
||||
|
||||
========================================
|
||||
Verification Complete
|
||||
========================================
|
||||
|
||||
Legend:
|
||||
OK - Package has not changed since last tracking
|
||||
WARNING - Package code changed without version update
|
||||
|
||||
For detailed hash information, use:
|
||||
SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL
|
||||
========================================
|
||||
|
||||
============================================================================
|
||||
MARS-828 Installation Completed
|
||||
============================================================================
|
||||
|
||||
INSTALL_END
|
||||
______________________
|
||||
2026-01-28 06:49:23
|
||||
|
||||
1 row selected.
|
||||
|
||||
============================================================================
|
||||
@@ -1,194 +0,0 @@
|
||||
|
||||
============================================================================
|
||||
MARS-828 Rollback Starting (AUTO MODE - No Confirmation)
|
||||
============================================================================
|
||||
|
||||
ROLLBACK_START
|
||||
______________________
|
||||
2026-01-29 19:52:30
|
||||
|
||||
Elapsed: 00:00:00.065
|
||||
============================================================================
|
||||
|
||||
Step 1/6: Restoring FILE_ARCHIVER Package Specification v2.0.0
|
||||
===============================================================
|
||||
|
||||
Package CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
Elapsed: 00:00:00.110
|
||||
|
||||
Step 2/6: Restoring FILE_ARCHIVER Package Body v2.0.0
|
||||
======================================================
|
||||
|
||||
Package Body CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
Elapsed: 00:00:00.133
|
||||
|
||||
Step 3/6: Dropping validation trigger
|
||||
======================================
|
||||
========================================
|
||||
MARS-828: Dropping archival strategy validation trigger
|
||||
========================================
|
||||
|
||||
Error starting at line : 10 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\93_MARS_828_rollback_trigger.sql
|
||||
In command -
|
||||
DROP TRIGGER CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL
|
||||
Error report -
|
||||
ORA-04080: trigger 'TRG_BI_A_SRC_FILE_CFG_ARCH_VAL' does not exist
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-04080/
|
||||
04080. 00000 - "trigger '%s' does not exist"
|
||||
*Cause: The TRIGGER name is invalid.
|
||||
*Action: Check the trigger name.
|
||||
Elapsed: 00:00:00.135
|
||||
|
||||
TRIGGER_COUNT
|
||||
________________
|
||||
0
|
||||
|
||||
Elapsed: 00:00:00.070
|
||||
========================================
|
||||
Validation trigger dropped successfully
|
||||
========================================
|
||||
|
||||
Step 4/6: Dropping archival strategy columns
|
||||
=============================================
|
||||
========================================
|
||||
MARS-828: Removing archival strategy columns
|
||||
========================================
|
||||
|
||||
Error starting at line : 11 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\94_MARS_828_rollback_columns.sql
|
||||
In command -
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
DROP CONSTRAINT CHK_ARCHIVAL_STRATEGY
|
||||
Error report -
|
||||
ORA-02443: Cannot drop constraint - nonexistent constraint
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-02443/
|
||||
02443. 00000 - "Cannot drop constraint - nonexistent constraint"
|
||||
*Cause: alter table drop constraint <constraint_name>
|
||||
*Action: make sure you supply correct constraint name.
|
||||
Elapsed: 00:00:00.118
|
||||
|
||||
Error starting at line : 15 File @ C:\_git\_local_rep\working_dir_02\MARS_Packages\REL01_ADDITIONS\MARS-828\94_MARS_828_rollback_columns.sql
|
||||
In command -
|
||||
ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP (
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
)
|
||||
Error report -
|
||||
ORA-00904: "MINIMUM_AGE_MONTHS": invalid identifier
|
||||
|
||||
https://docs.oracle.com/error-help/db/ora-00904/
|
||||
00904. 00000 - "%s: invalid identifier"
|
||||
*Cause: The identifier or column name entered was invalid.
|
||||
*Action: Ensure the following
|
||||
Elapsed: 00:00:00.113
|
||||
|
||||
no rows selected
|
||||
Elapsed: 00:00:00.066
|
||||
========================================
|
||||
Archival strategy columns removed successfully
|
||||
========================================
|
||||
|
||||
Step 5/6: Tracking rollback version
|
||||
====================================
|
||||
========================================
|
||||
Package Version Tracking
|
||||
========================================
|
||||
EnvironmentID set to: dev
|
||||
[2026-01-29 19:52:34] [INFO] ENV_MANAGER.TRACK_PACKAGE_VERSION: Start TRACK_PACKAGE_VERSION
|
||||
[2026-01-29 19:52:34] [INFO] ENV_MANAGER.TRACK_PACKAGE_VERSION: End TRACK_PACKAGE_VERSION - Record inserted
|
||||
|
||||
Summary:
|
||||
--------
|
||||
Packages tracked: 1/1
|
||||
|
||||
Tracked Packages:
|
||||
CT_MRDS.FILE_ARCHIVER v2.0.0
|
||||
========================================
|
||||
|
||||
|
||||
PL/SQL procedure successfully completed.
|
||||
|
||||
Elapsed: 00:00:00.232
|
||||
|
||||
Step 6/6: Verifying tracked packages
|
||||
=====================================
|
||||
|
||||
========================================
|
||||
Package Version Verification
|
||||
========================================
|
||||
|
||||
|
||||
PACKAGE_OWNER PACKAGE_NAME VERSION STATUS
|
||||
________________ ___________________ __________ ______________________________________________________
|
||||
CT_MRDS DATA_EXPORTER 2.6.3 OK: Package CT_MRDS.DATA_EXPORTER has not changed.
|
||||
Last Tracked: 2026-01-28 19:18:36
|
||||
Version: 2.6.3
|
||||
CT_MRDS ENV_MANAGER 3.2.0 OK: Package CT_MRDS.ENV_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-28 19:18:36
|
||||
Version: 3.2.0
|
||||
CT_MRDS FILE_ARCHIVER 2.0.0 OK: Package CT_MRDS.FILE_ARCHIVER has not changed.
|
||||
Last Tracked: 2026-01-29 19:52:34
|
||||
Version: 2.0.0
|
||||
CT_MRDS FILE_MANAGER 3.4.0 OK: Package CT_MRDS.FILE_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-26 11:42:32
|
||||
Version: 3.4.0
|
||||
CT_MRDS WORKFLOW_MANAGER 1.7.1 OK: Package CT_MRDS.WORKFLOW_MANAGER has not changed.
|
||||
Last Tracked: 2025-11-25 16:00:36
|
||||
Version: 1.7.1
|
||||
ODS FILE_MANAGER_ODS 2.1.0 WARNING: Package ODS.FILE_MANAGER_ODS has changed!
|
||||
========================================
|
||||
Last Tracked Version: 2.1.0
|
||||
Last Tracked Date: 2025-11-26 08:58:57
|
||||
|
||||
BODY Changed:
|
||||
Current Hash: 1d167a53256c10dd...
|
||||
Last Hash: NULL...
|
||||
|
||||
RECOMMENDATION:
|
||||
1. Update PACKAGE_VERSION constant
|
||||
2. Update PACKAGE_BUILD_DATE constant
|
||||
3. Add entry to VERSION_HISTORY
|
||||
4. Call TRACK_PACKAGE_VERSION to update tracking
|
||||
Elapsed: 00:00:00.388
|
||||
|
||||
========================================
|
||||
Verification Complete
|
||||
========================================
|
||||
|
||||
Legend:
|
||||
OK - Package has not changed since last tracking
|
||||
WARNING - Package code changed without version update
|
||||
|
||||
For detailed hash information, use:
|
||||
SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL
|
||||
========================================
|
||||
|
||||
Verification: Package Compilation Status
|
||||
=========================================
|
||||
|
||||
OBJECT_NAME OBJECT_TYPE STATUS LAST_DDL_TIME
|
||||
________________ _______________ _________ ________________
|
||||
FILE_ARCHIVER PACKAGE VALID 29-JAN-26
|
||||
FILE_ARCHIVER PACKAGE BODY VALID 29-JAN-26
|
||||
|
||||
2 rows selected.
|
||||
|
||||
Elapsed: 00:00:00.100
|
||||
|
||||
============================================================================
|
||||
MARS-828 Rollback Completed
|
||||
============================================================================
|
||||
|
||||
ROLLBACK_END
|
||||
______________________
|
||||
2026-01-29 19:52:36
|
||||
|
||||
1 row selected.
|
||||
|
||||
Elapsed: 00:00:00.058
|
||||
|
||||
Log file: ../log/ROLLBACK_MARS_828_AUTO_G45C5E88148E17E_GGMICHALSKI_20260129_195230.log
|
||||
============================================================================
|
||||
@@ -1,197 +0,0 @@
|
||||
|
||||
============================================================================
|
||||
MARS-828 Rollback Starting
|
||||
============================================================================
|
||||
WARNING: This will restore FILE_ARCHIVER to v2.0.0
|
||||
|
||||
CRITICAL IMPACT:
|
||||
1. All archival strategies revert to THRESHOLD_BASED
|
||||
2. ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS columns will be dropped
|
||||
3. Validation trigger will be removed
|
||||
4. Reconfigure archival thresholds after rollback
|
||||
|
||||
Timestamp:
|
||||
|
||||
ROLLBACK_START
|
||||
______________________
|
||||
2026-01-29 19:52:12
|
||||
|
||||
Elapsed: 00:00:00.068
|
||||
============================================================================
|
||||
Type YES to continue with rollback, or Ctrl+C to abort: YES
|
||||
old:BEGIN
|
||||
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user');
|
||||
END IF;
|
||||
END;
|
||||
|
||||
new:BEGIN
|
||||
IF 'YES' IS NULL OR TRIM('YES') IS NULL OR UPPER(TRIM('YES')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user');
|
||||
END IF;
|
||||
END;
|
||||
|
||||
PL/SQL procedure successfully completed.
|
||||
|
||||
Elapsed: 00:00:00.096
|
||||
|
||||
Step 1/6: Restoring FILE_ARCHIVER Package Specification v2.0.0
|
||||
===============================================================
|
||||
|
||||
Package CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
Elapsed: 00:00:00.382
|
||||
|
||||
Step 2/6: Restoring FILE_ARCHIVER Package Body v2.0.0
|
||||
======================================================
|
||||
|
||||
Package Body CT_MRDS.FILE_ARCHIVER compiled
|
||||
|
||||
Elapsed: 00:00:00.288
|
||||
|
||||
Step 3/6: Dropping validation trigger
|
||||
======================================
|
||||
========================================
|
||||
MARS-828: Dropping archival strategy validation trigger
|
||||
========================================
|
||||
|
||||
Trigger CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL dropped.
|
||||
|
||||
Elapsed: 00:00:00.193
|
||||
|
||||
TRIGGER_COUNT
|
||||
________________
|
||||
0
|
||||
|
||||
Elapsed: 00:00:00.335
|
||||
========================================
|
||||
Validation trigger dropped successfully
|
||||
========================================
|
||||
|
||||
Step 4/6: Dropping archival strategy columns
|
||||
=============================================
|
||||
========================================
|
||||
MARS-828: Removing archival strategy columns
|
||||
========================================
|
||||
|
||||
Table CT_MRDS.A_SOURCE_FILE_CONFIG altered.
|
||||
|
||||
Elapsed: 00:00:00.131
|
||||
|
||||
Table CT_MRDS.A_SOURCE_FILE_CONFIG altered.
|
||||
|
||||
Elapsed: 00:00:00.206
|
||||
|
||||
no rows selected
|
||||
Elapsed: 00:00:00.341
|
||||
========================================
|
||||
Archival strategy columns removed successfully
|
||||
========================================
|
||||
|
||||
Step 5/6: Tracking rollback version
|
||||
====================================
|
||||
========================================
|
||||
Package Version Tracking
|
||||
========================================
|
||||
EnvironmentID set to: dev
|
||||
[2026-01-29 19:52:16] [INFO] ENV_MANAGER.TRACK_PACKAGE_VERSION: Start TRACK_PACKAGE_VERSION
|
||||
[2026-01-29 19:52:17] [INFO] ENV_MANAGER.TRACK_PACKAGE_VERSION: End TRACK_PACKAGE_VERSION - Record inserted
|
||||
|
||||
Summary:
|
||||
--------
|
||||
Packages tracked: 1/1
|
||||
|
||||
Tracked Packages:
|
||||
CT_MRDS.FILE_ARCHIVER v2.0.0
|
||||
========================================
|
||||
|
||||
|
||||
PL/SQL procedure successfully completed.
|
||||
|
||||
Elapsed: 00:00:01.230
|
||||
|
||||
Step 6/6: Verifying tracked packages
|
||||
=====================================
|
||||
|
||||
========================================
|
||||
Package Version Verification
|
||||
========================================
|
||||
|
||||
|
||||
PACKAGE_OWNER PACKAGE_NAME VERSION STATUS
|
||||
________________ ___________________ __________ ______________________________________________________
|
||||
CT_MRDS DATA_EXPORTER 2.6.3 OK: Package CT_MRDS.DATA_EXPORTER has not changed.
|
||||
Last Tracked: 2026-01-28 19:18:36
|
||||
Version: 2.6.3
|
||||
CT_MRDS ENV_MANAGER 3.2.0 OK: Package CT_MRDS.ENV_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-28 19:18:36
|
||||
Version: 3.2.0
|
||||
CT_MRDS FILE_ARCHIVER 2.0.0 OK: Package CT_MRDS.FILE_ARCHIVER has not changed.
|
||||
Last Tracked: 2026-01-29 19:52:17
|
||||
Version: 2.0.0
|
||||
CT_MRDS FILE_MANAGER 3.4.0 OK: Package CT_MRDS.FILE_MANAGER has not changed.
|
||||
Last Tracked: 2026-01-26 11:42:32
|
||||
Version: 3.4.0
|
||||
CT_MRDS WORKFLOW_MANAGER 1.7.1 OK: Package CT_MRDS.WORKFLOW_MANAGER has not changed.
|
||||
Last Tracked: 2025-11-25 16:00:36
|
||||
Version: 1.7.1
|
||||
ODS FILE_MANAGER_ODS 2.1.0 WARNING: Package ODS.FILE_MANAGER_ODS has changed!
|
||||
========================================
|
||||
Last Tracked Version: 2.1.0
|
||||
Last Tracked Date: 2025-11-26 08:58:57
|
||||
|
||||
BODY Changed:
|
||||
Current Hash: 1d167a53256c10dd...
|
||||
Last Hash: NULL...
|
||||
|
||||
RECOMMENDATION:
|
||||
1. Update PACKAGE_VERSION constant
|
||||
2. Update PACKAGE_BUILD_DATE constant
|
||||
3. Add entry to VERSION_HISTORY
|
||||
4. Call TRACK_PACKAGE_VERSION to update tracking
|
||||
Elapsed: 00:00:00.448
|
||||
|
||||
========================================
|
||||
Verification Complete
|
||||
========================================
|
||||
|
||||
Legend:
|
||||
OK - Package has not changed since last tracking
|
||||
WARNING - Package code changed without version update
|
||||
|
||||
For detailed hash information, use:
|
||||
SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('OWNER', 'PACKAGE') FROM DUAL
|
||||
========================================
|
||||
|
||||
Verification: Package Compilation Status
|
||||
=========================================
|
||||
|
||||
OBJECT_NAME OBJECT_TYPE STATUS LAST_DDL_TIME
|
||||
________________ _______________ _________ ________________
|
||||
FILE_ARCHIVER PACKAGE VALID 29-JAN-26
|
||||
FILE_ARCHIVER PACKAGE BODY VALID 29-JAN-26
|
||||
|
||||
2 rows selected.
|
||||
|
||||
Elapsed: 00:00:00.202
|
||||
|
||||
============================================================================
|
||||
MARS-828 Rollback Completed
|
||||
============================================================================
|
||||
Completion Time:
|
||||
|
||||
ROLLBACK_END
|
||||
______________________
|
||||
2026-01-29 19:52:20
|
||||
|
||||
1 row selected.
|
||||
|
||||
Elapsed: 00:00:00.061
|
||||
|
||||
Rollback Summary:
|
||||
- Package: CT_MRDS.FILE_ARCHIVER
|
||||
- Restored Version: 2.0.0 (THRESHOLD_BASED archival only)
|
||||
- Removed Features: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID strategies
|
||||
|
||||
Log file: log/ROLLBACK_MARS_828_G45C5E88148E17E_GGMICHALSKI_20260129_195211.log
|
||||
============================================================================
|
||||
@@ -0,0 +1,110 @@
|
||||
-- ====================================================================
|
||||
-- A_SOURCE_FILE_CONFIG Table
|
||||
-- ====================================================================
|
||||
-- Purpose: Store source file configuration and processing rules
|
||||
-- MARS-1049: Added ENCODING column for CSV character set support
|
||||
-- MARS-828: Added ARCHIVAL_STRATEGY and MINIMUM_AGE_MONTHS for archival automation
|
||||
-- ====================================================================
|
||||
|
||||
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
|
||||
A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE,
|
||||
SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG'
|
||||
SOURCE_FILE_ID VARCHAR2(200),
|
||||
SOURCE_FILE_DESC VARCHAR2(2000),
|
||||
SOURCE_FILE_NAME_PATTERN VARCHAR2(200),
|
||||
TABLE_ID VARCHAR2(200),
|
||||
TEMPLATE_TABLE_NAME VARCHAR2(200),
|
||||
CONTAINER_FILE_KEY NUMBER(38,0),
|
||||
ARCHIVE_THRESHOLD_DAYS NUMBER(4,0),
|
||||
ARCHIVE_THRESHOLD_FILES_COUNT NUMBER(38,0),
|
||||
ARCHIVE_THRESHOLD_BYTES_SUM NUMBER(38,0),
|
||||
ODS_SCHEMA_NAME VARCHAR2(100),
|
||||
ARCHIVE_THRESHOLD_ROWS_COUNT NUMBER(38,0),
|
||||
HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3),
|
||||
ARCHIVAL_STRATEGY VARCHAR2(50),
|
||||
MINIMUM_AGE_MONTHS NUMBER(3,0),
|
||||
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
|
||||
IS_ARCHIVE_ENABLED CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
IS_KEEP_IN_TRASH CHAR(1) DEFAULT 'N' NOT NULL,
|
||||
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT CHK_IS_ARCHIVE_ENABLED CHECK (IS_ARCHIVE_ENABLED IN ('Y', 'N')),
|
||||
CONSTRAINT CHK_IS_KEEP_IN_TRASH CHECK (IS_KEEP_IN_TRASH IN ('Y', 'N')),
|
||||
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
|
||||
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
|
||||
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID)
|
||||
) TABLESPACE "DATA";
|
||||
|
||||
-- Primary key index (from production export)
|
||||
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK"
|
||||
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY")
|
||||
TABLESPACE "DATA";
|
||||
|
||||
-- Unique constraint index (from production export)
|
||||
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1"
|
||||
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID")
|
||||
TABLESPACE "DATA";
|
||||
|
||||
-- Column comments
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Primary key - unique identifier for source file configuration record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS
|
||||
'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS
|
||||
'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS
|
||||
'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS
|
||||
'Human-readable description of the source file and its purpose';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS
|
||||
'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS
|
||||
'Identifier for the target table where data will be loaded (without schema prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS
|
||||
'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS
|
||||
'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS
|
||||
'Threshold for THRESHOLD_BASED strategy: archive data older than N days';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS
|
||||
'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS
|
||||
'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS
|
||||
'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS
|
||||
'Schema name where ODS external tables are created (typically ODS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS
|
||||
'Number of hours before table statistics expire and need to be recalculated';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS
|
||||
'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS
|
||||
'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS
|
||||
'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS
|
||||
'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS
|
||||
'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy';
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;
|
||||
@@ -0,0 +1,66 @@
|
||||
-- ====================================================================
|
||||
-- A_SOURCE_FILE_RECEIVED Table
|
||||
-- ====================================================================
|
||||
-- Purpose: Track received files and their processing status
|
||||
-- ====================================================================
|
||||
|
||||
CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
|
||||
A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE,
|
||||
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
|
||||
SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL,
|
||||
CHECKSUM VARCHAR2(128),
|
||||
CREATED TIMESTAMP(6) WITH TIME ZONE,
|
||||
BYTES NUMBER,
|
||||
RECEPTION_DATE DATE NOT NULL,
|
||||
PROCESSING_STATUS VARCHAR2(200),
|
||||
EXTERNAL_TABLE_NAME VARCHAR2(200),
|
||||
PARTITION_YEAR VARCHAR2(4),
|
||||
PARTITION_MONTH VARCHAR2(2),
|
||||
ARCH_FILE_NAME VARCHAR2(1000),
|
||||
CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY),
|
||||
CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED', 'ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED'))
|
||||
) TABLESPACE "DATA";
|
||||
|
||||
-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)
|
||||
CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1
|
||||
ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES);
|
||||
|
||||
-- Column comments
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS
|
||||
'Primary key - unique identifier for received file record';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS
|
||||
'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS
|
||||
'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS
|
||||
'MD5 checksum of file content for integrity verification and duplicate detection';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS
|
||||
'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS
|
||||
'File size in bytes';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS
|
||||
'Date when file was registered in the system (extracted from CREATED timestamp)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS
|
||||
'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS
|
||||
'Name of temporary external table created for file validation (dropped after validation)';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS
|
||||
'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS
|
||||
'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning';
|
||||
|
||||
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_FILE_NAME IS
|
||||
'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)';
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE;
|
||||
1285
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb
Normal file
1285
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,279 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.3.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-11 12:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.3.0 (2026-02-11): Added IS_ARCHIVE_ENABLED and IS_KEEP_IN_TRASH columns to A_SOURCE_FILE_CONFIG for selective archiving and config-based TRASH policy. Removed pKeepInTrash parameter (now from config). Added ARCHIVE_ALL batch procedure with 3-level granularity (config/source/all). Added GATHER_TABLE_STAT_ALL batch statistics procedure with 3-level granularity. Added RESTORE_FILE_FROM_TRASH and PURGE_TRASH_FOLDER with 3-level granularity' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
|
||||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
|
||||
'3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) ||
|
||||
'3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) ||
|
||||
'3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
|
||||
|
||||
/**
|
||||
* @name GET_TABLE_STAT
|
||||
* @desc Private function to retrieve table statistics for archival processing.
|
||||
* Returns A_TABLE_STAT record with table metadata and row counts.
|
||||
* @param pSourceFileConfigKey - Configuration key for source file
|
||||
* @return CT_MRDS.A_TABLE_STAT%ROWTYPE - Table statistics record
|
||||
* @private Internal function for archival operations
|
||||
**/
|
||||
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER) RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
**/
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name FN_ARCHIVE_TABLE_DATA
|
||||
* @desc Function wrapper for ARCHIVE_TABLE_DATA procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
|
||||
* TRASH policy is controlled by A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH column ('Y'=keep in TRASH, 'N'=delete immediately).
|
||||
* @example SELECT FILE_ARCHIVER.FN_ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION FN_ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT
|
||||
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
|
||||
**/
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name FN_GATHER_TABLE_STAT
|
||||
* @desc Function wrapper for GATHER_TABLE_STAT procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main GATHER_TABLE_STAT procedure and captures execution result.
|
||||
* @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION FN_GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT_ALL
|
||||
* @desc Multi-level batch statistics gathering procedure with three granularity levels.
|
||||
* Processes configurations based on IS_ARCHIVE_ENABLED setting (when pOnlyEnabled=TRUE).
|
||||
* Gathers statistics for external tables and inserts data into A_TABLE_STAT and A_TABLE_STAT_HIST.
|
||||
* @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority)
|
||||
* @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (e.g., 'LM', 'C2D') (medium priority)
|
||||
* @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority)
|
||||
* @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y'
|
||||
* @example -- Level 1: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceFileConfigKey => 123);
|
||||
* @example -- Level 2: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pSourceKey => 'LM');
|
||||
* @example -- Level 3: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE);
|
||||
* @example -- All tables regardless of IS_ARCHIVE_ENABLED: CALL FILE_ARCHIVER.GATHER_TABLE_STAT_ALL(pGatherAll => TRUE, pOnlyEnabled => FALSE);
|
||||
**/
|
||||
PROCEDURE GATHER_TABLE_STAT_ALL (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
|
||||
pGatherAll IN BOOLEAN DEFAULT FALSE,
|
||||
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name FN_GATHER_TABLE_STAT_ALL
|
||||
* @desc Function wrapper for GATHER_TABLE_STAT_ALL procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main GATHER_TABLE_STAT_ALL procedure and captures execution result.
|
||||
* @param pSourceFileConfigKey - (LEVEL 1) Gather stats for specific configuration key (highest priority)
|
||||
* @param pSourceKey - (LEVEL 2) Gather stats for all tables in source system (medium priority)
|
||||
* @param pGatherAll - (LEVEL 3) When TRUE, gather stats for ALL tables across all sources (lowest priority)
|
||||
* @param pOnlyEnabled - When TRUE (default), only process tables with IS_ARCHIVE_ENABLED='Y'
|
||||
* @example SELECT FILE_ARCHIVER.FN_GATHER_TABLE_STAT_ALL(pSourceKey => 'LM') FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION FN_GATHER_TABLE_STAT_ALL (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
|
||||
pGatherAll IN BOOLEAN DEFAULT FALSE,
|
||||
pOnlyEnabled IN BOOLEAN DEFAULT TRUE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_ALL
|
||||
* @desc Multi-level batch archival procedure with three granularity levels.
|
||||
* Only processes configurations where IS_ARCHIVE_ENABLED='Y'.
|
||||
* TRASH policy for each table is controlled by individual IS_KEEP_IN_TRASH column.
|
||||
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
|
||||
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (e.g., 'LM', 'C2D') (medium priority)
|
||||
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)
|
||||
* @example -- Level 1: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceFileConfigKey => 123);
|
||||
* @example -- Level 2: CALL FILE_ARCHIVER.ARCHIVE_ALL(pSourceKey => 'LM');
|
||||
* @example -- Level 3: CALL FILE_ARCHIVER.ARCHIVE_ALL(pArchiveAll => TRUE);
|
||||
**/
|
||||
PROCEDURE ARCHIVE_ALL (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
|
||||
pArchiveAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name FN_ARCHIVE_ALL
|
||||
* @desc Function wrapper for ARCHIVE_ALL procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main ARCHIVE_ALL procedure and captures execution result.
|
||||
* @param pSourceFileConfigKey - (LEVEL 1) Archive specific configuration key (highest priority)
|
||||
* @param pSourceKey - (LEVEL 2) Archive all enabled tables for source system (medium priority)
|
||||
* @param pArchiveAll - (LEVEL 3) When TRUE, archive ALL enabled tables across all sources (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.FN_ARCHIVE_ALL(pSourceKey => 'LM') FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION FN_ARCHIVE_ALL (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE DEFAULT NULL,
|
||||
pArchiveAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Restores files from TRASH folder back to ODS at three different granularity levels.
|
||||
* Moves files from TRASH subfolder back to ODS subfolder in DATA bucket.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341);
|
||||
* @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE);
|
||||
**/
|
||||
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Function overload for RESTORE_FILE_FROM_TRASH procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Deletes files from TRASH folder at three different granularity levels.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341);
|
||||
* @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE);
|
||||
**/
|
||||
PROCEDURE PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Function overload for PURGE_TRASH_FOLDER procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main PURGE_TRASH_FOLDER procedure and captures execution result.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_ARCHIVER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 2.0.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_ARCHIVER
|
||||
* Version: 2.0.0
|
||||
* Build Date: 2025-10-22 16:45:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_ARCHIVER Version History:
|
||||
* 2.0.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
2009
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_MANAGER.pkb
Normal file
2009
MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_MANAGER.pkb
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,639 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_MANAGER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.5.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-24 13:35:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.5.1 (2026-02-24): Fixed TIMESTAMP field syntax in GENERATE_EXTERNAL_TABLE_PARAMS for SQL*Loader compatibility (CHAR(35) DATE_FORMAT TIMESTAMP MASK format)' || CHR(13)||CHR(10) ||
|
||||
'3.3.2 (2026-02-20): MARS-828 - Fixed threshold column names in GET_DET_SOURCE_FILE_CONFIG_INFO for MARS-828 compatibility' || CHR(13)||CHR(10) ||
|
||||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
|
||||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
|
||||
'3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) ||
|
||||
'3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) ||
|
||||
'2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-01): Initial release with file processing and validation capabilities';
|
||||
|
||||
TYPE tSourceFileReceived IS RECORD
|
||||
(
|
||||
A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE,
|
||||
A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
SOURCE_FILE_PREFIX_INBOX VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ODS VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430),
|
||||
SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430),
|
||||
SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE,
|
||||
RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE,
|
||||
PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE,
|
||||
EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE
|
||||
);
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
vgSourceFileConfigKey PLS_INTEGER;
|
||||
vgMsgTmp VARCHAR2(32000);
|
||||
|
||||
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG
|
||||
* @desc Get source file type by matching the source file name against source file type naming patterns
|
||||
* or by specifying the id of a received source file.
|
||||
* @example ...
|
||||
* @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE"
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL
|
||||
, pSourceFileReceivedKey IN NUMBER DEFAULT NULL
|
||||
, pSourceFileConfigKey IN NUMBER DEFAULT NULL)
|
||||
RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table.
|
||||
* This overload automatically determines source file type from the file name.
|
||||
* It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name REGISTER_SOURCE_FILE_RECEIVED
|
||||
* @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig.
|
||||
* Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
|
||||
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(
|
||||
* pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'
|
||||
* ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... );
|
||||
* @ex_rslt 3245
|
||||
**/
|
||||
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
|
||||
pSourceFileReceivedName IN VARCHAR2,
|
||||
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
|
||||
)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name SET_SOURCE_FILE_RECEIVED_STATUS
|
||||
* @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column
|
||||
* based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY
|
||||
* and provided value of pStatus parameter
|
||||
* @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION');
|
||||
**/
|
||||
PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(
|
||||
pSourceFileReceivedKey IN PLS_INTEGER,
|
||||
pStatus IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_EXTERNAL_TABLE_COLUMNS
|
||||
* @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name.
|
||||
* It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure.
|
||||
* @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual;
|
||||
* @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE,
|
||||
* "REV_NUMBER" NUMBER(28,0),
|
||||
* "REF_DATE" DATE,
|
||||
* "FREE_TEXT" VARCHAR2(1000 CHAR),
|
||||
* "MLF_BS_TOTAL" NUMBER(28,10),
|
||||
* "DF_BS_TOTAL" NUMBER(28,10),
|
||||
* "MLF_SF_TOTAL" NUMBER(28,10),
|
||||
* "DF_SF_TOTAL" NUMBER(28,10)
|
||||
**/
|
||||
FUNCTION GET_EXTERNAL_TABLE_COLUMNS (
|
||||
pTargetTableTemplate IN VARCHAR2
|
||||
)
|
||||
RETURN CLOB;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252')
|
||||
* If provided, adds CHARACTERSET clause to external table definition
|
||||
* @example
|
||||
* begin
|
||||
* FILE_MANAGER.CREATE_EXTERNAL_TABLE(
|
||||
* pTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
|
||||
* pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/',
|
||||
* pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/',
|
||||
* pFileName => NULL,
|
||||
* pDelimiter => ',',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
* end;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pTableName IN VARCHAR2,
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pPrefix IN VARCHAR2,
|
||||
pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri,
|
||||
pFileName IN VARCHAR2 DEFAULT NULL,
|
||||
pDelimiter IN VARCHAR2 DEFAULT ',',
|
||||
pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name CREATE_EXTERNAL_TABLE
|
||||
* @desc Creates External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);;
|
||||
**/
|
||||
PROCEDURE CREATE_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_SOURCE_FILE_RECEIVED
|
||||
* @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
|
||||
* It validate External table build upon single file
|
||||
* provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED
|
||||
(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
/**
|
||||
* @name VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE.
|
||||
* It validates External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt FAILED
|
||||
**/
|
||||
FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
/**
|
||||
* @name S_VALIDATE_EXTERNAL_TABLE
|
||||
* @desc A function which checks if SELECT query reterns any rows.
|
||||
* It trys to selects External Table provided by parameter pTableName.
|
||||
* It returns: PASSED or FAILED.
|
||||
* @example
|
||||
* declare
|
||||
* vStatus VARCHAR2(100);
|
||||
* begin
|
||||
* vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
|
||||
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
|
||||
* end;
|
||||
*
|
||||
* @ex_rslt PASSED
|
||||
**/
|
||||
FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DROP_EXTERNAL_TABLE
|
||||
* @desc It drops External Table for single file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);
|
||||
**/
|
||||
PROCEDURE DROP_EXTERNAL_TABLE (
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name COPY_FILE
|
||||
* @desc It copies file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS'
|
||||
* @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE COPY_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name MOVE_FILE
|
||||
* @desc It moves file provided by
|
||||
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
|
||||
* into destination provided by pDestination parameter.
|
||||
* pDestination parameter allowed values are: 'ODS', 'QUARANTINE'
|
||||
* @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
|
||||
**/
|
||||
PROCEDURE MOVE_FILE(
|
||||
pSourceFileReceivedKey IN NUMBER,
|
||||
pDestination IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_FOLDER_CONTENTS
|
||||
* @desc It deletes all files from specified folder in the cloud storage.
|
||||
* The procedure lists all objects in the specified folder prefix and deletes them one by one.
|
||||
* pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE'
|
||||
* pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/')
|
||||
* @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/');
|
||||
**/
|
||||
PROCEDURE DELETE_FOLDER_CONTENTS(
|
||||
pBucketArea IN VARCHAR2,
|
||||
pFolderPrefix IN VARCHAR2
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter.
|
||||
* Ubmrella procedure that calls:
|
||||
* - REGISTER_SOURCE_FILE_RECEIVED;
|
||||
* - CREATE_EXTERNAL_TABLE;
|
||||
* - VALIDATE_SOURCE_FILE_RECEIVED;
|
||||
* - DROP_EXTERNAL_TABLE;
|
||||
* - MOVE_FILE;
|
||||
* @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
**/
|
||||
PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name PROCESS_SOURCE_FILE
|
||||
* @desc It process file provided by pSourceFileReceivedName parameter and return processing result value.
|
||||
* It returns (success/failure) => 0 / -(value).
|
||||
* Ubmrella function that calls PROCESS_SOURCE_FILE procedure.
|
||||
* @example
|
||||
* declare
|
||||
* vResult PLS_INTEGER;
|
||||
* begin
|
||||
* vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
|
||||
* DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult);
|
||||
* end;
|
||||
* @ex_rslt 0
|
||||
* -20021
|
||||
**/
|
||||
FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
|
||||
RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DATE_FORMAT
|
||||
* @desc Returns date format for specified template table name and column name.
|
||||
* Date is taken from configuration A_COLUMN_DATE_FORMAT table.
|
||||
* @example select FILE_MANAGER.GET_DATE_FORMAT(
|
||||
* pTemplateTableName => 'STANDING_FACILITIES_HEADER',
|
||||
* pColumnName => 'SNAPSHOT_DATE')
|
||||
* from dual;
|
||||
* @ex_rslt DD/MM/YYYY HH24:MI:SS
|
||||
**/
|
||||
FUNCTION GET_DATE_FORMAT(
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnName IN VARCHAR2
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GENERATE_EXTERNAL_TABLE_PARAMS
|
||||
* @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName.
|
||||
* @example
|
||||
* declare
|
||||
* vColumnList CLOB;
|
||||
* vFieldList CLOB;
|
||||
* begin
|
||||
* FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER'
|
||||
* ,pColumnList => vColumnList
|
||||
* ,pFieldList => vFieldList
|
||||
* );
|
||||
* DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList);
|
||||
* DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList);
|
||||
* end;
|
||||
* /
|
||||
**/
|
||||
PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS (
|
||||
pTemplateTableName IN VARCHAR2,
|
||||
pColumnList OUT CLOB,
|
||||
pFieldList OUT CLOB
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE
|
||||
* @desc Insert a new record to A_SOURCE table.
|
||||
* pSourceKey is a PRIMARY KEY value.
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE,
|
||||
pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name DELETE_SOURCE_CASCADE
|
||||
* @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables:
|
||||
* - A_SOURCE_FILE_CONFIG
|
||||
* - A_SOURCE_FILE_RECEIVED
|
||||
* - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems)
|
||||
* The procedure checks if template tables are shared before deleting date format configurations.
|
||||
* If a template table is used by multiple source systems, date formats are preserved.
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS');
|
||||
**/
|
||||
PROCEDURE DELETE_SOURCE_CASCADE (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID)
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record.
|
||||
* @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(
|
||||
* pSourceFileId => 'UC_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_SOURCE_FILE_CONFIG_KEY
|
||||
* @desc For specified input parameters,
|
||||
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY.
|
||||
* @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY (
|
||||
* pSourceFileType => 'INPUT'
|
||||
* ,pSourceFileId => 'UC_DISSEM'
|
||||
* ,pTableId => 'UC_NMA_DISSEM')
|
||||
* from dual;
|
||||
* @ex_rslt 126
|
||||
**/
|
||||
FUNCTION GET_SOURCE_FILE_CONFIG_KEY (
|
||||
pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT'
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_SOURCE_FILE_CONFIG
|
||||
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
|
||||
* MARS-1049: Added pEncoding parameter for CSV character set specification.
|
||||
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
|
||||
* If NULL, no CHARACTERSET clause is added to external table definitions
|
||||
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
|
||||
* pSourceKey => 'C2D', pSourceFileType => 'INPUT',
|
||||
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
|
||||
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
|
||||
* pEncoding => 'UTF8'
|
||||
* );
|
||||
**/
|
||||
PROCEDURE ADD_SOURCE_FILE_CONFIG (
|
||||
pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE
|
||||
,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE
|
||||
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
|
||||
,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE
|
||||
,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE
|
||||
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
|
||||
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
|
||||
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
|
||||
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name ADD_COLUMN_DATE_FORMAT
|
||||
* @desc Insert a new record to A_COLUMN_DATE_FORMAT table.
|
||||
**/
|
||||
PROCEDURE ADD_COLUMN_DATE_FORMAT (
|
||||
pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE
|
||||
,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE
|
||||
,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_BUCKET_URI
|
||||
* @desc Function used to get string with bucket http url.
|
||||
* Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
|
||||
* @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual;
|
||||
* @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/
|
||||
**/
|
||||
FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2)
|
||||
RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_CONFIG_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_CONFIG record
|
||||
* for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
* pSourceFileConfigKey => 128
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
* @ex_rslt
|
||||
* Details about File Configuration:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 128
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Details about related Container Config:
|
||||
* --------------------------------
|
||||
* A_SOURCE_FILE_CONFIG_KEY = 126
|
||||
* A_SOURCE_KEY = C2D
|
||||
* ...
|
||||
* --------------------------------
|
||||
*
|
||||
* Column Date Format config entries:
|
||||
* --------------------------------
|
||||
* TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM
|
||||
* ...
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_SOURCE_FILE_RECEIVED_INFO
|
||||
* @desc Function returns details about A_SOURCE_FILE_RECEIVED record
|
||||
* for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY).
|
||||
* If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
|
||||
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
|
||||
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
* pSourceFileReceivedKey => 377
|
||||
* ,pIncludeConfigInfo => 1
|
||||
* ,pIncludeContainerInfo => 1
|
||||
* ,pIncludeColumnFormatInfo => 1
|
||||
* ) from dual;
|
||||
*
|
||||
**/
|
||||
FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE
|
||||
,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
|
||||
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GET_DET_USER_LOAD_OPERATIONS
|
||||
* @desc Function returns details from USER_LOAD_OPERATIONS table
|
||||
* for specified pOperationId.
|
||||
* @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual;
|
||||
* @ex_rslt
|
||||
* Details about USER_LOAD_OPERATIONS where ID = 3608
|
||||
* --------------------------------
|
||||
* ID = 3608
|
||||
* TYPE = VALIDATE
|
||||
* SID = 31260
|
||||
* SERIAL# = 52915
|
||||
* START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE
|
||||
* UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE
|
||||
* STATUS = FAILED
|
||||
* OWNER_NAME = CT_MRDS
|
||||
* TABLE_NAME = STANDING_FACILITIES_HEADER
|
||||
* PARTITION_NAME =
|
||||
* SUBPARTITION_NAME =
|
||||
* FILE_URI_LIST =
|
||||
* ROWS_LOADED =
|
||||
* LOGFILE_TABLE = VALIDATE$3608_LOG
|
||||
* BADFILE_TABLE = VALIDATE$3608_BAD
|
||||
* STATUS_TABLE =
|
||||
* TEMPEXT_TABLE =
|
||||
* CREDENTIAL_NAME =
|
||||
* EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE
|
||||
* --------------------------------
|
||||
**/
|
||||
FUNCTION GET_DET_USER_LOAD_OPERATIONS (
|
||||
pOperationId PLS_INTEGER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name ANALYZE_VALIDATION_ERRORS
|
||||
* @desc Wrapper function that analyzes validation errors for a source file using its received key.
|
||||
* Automatically derives template schema, table name, CSV URI and validation log table
|
||||
* from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS.
|
||||
* @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL;
|
||||
* @ex_rslt Detailed validation analysis report with column mismatches and solutions
|
||||
**/
|
||||
FUNCTION ANALYZE_VALIDATION_ERRORS(
|
||||
pSourceFileReceivedKey IN NUMBER
|
||||
) RETURN VARCHAR2;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_MANAGER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 3.2.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_MANAGER
|
||||
* Version: 3.2.0
|
||||
* Build Date: 2025-10-22 16:30:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_MANAGER Version History:
|
||||
* 3.2.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
|
||||
/
|
||||
@@ -0,0 +1,40 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Archival Strategy Validation Trigger
|
||||
-- ===================================================================
|
||||
-- Purpose: Validates archival strategy configuration consistency
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Version: 1.0.0
|
||||
-- ===================================================================
|
||||
|
||||
CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BI_A_SRC_FILE_CFG_ARCH_VAL
|
||||
BEFORE INSERT OR UPDATE ON CT_MRDS.A_SOURCE_FILE_CONFIG
|
||||
FOR EACH ROW
|
||||
DECLARE
|
||||
vErrorMsg VARCHAR2(1000);
|
||||
BEGIN
|
||||
-- Validate MINIMUM_AGE_MONTHS required for specific strategies
|
||||
IF :NEW.ARCHIVAL_STRATEGY IN ('MINIMUM_AGE_MONTHS', 'HYBRID')
|
||||
AND :NEW.MINIMUM_AGE_MONTHS IS NULL THEN
|
||||
vErrorMsg := 'MINIMUM_AGE_MONTHS is required for ' || :NEW.ARCHIVAL_STRATEGY || ' strategy';
|
||||
RAISE_APPLICATION_ERROR(-20999, vErrorMsg);
|
||||
END IF;
|
||||
|
||||
-- Validate MINIMUM_AGE_MONTHS is non-negative (0 = current month only)
|
||||
IF :NEW.MINIMUM_AGE_MONTHS IS NOT NULL
|
||||
AND :NEW.MINIMUM_AGE_MONTHS < 0 THEN
|
||||
RAISE_APPLICATION_ERROR(-20998, 'MINIMUM_AGE_MONTHS must be greater than or equal to 0');
|
||||
END IF;
|
||||
|
||||
-- Warn if MINIMUM_AGE_MONTHS set but strategy doesn't use it
|
||||
IF :NEW.MINIMUM_AGE_MONTHS IS NOT NULL
|
||||
AND :NEW.ARCHIVAL_STRATEGY NOT IN ('MINIMUM_AGE_MONTHS', 'HYBRID') THEN
|
||||
-- This is just a warning scenario - allow but could log
|
||||
NULL;
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
RAISE;
|
||||
END;
|
||||
/
|
||||
148
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql
Normal file
148
MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql
Normal file
@@ -0,0 +1,148 @@
|
||||
-- ===================================================================
|
||||
-- MARS-828: Enhanced Archival Strategies Rollback
|
||||
-- ===================================================================
|
||||
-- Purpose: Rollback all changes from MARS-828 installation
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
|
||||
-- Dynamic spool file generation (using SYS_CONTEXT - no DBA privileges required)
|
||||
-- IMPORTANT: Ensure log/ directory exists before SPOOL (use host mkdir)
|
||||
host mkdir log 2>nul
|
||||
|
||||
var filename VARCHAR2(100)
|
||||
BEGIN
|
||||
:filename := 'log/ROLLBACK_MARS_828_' || SYS_CONTEXT('USERENV', 'CON_NAME') || '_' || TO_CHAR(SYSDATE,'YYYYMMDD_HH24MISS') || '.log';
|
||||
END;
|
||||
/
|
||||
column filename new_value _filename
|
||||
select :filename filename from dual;
|
||||
spool &_filename
|
||||
|
||||
SET ECHO OFF
|
||||
SET TIMING ON
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
SET LINESIZE 200
|
||||
SET PAGESIZE 1000
|
||||
SET PAUSE OFF
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT MARS-828 Rollback Starting
|
||||
PROMPT ============================================================================
|
||||
PROMPT This will restore FILE_ARCHIVER to v2.0.0
|
||||
PROMPT
|
||||
PROMPT Rollback steps:
|
||||
PROMPT 1. Rollback TRASH retention statuses
|
||||
PROMPT 2. Revoke T_FILENAME privileges
|
||||
PROMPT 3. Remove validation trigger
|
||||
PROMPT 4. Remove column comments (OPTIONAL - does not affect functionality)
|
||||
PROMPT 5. Revert threshold column renames (restore original naming)
|
||||
PROMPT 6. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH)
|
||||
PROMPT 7. Restore FILE_ARCHIVER package to v2.0.0
|
||||
PROMPT 8. Restore FILE_MANAGER package to v3.3.1
|
||||
PROMPT 9. Revert all archival strategies to THRESHOLD_BASED
|
||||
PROMPT
|
||||
PROMPT Timestamp:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL;
|
||||
PROMPT ============================================================================
|
||||
|
||||
-- Confirm rollback with user
|
||||
ACCEPT continue CHAR PROMPT 'Type YES to continue with rollback, or Ctrl+C to abort: '
|
||||
WHENEVER SQLERROR EXIT SQL.SQLCODE
|
||||
BEGIN
|
||||
IF '&continue' IS NULL OR TRIM('&continue') IS NULL OR UPPER(TRIM('&continue')) != 'YES' THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'Rollback aborted by user');
|
||||
END IF;
|
||||
END;
|
||||
/
|
||||
WHENEVER SQLERROR CONTINUE
|
||||
|
||||
-- Rollback steps (in reverse order)
|
||||
PROMPT
|
||||
PROMPT Step 1/9: Rolling back TRASH retention statuses
|
||||
PROMPT ================================================
|
||||
@@90_MARS_828_rollback_trash_retention_statuses.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 2/9: Revoking T_FILENAME privileges from MRDS_LOADER
|
||||
PROMPT ==========================================================
|
||||
@@95_MARS_828_rollback_grant_t_filename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 3/9: Dropping validation trigger
|
||||
PROMPT ======================================
|
||||
@@93_MARS_828_rollback_trigger.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 4/9 (OPTIONAL): Removing column comments
|
||||
PROMPT ==============================================
|
||||
PROMPT NOTE: This is optional - comments do not affect functionality
|
||||
PROMPT Skipping column comments removal in standard rollback
|
||||
PROMPT Execute 94b_MARS_828_rollback_column_comments.sql manually if needed
|
||||
PROMPT
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 5/9: Reverting threshold column renames
|
||||
PROMPT =============================================
|
||||
@@94a_MARS_828_rollback_threshold_rename.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 6/9: Dropping all archival configuration columns
|
||||
PROMPT ======================================================
|
||||
@@94_MARS_828_rollback_columns.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 7/9: Restoring FILE_ARCHIVER Package Specification v2.0.0
|
||||
PROMPT ===============================================================
|
||||
@@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 8/11: Restoring FILE_ARCHIVER Package Body v2.0.0
|
||||
PROMPT =======================================================
|
||||
@@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 9/11: Restoring FILE_MANAGER Package Specification v3.3.1
|
||||
PROMPT ===============================================================
|
||||
@@97_MARS_828_rollback_FILE_MANAGER_SPEC.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 10/11: Restoring FILE_MANAGER Package Body v3.3.1
|
||||
PROMPT ======================================================
|
||||
@@98_MARS_828_rollback_FILE_MANAGER_BODY.sql
|
||||
|
||||
PROMPT
|
||||
PROMPT Step 11/11: Verifying tracked packages
|
||||
PROMPT ======================================
|
||||
@@verify_packages_version.sql
|
||||
|
||||
-- Verify rollback
|
||||
PROMPT
|
||||
PROMPT Verification: Package Compilation Status
|
||||
PROMPT =========================================
|
||||
SELECT object_name, object_type, status, last_ddl_time
|
||||
FROM all_objects
|
||||
WHERE owner = 'CT_MRDS'
|
||||
AND object_name IN ('FILE_ARCHIVER', 'FILE_MANAGER')
|
||||
AND object_type IN ('PACKAGE', 'PACKAGE BODY')
|
||||
ORDER BY object_name, object_type;
|
||||
|
||||
PROMPT
|
||||
PROMPT ============================================================================
|
||||
PROMPT MARS-828 Rollback Completed
|
||||
PROMPT ============================================================================
|
||||
PROMPT Completion Time:
|
||||
SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_end FROM DUAL;
|
||||
PROMPT
|
||||
PROMPT Rollback Summary:
|
||||
PROMPT - Packages Rolled Back:
|
||||
PROMPT * CT_MRDS.FILE_ARCHIVER to v2.0.0 (THRESHOLD_BASED archival only)
|
||||
PROMPT * CT_MRDS.FILE_MANAGER to v3.3.1 (pre-MARS-828 threshold column compatibility)
|
||||
PROMPT - Removed Features: CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID strategies
|
||||
PROMPT
|
||||
PROMPT Log file: &_filename
|
||||
PROMPT ============================================================================
|
||||
|
||||
spool off
|
||||
|
||||
quit;
|
||||
@@ -0,0 +1,443 @@
|
||||
create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER
|
||||
AS
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
|
||||
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
|
||||
IS
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vCount PLS_INTEGER;
|
||||
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
|
||||
SELECT count(*) , min(SOURCE_FILE_TYPE)
|
||||
INTO vCount, vSourceFileType
|
||||
FROM CT_MRDS.A_TABLE_STAT s
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
IF vCount=0 and vSourceFileType='INPUT' THEN
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey);
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
SELECT *
|
||||
INTO vTableStat
|
||||
FROM CT_MRDS.A_TABLE_STAT
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
-- EXCEPTION
|
||||
-- WHEN NO_DATA_FOUND THEN
|
||||
--
|
||||
END;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
|
||||
RETURN vTableStat;
|
||||
|
||||
END GET_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vQuery VARCHAR2(4000);
|
||||
vTableName VARCHAR2(200);
|
||||
vUri VARCHAR2(1000);
|
||||
vfiles T_FILENAMES;
|
||||
vFilename VARCHAR2(300);
|
||||
vOperationId NUMBER := -1;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
|
||||
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
|
||||
vProcessControlStatus VARCHAR2(60) := 'OK';
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
|
||||
end if;
|
||||
|
||||
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
end if;
|
||||
|
||||
if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM';
|
||||
else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO');
|
||||
end if;
|
||||
|
||||
if LENGTH(vArchivalTriggeredBy)>0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO');
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
vQuery := '
|
||||
select t_filename(
|
||||
file$name
|
||||
,file$path
|
||||
, to_char(h.workflow_start,''yyyy'')
|
||||
, to_char(h.workflow_start,''mm'')
|
||||
)
|
||||
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
where extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
;
|
||||
|
||||
-- Get all files that will be archived into "vfiles" collection ("regular data files")
|
||||
execute immediate vQuery bulk collect into vfiles;
|
||||
|
||||
-- Start EXPORT "regular data files" to parquet and DROP "csv"
|
||||
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
|
||||
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
|
||||
vQuery:=
|
||||
'select
|
||||
s.*
|
||||
-- ,r.partition_year
|
||||
-- ,r.partition_month
|
||||
from '|| vTableName ||' s
|
||||
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
on s.file$name = r.source_file_name
|
||||
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
|
||||
and r.partition_year='''||ym_loop.year||'''
|
||||
and r.partition_month='''||ym_loop.month||'''
|
||||
and r.PROCESSING_STATUS = ''INGESTED''
|
||||
'
|
||||
;
|
||||
vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
|
||||
|
||||
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => ENV_MANAGER.gvCredentialName,
|
||||
file_uri_list => vUri||'d' ,
|
||||
format => json_object('type' value 'parquet'),
|
||||
query => vQuery,
|
||||
operation_id => vOperationId
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vProcessControlStatus :='EXPORT_FAILURE';
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED);
|
||||
|
||||
END;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
|
||||
|
||||
-- Get USER_LOAD_OPERATIONS info
|
||||
select *
|
||||
into vUserLoadOperations
|
||||
from USER_LOAD_OPERATIONS
|
||||
where id = vOperationId;
|
||||
|
||||
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS);
|
||||
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.');
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
|
||||
END IF;
|
||||
|
||||
SELECT
|
||||
object_name
|
||||
into vFilename
|
||||
from DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vUri)
|
||||
where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7'))
|
||||
between vUserLoadOperations.START_TIME
|
||||
and vUserLoadOperations.UPDATE_TIME
|
||||
;
|
||||
|
||||
-- Try to drop EXPORTED FILES ("regular data files")
|
||||
BEGIN
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
|
||||
|
||||
-- first change of status
|
||||
BEGIN
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'ARCHIVED'
|
||||
,ARCH_PATH = vUri||vFilename
|
||||
WHERE r.a_source_file_config_key= pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
AND r.processing_status = 'INGESTED'
|
||||
;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
|
||||
|
||||
-- move file to trash before dropping
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => f.pathname||'/'||f.filename,
|
||||
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
rollback;
|
||||
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
commit;
|
||||
END LOOP;
|
||||
|
||||
--------------------------------------------------------------------
|
||||
-- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
|
||||
IF vProcessControlStatus = 'OK' THEN
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
|
||||
--Drop file from TRASH
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
END LOOP;
|
||||
|
||||
--ROLLBACK PART
|
||||
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
|
||||
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
|
||||
FOR f in ( SELECT vf.filename, vf.pathname
|
||||
FROM TABLE(vfiles) vf
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
ON r.source_file_name = vf.filename
|
||||
AND r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED'
|
||||
AND vf.year = ym_loop.year
|
||||
AND vf.month = ym_loop.month
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_object_uri => f.pathname||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'INGESTED'
|
||||
,ARCH_PATH = NULL
|
||||
WHERE r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vUri||vFilename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters);
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vUri||vFilename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
|
||||
END;
|
||||
-- END of "Try to drop EXPORTED FILES"
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
END LOOP; --ym_loop end (YEAR_MONTH)
|
||||
|
||||
COMMIT;
|
||||
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO');
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END ARCHIVE_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableName VARCHAR2(200);
|
||||
vQuery VARCHAR2(32000);
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
|
||||
vQuery :=
|
||||
'with tmp as (
|
||||
select
|
||||
s.*
|
||||
,file$name as filename
|
||||
,h.workflow_start
|
||||
, to_char(h.workflow_start,''yyyy'') as year
|
||||
, to_char(h.workflow_start,''mm'') as month
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
)
|
||||
, tmp_gr as (
|
||||
select
|
||||
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
|
||||
from tmp
|
||||
group by filename
|
||||
)
|
||||
select
|
||||
NULL as A_TABLE_STAT_KEY
|
||||
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
|
||||
,'''||vTableName||''' as TABLE_NAME
|
||||
,count(*) as FILE_COUNT
|
||||
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then 1 else 0 end) as OLD_FILE_COUNT
|
||||
,sum (row_count_per_file) as ROW_COUNT
|
||||
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then row_count_per_file else 0 end) as OLD_ROW_COUNT
|
||||
,sum(r.bytes) as BYTES
|
||||
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then r.bytes else 0 end) as OLD_BYTES
|
||||
,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
,systimestamp as CREATED
|
||||
from tmp_gr t
|
||||
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => '''||ENV_MANAGER.gvCredentialName||''',
|
||||
location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/''
|
||||
)
|
||||
) r
|
||||
on t.filename = r.object_name'
|
||||
;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
|
||||
execute immediate vQuery into vStats;
|
||||
|
||||
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
|
||||
insert into A_TABLE_STAT_HIST values vStats;
|
||||
delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
insert into A_TABLE_STAT values vStats;
|
||||
COMMIT;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END GATHER_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN PACKAGE_VERSION;
|
||||
END GET_VERSION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_BUILD_INFO
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersion => PACKAGE_VERSION,
|
||||
pBuildDate => PACKAGE_BUILD_DATE,
|
||||
pAuthor => PACKAGE_AUTHOR
|
||||
);
|
||||
END GET_BUILD_INFO;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION_HISTORY
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersionHistory => VERSION_HISTORY
|
||||
);
|
||||
END GET_VERSION_HISTORY;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,90 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.0.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-10-22 16:45:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
**/
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT
|
||||
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
|
||||
**/
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_ARCHIVER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 2.0.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_ARCHIVER
|
||||
* Version: 2.0.0
|
||||
* Build Date: 2025-10-22 16:45:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_ARCHIVER Version History:
|
||||
* 2.0.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,495 @@
|
||||
create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER
|
||||
AS
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE
|
||||
----------------------------------------------------------------------------------------------------
|
||||
/**
|
||||
* @name GET_ARCHIVAL_WHERE_CLAUSE
|
||||
* @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration.
|
||||
* Supports four strategies: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID.
|
||||
* @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY
|
||||
* @return VARCHAR2 - WHERE clause for filtering archival candidates
|
||||
**/
|
||||
FUNCTION GET_ARCHIVAL_WHERE_CLAUSE(
|
||||
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
|
||||
) RETURN VARCHAR2
|
||||
IS
|
||||
vWhereClause VARCHAR2(4000);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
CASE pSourceFileConfig.ARCHIVAL_STRATEGY
|
||||
-- Legacy threshold-based strategy (backward compatible)
|
||||
WHEN 'THRESHOLD_BASED' THEN
|
||||
vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD;
|
||||
|
||||
-- Archive all data except current month
|
||||
WHEN 'CURRENT_MONTH_ONLY' THEN
|
||||
vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'')';
|
||||
|
||||
-- Archive only data older than X months
|
||||
WHEN 'MINIMUM_AGE_MONTHS' THEN
|
||||
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy');
|
||||
END IF;
|
||||
vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
|
||||
|
||||
-- Hybrid: Current month exclusion AND minimum age requirement
|
||||
WHEN 'HYBRID' THEN
|
||||
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy');
|
||||
END IF;
|
||||
vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' ||
|
||||
'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY);
|
||||
END CASE;
|
||||
|
||||
RETURN vWhereClause;
|
||||
END GET_ARCHIVAL_WHERE_CLAUSE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
|
||||
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
|
||||
IS
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vCount PLS_INTEGER;
|
||||
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
|
||||
SELECT count(*) , min(SOURCE_FILE_TYPE)
|
||||
INTO vCount, vSourceFileType
|
||||
FROM CT_MRDS.A_TABLE_STAT s
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
IF vCount=0 and vSourceFileType='INPUT' THEN
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey);
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
SELECT *
|
||||
INTO vTableStat
|
||||
FROM CT_MRDS.A_TABLE_STAT
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
-- EXCEPTION
|
||||
-- WHEN NO_DATA_FOUND THEN
|
||||
--
|
||||
END;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
|
||||
RETURN vTableStat;
|
||||
|
||||
END GET_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vQuery VARCHAR2(4000);
|
||||
vTableName VARCHAR2(200);
|
||||
vUri VARCHAR2(1000);
|
||||
vfiles T_FILENAMES;
|
||||
vFilename VARCHAR2(300);
|
||||
vOperationId NUMBER := -1;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
|
||||
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
|
||||
vProcessControlStatus VARCHAR2(60) := 'OK';
|
||||
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
|
||||
end if;
|
||||
|
||||
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
end if;
|
||||
|
||||
if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM';
|
||||
else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO');
|
||||
end if;
|
||||
|
||||
if LENGTH(vArchivalTriggeredBy)>0 THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO');
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
|
||||
-- Use strategy-based WHERE clause (MARS-828)
|
||||
vQuery := '
|
||||
select t_filename(
|
||||
file$name
|
||||
,file$path
|
||||
, to_char(h.workflow_start,''yyyy'')
|
||||
, to_char(h.workflow_start,''mm'')
|
||||
)
|
||||
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig)
|
||||
;
|
||||
|
||||
-- Get all files that will be archived into "vfiles" collection ("regular data files")
|
||||
execute immediate vQuery bulk collect into vfiles;
|
||||
|
||||
-- Start EXPORT "regular data files" to parquet and DROP "csv"
|
||||
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
|
||||
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
|
||||
vQuery:=
|
||||
'select
|
||||
s.*
|
||||
-- ,r.partition_year
|
||||
-- ,r.partition_month
|
||||
from '|| vTableName ||' s
|
||||
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
on s.file$name = r.source_file_name
|
||||
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
|
||||
and r.partition_year='''||ym_loop.year||'''
|
||||
and r.partition_month='''||ym_loop.month||'''
|
||||
and r.PROCESSING_STATUS = ''INGESTED''
|
||||
'
|
||||
;
|
||||
vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
|
||||
|
||||
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => ENV_MANAGER.gvCredentialName,
|
||||
file_uri_list => vUri||'d' ,
|
||||
format => json_object('type' value 'parquet'),
|
||||
query => vQuery,
|
||||
operation_id => vOperationId
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vProcessControlStatus :='EXPORT_FAILURE';
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED);
|
||||
|
||||
END;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
|
||||
|
||||
-- Get USER_LOAD_OPERATIONS info
|
||||
select *
|
||||
into vUserLoadOperations
|
||||
from USER_LOAD_OPERATIONS
|
||||
where id = vOperationId;
|
||||
|
||||
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS);
|
||||
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.');
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
|
||||
END IF;
|
||||
|
||||
SELECT
|
||||
object_name
|
||||
into vFilename
|
||||
from DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => 'OCI$RESOURCE_PRINCIPAL',
|
||||
location_uri => vUri)
|
||||
where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7'))
|
||||
between vUserLoadOperations.START_TIME
|
||||
and vUserLoadOperations.UPDATE_TIME
|
||||
;
|
||||
|
||||
-- Try to drop EXPORTED FILES ("regular data files")
|
||||
BEGIN
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
|
||||
|
||||
-- first change of status
|
||||
BEGIN
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'ARCHIVED'
|
||||
,ARCH_PATH = vUri||vFilename
|
||||
WHERE r.a_source_file_config_key= pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
AND r.processing_status = 'INGESTED'
|
||||
;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
|
||||
|
||||
-- move file to trash before dropping
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => f.pathname||'/'||f.filename,
|
||||
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
rollback;
|
||||
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
commit;
|
||||
END LOOP;
|
||||
|
||||
--------------------------------------------------------------------
|
||||
-- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
|
||||
IF vProcessControlStatus = 'OK' THEN
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
|
||||
--Drop file from TRASH
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
END LOOP;
|
||||
|
||||
--ROLLBACK PART
|
||||
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
|
||||
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
|
||||
FOR f in ( SELECT vf.filename, vf.pathname
|
||||
FROM TABLE(vfiles) vf
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
ON r.source_file_name = vf.filename
|
||||
AND r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED'
|
||||
AND vf.year = ym_loop.year
|
||||
AND vf.month = ym_loop.month
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_object_uri => f.pathname||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'INGESTED'
|
||||
,ARCH_PATH = NULL
|
||||
WHERE r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vUri||vFilename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters);
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vUri||vFilename);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
|
||||
END;
|
||||
-- END of "Try to drop EXPORTED FILES"
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO');
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
END LOOP; --ym_loop end (YEAR_MONTH)
|
||||
|
||||
COMMIT;
|
||||
|
||||
ELSE
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO');
|
||||
END IF;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END ARCHIVE_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableName VARCHAR2(200);
|
||||
vQuery VARCHAR2(32000);
|
||||
BEGIN
|
||||
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
|
||||
|
||||
-- Use strategy-based WHERE clause for statistics (MARS-828)
|
||||
vQuery :=
|
||||
'with tmp as (
|
||||
select
|
||||
s.*
|
||||
,file$name as filename
|
||||
,h.workflow_start
|
||||
, to_char(h.workflow_start,''yyyy'') as year
|
||||
, to_char(h.workflow_start,''mm'') as month
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
)
|
||||
, tmp_gr as (
|
||||
select
|
||||
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
|
||||
from tmp
|
||||
group by filename
|
||||
)
|
||||
select
|
||||
NULL as A_TABLE_STAT_KEY
|
||||
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
|
||||
,'''||vTableName||''' as TABLE_NAME
|
||||
,count(*) as FILE_COUNT
|
||||
,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then 1 else 0 end) as OLD_FILE_COUNT
|
||||
,sum (row_count_per_file) as ROW_COUNT
|
||||
,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT
|
||||
,sum(r.bytes) as BYTES
|
||||
,sum(case when ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || ' then r.bytes else 0 end) as OLD_BYTES
|
||||
,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
,systimestamp as CREATED
|
||||
from tmp_gr t
|
||||
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => '''||ENV_MANAGER.gvCredentialName||''',
|
||||
location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/''
|
||||
)
|
||||
) r
|
||||
on t.filename = r.object_name'
|
||||
;
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
|
||||
execute immediate vQuery into vStats;
|
||||
|
||||
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
|
||||
insert into A_TABLE_STAT_HIST values vStats;
|
||||
delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
insert into A_TABLE_STAT values vStats;
|
||||
COMMIT;
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END GATHER_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN PACKAGE_VERSION;
|
||||
END GET_VERSION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_BUILD_INFO
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersion => PACKAGE_VERSION,
|
||||
pBuildDate => PACKAGE_BUILD_DATE,
|
||||
pAuthor => PACKAGE_AUTHOR
|
||||
);
|
||||
END GET_BUILD_INFO;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION_HISTORY
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersionHistory => VERSION_HISTORY
|
||||
);
|
||||
END GET_VERSION_HISTORY;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,91 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.0.0';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-01-27 12:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
**/
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT
|
||||
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
|
||||
**/
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_ARCHIVER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 2.0.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_ARCHIVER
|
||||
* Version: 2.0.0
|
||||
* Build Date: 2025-10-22 16:45:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_ARCHIVER Version History:
|
||||
* 2.0.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,998 @@
|
||||
create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER
|
||||
AS
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PRIVATE FUNCTION: GET_ARCHIVAL_WHERE_CLAUSE
|
||||
----------------------------------------------------------------------------------------------------
|
||||
/**
|
||||
* @name GET_ARCHIVAL_WHERE_CLAUSE
|
||||
* @desc Private function that generates WHERE clause based on ARCHIVAL_STRATEGY configuration.
|
||||
* Supports three strategies: THRESHOLD_BASED, MINIMUM_AGE_MONTHS, HYBRID.
|
||||
* @param pSourceFileConfig - Source file configuration record with ARCHIVAL_STRATEGY
|
||||
* @return VARCHAR2 - WHERE clause for filtering archival candidates
|
||||
**/
|
||||
FUNCTION GET_ARCHIVAL_WHERE_CLAUSE(
|
||||
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
|
||||
) RETURN VARCHAR2
|
||||
IS
|
||||
vWhereClause VARCHAR2(4000);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
CASE pSourceFileConfig.ARCHIVAL_STRATEGY
|
||||
-- Legacy threshold-based strategy (backward compatible)
|
||||
WHEN 'THRESHOLD_BASED' THEN
|
||||
vWhereClause := 'extract(day from (systimestamp - workflow_start)) > ' || pSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD;
|
||||
|
||||
-- Archive data older than X months (0 = current month only)
|
||||
WHEN 'MINIMUM_AGE_MONTHS' THEN
|
||||
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for MINIMUM_AGE_MONTHS strategy');
|
||||
END IF;
|
||||
vWhereClause := 'workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
|
||||
|
||||
-- Hybrid: Current month exclusion AND minimum age requirement
|
||||
WHEN 'HYBRID' THEN
|
||||
IF pSourceFileConfig.MINIMUM_AGE_MONTHS IS NULL THEN
|
||||
RAISE_APPLICATION_ERROR(-20001, 'MINIMUM_AGE_MONTHS must be configured for HYBRID strategy');
|
||||
END IF;
|
||||
vWhereClause := 'TRUNC(workflow_start, ''MM'') < TRUNC(SYSDATE, ''MM'') ' ||
|
||||
'AND workflow_start < ADD_MONTHS(TRUNC(SYSDATE, ''MM''), -' || pSourceFileConfig.MINIMUM_AGE_MONTHS || ')';
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20002, 'Invalid ARCHIVAL_STRATEGY: ' || pSourceFileConfig.ARCHIVAL_STRATEGY);
|
||||
END CASE;
|
||||
|
||||
RETURN vWhereClause;
|
||||
END GET_ARCHIVAL_WHERE_CLAUSE;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
|
||||
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
|
||||
IS
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vCount PLS_INTEGER;
|
||||
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
|
||||
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
|
||||
SELECT count(*) , min(SOURCE_FILE_TYPE)
|
||||
INTO vCount, vSourceFileType
|
||||
FROM CT_MRDS.A_TABLE_STAT s
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
|
||||
IF vCount=0 and vSourceFileType='INPUT' THEN
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey);
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
SELECT *
|
||||
INTO vTableStat
|
||||
FROM CT_MRDS.A_TABLE_STAT
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
-- EXCEPTION
|
||||
-- WHEN NO_DATA_FOUND THEN
|
||||
--
|
||||
END;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
|
||||
RETURN vTableStat;
|
||||
|
||||
END GET_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
pKeepInTrash IN BOOLEAN DEFAULT TRUE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vQuery VARCHAR2(4000);
|
||||
vTableName VARCHAR2(200);
|
||||
vUri VARCHAR2(1000);
|
||||
vfiles T_FILENAMES;
|
||||
vFilename VARCHAR2(300);
|
||||
vOperationId NUMBER := -1;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
|
||||
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
|
||||
vProcessControlStatus VARCHAR2(60) := 'OK';
|
||||
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
|
||||
end if;
|
||||
|
||||
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
end if;
|
||||
|
||||
-- Strategy-based trigger logic (MARS-828)
|
||||
IF vSourceFileConfig.ARCHIVAL_STRATEGY = 'MINIMUM_AGE_MONTHS' THEN
|
||||
-- MINIMUM_AGE_MONTHS: Archive based on age only, ignore thresholds
|
||||
vArchivalTriggeredBy := 'AGE_BASED';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival strategy: MINIMUM_AGE_MONTHS (threshold-independent)','INFO');
|
||||
ELSE
|
||||
-- THRESHOLD_BASED and HYBRID: Check thresholds
|
||||
if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT';
|
||||
elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM';
|
||||
else CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO');
|
||||
end if;
|
||||
END IF;
|
||||
|
||||
if LENGTH(vArchivalTriggeredBy)>0 THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO');
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
|
||||
-- Use strategy-based WHERE clause (MARS-828)
|
||||
-- Using GROUP BY instead of DISTINCT to avoid ORA-22950 (object type ordering issue)
|
||||
vQuery := '
|
||||
select CT_MRDS.t_filename(
|
||||
file$name
|
||||
,file$path
|
||||
, to_char(h.workflow_start,''yyyy'')
|
||||
, to_char(h.workflow_start,''mm'')
|
||||
)
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
where ' || GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig) || '
|
||||
group by file$name, file$path, to_char(h.workflow_start,''yyyy''), to_char(h.workflow_start,''mm'')'
|
||||
;
|
||||
|
||||
-- Get all files that will be archived into "vfiles" collection ("regular data files")
|
||||
execute immediate vQuery bulk collect into vfiles;
|
||||
|
||||
-- Start EXPORT "regular data files" to parquet and DROP "csv"
|
||||
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
|
||||
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
|
||||
vQuery:=
|
||||
'select
|
||||
s.*
|
||||
from '|| vTableName ||' s
|
||||
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
on s.file$name = r.source_file_name
|
||||
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
|
||||
and r.PROCESSING_STATUS = ''INGESTED''
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
and to_char(h.workflow_start,''yyyy'') = '''||ym_loop.year||'''
|
||||
and to_char(h.workflow_start,''mm'') = '''||ym_loop.month||'''
|
||||
'
|
||||
;
|
||||
vUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
|
||||
|
||||
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.EXPORT_DATA(
|
||||
credential_name => ENV_MANAGER.gvCredentialName,
|
||||
file_uri_list => vUri||'d' ,
|
||||
format => json_object('type' value 'parquet'),
|
||||
query => vQuery,
|
||||
operation_id => vOperationId
|
||||
);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
vProcessControlStatus :='EXPORT_FAILURE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED);
|
||||
|
||||
END;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
|
||||
|
||||
-- Get USER_LOAD_OPERATIONS info
|
||||
select *
|
||||
into vUserLoadOperations
|
||||
from USER_LOAD_OPERATIONS
|
||||
where id = vOperationId;
|
||||
|
||||
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS);
|
||||
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
|
||||
CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.');
|
||||
ELSE
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month,'INFO', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
|
||||
END IF;
|
||||
|
||||
-- Note: DBMS_CLOUD.EXPORT_DATA may create multiple parquet files (parallel execution)
|
||||
-- Instead of tracking individual files, we store the archive directory prefix
|
||||
-- ARCH_PATH will contain the directory URI where all parquet files are located
|
||||
vFilename := vUri; -- Store directory prefix instead of individual filename
|
||||
|
||||
-- Try to drop EXPORTED FILES ("regular data files")
|
||||
BEGIN
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
|
||||
|
||||
-- first change of status to ARCHIVED_AND_TRASHED (file will be moved to TRASH folder)
|
||||
BEGIN
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' -- Status reflects file is archived and kept in TRASH
|
||||
,ARCH_PATH = vFilename -- Now contains directory prefix, not individual file
|
||||
,PARTITION_YEAR = ym_loop.year -- Record which partition year the data was archived to
|
||||
,PARTITION_MONTH = ym_loop.month -- Record which partition month the data was archived to
|
||||
WHERE r.a_source_file_config_key= pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
AND r.processing_status = 'INGESTED'
|
||||
;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_AND_TRASHED_FAILURE';
|
||||
|
||||
-- move file to TRASH subfolder (DATA bucket: ODS/ → TRASH/) before dropping
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => f.pathname||'/'||f.filename,
|
||||
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH folder.','ERROR', f.pathname||'/'||f.filename);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
rollback;
|
||||
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
END;
|
||||
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
|
||||
commit;
|
||||
END LOOP;
|
||||
|
||||
--------------------------------------------------------------------
|
||||
-- IF All goes fine till this point, we drop files from TRASH folder (if not then ROLLBACK PART)
|
||||
-- TRASH is a subfolder in DATA bucket (e.g., TRASH/LM/TABLE_NAME instead of ODS/LM/TABLE_NAME)
|
||||
IF vProcessControlStatus = 'OK' THEN
|
||||
IF NOT pKeepInTrash THEN
|
||||
-- Delete files from TRASH folder (cleanup) and update status to ARCHIVED_AND_PURGED
|
||||
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
|
||||
DBMS_CLOUD.DELETE_OBJECT(credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
|
||||
|
||||
-- Update status to ARCHIVED_AND_PURGED
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
END LOOP;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('All archived files removed from TRASH folder and marked as ARCHIVED_AND_PURGED.','INFO');
|
||||
ELSE
|
||||
-- Keep files in TRASH folder (status remains ARCHIVED_AND_TRASHED)
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archived files kept in TRASH folder for retention (status: ARCHIVED_AND_TRASHED).','INFO');
|
||||
END IF;
|
||||
|
||||
--ROLLBACK PART
|
||||
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH subfolder in DATA bucket)
|
||||
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
|
||||
FOR f in ( SELECT vf.filename, vf.pathname
|
||||
FROM TABLE(vfiles) vf
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
ON r.source_file_name = vf.filename
|
||||
AND r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.PROCESSING_STATUS IN ('ARCHIVED_AND_TRASHED', 'ARCHIVED_AND_PURGED')
|
||||
AND vf.year = ym_loop.year
|
||||
AND vf.month = ym_loop.month
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
|
||||
target_object_uri => f.pathname||'/'||f.filename,
|
||||
target_credential_name => ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH folder.','DEBUG', f.pathname||'/'||f.filename);
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
SET PROCESSING_STATUS = 'INGESTED'
|
||||
,ARCH_PATH = NULL
|
||||
WHERE r.a_source_file_config_key = pSourceFileConfigKey
|
||||
AND r.source_file_name = f.filename
|
||||
;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH folder.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
-- ROLLBACK: Delete all parquet files from archive directory
|
||||
FOR arch_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vFilename -- vFilename now contains directory prefix
|
||||
)
|
||||
) LOOP
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vFilename || arch_file.object_name
|
||||
);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name);
|
||||
END LOOP;
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters);
|
||||
-- ROLLBACK: Delete all parquet files from archive directory
|
||||
FOR arch_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vFilename -- vFilename now contains directory prefix
|
||||
)
|
||||
) LOOP
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vFilename || arch_file.object_name
|
||||
);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vFilename || arch_file.object_name);
|
||||
END LOOP;
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
|
||||
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
|
||||
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, CT_MRDS.ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
|
||||
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, CT_MRDS.ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
|
||||
END;
|
||||
-- END of "Try to drop EXPORTED FILES"
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
|
||||
END LOOP; --ym_loop end (YEAR_MONTH)
|
||||
|
||||
COMMIT;
|
||||
|
||||
ELSE
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO');
|
||||
END IF;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN CT_MRDS.ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
|
||||
END ARCHIVE_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vTableName VARCHAR2(200);
|
||||
vQuery VARCHAR2(32000);
|
||||
vWhereClause VARCHAR2(4000);
|
||||
vOdsBucketUri VARCHAR2(1000);
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
|
||||
|
||||
-- Get WHERE clause based on archival strategy (MARS-828)
|
||||
vWhereClause := GET_ARCHIVAL_WHERE_CLAUSE(vSourceFileConfig);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vWhereClause','DEBUG',vWhereClause);
|
||||
|
||||
-- Get ODS bucket URI before building query
|
||||
vOdsBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ODS') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
|
||||
|
||||
-- Use strategy-based WHERE clause for statistics (MARS-828)
|
||||
vQuery :=
|
||||
'with tmp as (
|
||||
select
|
||||
s.*
|
||||
,file$name as filename
|
||||
,h.workflow_start
|
||||
, to_char(h.workflow_start,''yyyy'') as year
|
||||
, to_char(h.workflow_start,''mm'') as month
|
||||
from '||vTableName||' s
|
||||
join CT_MRDS.a_workflow_history h
|
||||
on s.a_workflow_history_key = h.a_workflow_history_key
|
||||
)
|
||||
, tmp_gr as (
|
||||
select
|
||||
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
|
||||
from tmp
|
||||
group by filename
|
||||
)
|
||||
select
|
||||
NULL as A_TABLE_STAT_KEY
|
||||
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
|
||||
,'''||vTableName||''' as TABLE_NAME
|
||||
,count(*) as FILE_COUNT
|
||||
,sum(case when ' || vWhereClause || ' then 1 else 0 end) as OLD_FILE_COUNT
|
||||
,sum (row_count_per_file) as ROW_COUNT
|
||||
,sum(case when ' || vWhereClause || ' then row_count_per_file else 0 end) as OLD_ROW_COUNT
|
||||
,sum(r.bytes) as BYTES
|
||||
,sum(case when ' || vWhereClause || ' then r.bytes else 0 end) as OLD_BYTES
|
||||
,'||COALESCE(TO_CHAR(vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD), 'NULL')||' as DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
,systimestamp as CREATED
|
||||
from tmp_gr t
|
||||
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => '''||CT_MRDS.ENV_MANAGER.gvCredentialName||''',
|
||||
location_uri => '''||vOdsBucketUri||'''
|
||||
)
|
||||
) r
|
||||
on t.filename = r.object_name'
|
||||
;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
|
||||
execute immediate vQuery into vStats;
|
||||
|
||||
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
|
||||
insert into CT_MRDS.A_TABLE_STAT_HIST values vStats;
|
||||
delete from CT_MRDS.A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
|
||||
insert into CT_MRDS.A_TABLE_STAT values vStats;
|
||||
COMMIT;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END GATHER_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- TRASH FOLDER MANAGEMENT PROCEDURES
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vTrashPath VARCHAR2(1000);
|
||||
vOdsPath VARCHAR2(1000);
|
||||
vFilesRestored PLS_INTEGER := 0;
|
||||
vFilesUpdated PLS_INTEGER := 0;
|
||||
vRestoreLevel VARCHAR2(50);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||
vRestoreLevel := 'LEVEL_3_SINGLE_FILE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
|
||||
-- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||
BEGIN
|
||||
SELECT *
|
||||
INTO vSourceFileReceived
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY);
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
END;
|
||||
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath);
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_PATH = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||
vRestoreLevel := 'LEVEL_2_CONFIG_FILES';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||
|
||||
-- LEVEL 2: Restore all files for specific pSourceFileConfigKey
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
FOR file_rec IN (
|
||||
SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := vFilesRestored + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_PATH = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pRestoreAll THEN
|
||||
vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status');
|
||||
|
||||
-- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||
FOR file_rec IN (
|
||||
SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME,
|
||||
c.A_SOURCE_KEY, c.TABLE_ID
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := vFilesRestored + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_PATH = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE');
|
||||
END IF;
|
||||
|
||||
COMMIT;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END RESTORE_FILE_FROM_TRASH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vTrashLocationUri VARCHAR2(1000);
|
||||
vFilesDeleted PLS_INTEGER := 0;
|
||||
vFilesUpdated PLS_INTEGER := 0;
|
||||
vPurgeLevel VARCHAR2(50);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||
vPurgeLevel := 'LEVEL_3_SINGLE_FILE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
|
||||
-- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||
BEGIN
|
||||
SELECT *
|
||||
INTO vSourceFileReceived
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY);
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
END;
|
||||
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri
|
||||
);
|
||||
vFilesDeleted := 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||
vPurgeLevel := 'LEVEL_2_CONFIG_FILES';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||
|
||||
-- LEVEL 2: Delete all files for specific pSourceFileConfigKey
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri);
|
||||
|
||||
BEGIN
|
||||
FOR trash_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vTrashLocationUri
|
||||
)
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri || trash_file.object_name
|
||||
);
|
||||
vFilesDeleted := vFilesDeleted + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pPurgeAll THEN
|
||||
vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status');
|
||||
|
||||
-- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||
FOR config_rec IN (
|
||||
SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri);
|
||||
|
||||
BEGIN
|
||||
FOR trash_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vTrashLocationUri
|
||||
)
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri || trash_file.object_name
|
||||
);
|
||||
vFilesDeleted := vFilesDeleted + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE');
|
||||
END IF;
|
||||
|
||||
COMMIT;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END PURGE_TRASH_FOLDER;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
PRAGMA AUTONOMOUS_TRANSACTION;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
PURGE_TRASH_FOLDER(
|
||||
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||
pPurgeAll => pPurgeAll
|
||||
);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END PURGE_TRASH_FOLDER;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
PRAGMA AUTONOMOUS_TRANSACTION;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
RESTORE_FILE_FROM_TRASH(
|
||||
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||
pRestoreAll => pRestoreAll
|
||||
);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END RESTORE_FILE_FROM_TRASH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN PACKAGE_VERSION;
|
||||
END GET_VERSION;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_BUILD_INFO
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN CT_MRDS.ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersion => PACKAGE_VERSION,
|
||||
pBuildDate => PACKAGE_BUILD_DATE,
|
||||
pAuthor => PACKAGE_AUTHOR
|
||||
);
|
||||
END GET_BUILD_INFO;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GET_VERSION_HISTORY
|
||||
RETURN VARCHAR2
|
||||
IS
|
||||
BEGIN
|
||||
RETURN CT_MRDS.ENV_MANAGER.FORMAT_VERSION_HISTORY(
|
||||
pPackageName => 'FILE_ARCHIVER',
|
||||
pVersionHistory => VERSION_HISTORY
|
||||
);
|
||||
END GET_VERSION_HISTORY;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
pKeepInTrash IN BOOLEAN DEFAULT TRUE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pKeepInTrash => '||CASE WHEN pKeepInTrash THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
ARCHIVE_TABLE_DATA(pSourceFileConfigKey => pSourceFileConfigKey, pKeepInTrash => pKeepInTrash);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END ARCHIVE_TABLE_DATA;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END GATHER_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -0,0 +1,195 @@
|
||||
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
|
||||
AUTHID CURRENT_USER
|
||||
AS
|
||||
/**
|
||||
* General comment for package: Please put comments for functions and procedures as shown in below example.
|
||||
* It is a standard.
|
||||
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
|
||||
* which returns documentation text for confluence page (to Copy-Paste it).
|
||||
**/
|
||||
|
||||
-- Example comment:
|
||||
/**
|
||||
* @name EX_PROCEDURE_NAME
|
||||
* @desc Procedure description
|
||||
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
|
||||
* @ex_rslt Example Result
|
||||
**/
|
||||
|
||||
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
|
||||
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.2.1';
|
||||
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2026-02-10 09:00:00';
|
||||
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
|
||||
|
||||
-- Version History (Latest changes first)
|
||||
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
|
||||
'3.2.1 (2026-02-10): Fixed status update - ARCHIVED → ARCHIVED_AND_TRASHED when moving files to TRASH folder (critical bug fix)' || CHR(13)||CHR(10) ||
|
||||
'3.2.0 (2026-02-06): Added pKeepInTrash parameter (DEFAULT TRUE) to ARCHIVE_TABLE_DATA for TRASH folder retention control - files kept in TRASH subfolder (DATA bucket) by default for safety and compliance' || CHR(13)||CHR(10) ||
|
||||
'3.1.2 (2026-02-06): Fixed missing PARTITION_YEAR/PARTITION_MONTH assignments in UPDATE statement and export query circular dependency (now filters by workflow_start instead of partition fields)' || CHR(13)||CHR(10) ||
|
||||
'3.1.1 (2026-02-06): Fixed ORA-01422 error when DBMS_CLOUD.EXPORT_DATA creates multiple parquet files (parallel execution). Now stores archive directory prefix instead of individual filenames' || CHR(13)||CHR(10) ||
|
||||
'3.1.0 (2026-01-29): Added function overloads for ARCHIVE_TABLE_DATA and GATHER_TABLE_STAT returning SQLCODE for Python library integration' || CHR(13)||CHR(10) ||
|
||||
'3.0.0 (2026-01-27): MARS-828 - Added flexible archival strategies (MINIMUM_AGE_MONTHS with 0=current month, HYBRID) via ARCHIVAL_STRATEGY configuration' || CHR(13)||CHR(10) ||
|
||||
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
|
||||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
|
||||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
|
||||
|
||||
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_TABLE_DATA
|
||||
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
|
||||
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
|
||||
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
|
||||
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
|
||||
**/
|
||||
PROCEDURE ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
pKeepInTrash IN BOOLEAN DEFAULT TRUE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name ARCHIVE_TABLE_DATA
|
||||
* @desc Function overload for ARCHIVE_TABLE_DATA procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main ARCHIVE_TABLE_DATA procedure and captures execution result.
|
||||
* @param pKeepInTrash - When TRUE (default), files are kept in TRASH folder (DATA bucket subfolder) for safety. When FALSE, files are deleted from TRASH after successful archive.
|
||||
* @example SELECT FILE_ARCHIVER.ARCHIVE_TABLE_DATA(pSourceFileConfigKey => 123) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION ARCHIVE_TABLE_DATA (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE,
|
||||
pKeepInTrash IN BOOLEAN DEFAULT TRUE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT
|
||||
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
|
||||
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
|
||||
**/
|
||||
PROCEDURE GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name GATHER_TABLE_STAT
|
||||
* @desc Function overload for GATHER_TABLE_STAT procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main GATHER_TABLE_STAT procedure and captures execution result.
|
||||
* @example SELECT FILE_ARCHIVER.GATHER_TABLE_STAT(pSourceFileConfigKey => 123) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION GATHER_TABLE_STAT (
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Restores files from TRASH folder back to ODS at three different granularity levels.
|
||||
* Moves files from TRASH subfolder back to ODS subfolder in DATA bucket.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341);
|
||||
* @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE);
|
||||
**/
|
||||
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Function overload for RESTORE_FILE_FROM_TRASH procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Deletes files from TRASH folder at three different granularity levels.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341);
|
||||
* @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE);
|
||||
**/
|
||||
PROCEDURE PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Function overload for PURGE_TRASH_FOLDER procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main PURGE_TRASH_FOLDER procedure and captures execution result.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @name GET_VERSION
|
||||
* @desc Returns the current version number of the FILE_ARCHIVER package.
|
||||
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
|
||||
* @ex_rslt 2.0.0
|
||||
**/
|
||||
FUNCTION GET_VERSION RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_BUILD_INFO
|
||||
* @desc Returns comprehensive build information including version, build date, and author.
|
||||
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
|
||||
* @ex_rslt Package: FILE_ARCHIVER
|
||||
* Version: 2.0.0
|
||||
* Build Date: 2025-10-22 16:45:00
|
||||
* Author: Grzegorz Michalski
|
||||
**/
|
||||
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
|
||||
|
||||
/**
|
||||
* @name GET_VERSION_HISTORY
|
||||
* @desc Returns complete version history with all releases and changes.
|
||||
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
|
||||
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
|
||||
* @ex_rslt FILE_ARCHIVER Version History:
|
||||
* 2.0.0 (2025-10-22): Added package versioning system...
|
||||
**/
|
||||
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
|
||||
|
||||
END;
|
||||
|
||||
/
|
||||
@@ -1,253 +0,0 @@
|
||||
-- MARS-828: Test archival strategies
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2026-01-27
|
||||
-- Description: Comprehensive test scenarios for all archival strategies
|
||||
|
||||
SET SERVEROUTPUT ON SIZE UNLIMITED
|
||||
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Testing Archival Strategies
|
||||
PROMPT ========================================
|
||||
|
||||
-- Test 1: THRESHOLD_BASED (backward compatibility)
|
||||
PROMPT
|
||||
PROMPT Test 1: THRESHOLD_BASED strategy
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1000;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY,
|
||||
DAYS_FOR_ARCHIVE_THRESHOLD
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST1',
|
||||
'INPUT',
|
||||
'TEST_THRESHOLD',
|
||||
'Test threshold-based',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'THRESHOLD_BASED',
|
||||
30
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: THRESHOLD_BASED strategy configured');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: ' || SQLERRM);
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Test 2: CURRENT_MONTH_ONLY
|
||||
PROMPT
|
||||
PROMPT Test 2: CURRENT_MONTH_ONLY strategy
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1001;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST2',
|
||||
'INPUT',
|
||||
'TEST_CURRENT_MONTH',
|
||||
'Test current month only',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'CURRENT_MONTH_ONLY'
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: CURRENT_MONTH_ONLY strategy configured');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: ' || SQLERRM);
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Test 3: MINIMUM_AGE_MONTHS (should succeed)
|
||||
PROMPT
|
||||
PROMPT Test 3: MINIMUM_AGE_MONTHS strategy with valid config
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1002;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST3',
|
||||
'INPUT',
|
||||
'TEST_MIN_AGE',
|
||||
'Test minimum age months',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'MINIMUM_AGE_MONTHS',
|
||||
6
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: MINIMUM_AGE_MONTHS strategy configured with 6 months');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: ' || SQLERRM);
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Test 4: MINIMUM_AGE_MONTHS without value (should fail)
|
||||
PROMPT
|
||||
PROMPT Test 4: MINIMUM_AGE_MONTHS strategy without value (should fail)
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1003;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST4',
|
||||
'INPUT',
|
||||
'TEST_MIN_AGE_FAIL',
|
||||
'Test minimum age months failure',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'MINIMUM_AGE_MONTHS',
|
||||
NULL -- Should trigger error
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: Trigger did not fire - this should have failed!');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -20999 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Trigger correctly prevented invalid config');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: Unexpected error: ' || SQLERRM);
|
||||
END IF;
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Test 5: HYBRID strategy
|
||||
PROMPT
|
||||
PROMPT Test 5: HYBRID strategy
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1004;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY,
|
||||
MINIMUM_AGE_MONTHS
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST5',
|
||||
'INPUT',
|
||||
'TEST_HYBRID',
|
||||
'Test hybrid strategy',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'HYBRID',
|
||||
3
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: HYBRID strategy configured');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: ' || SQLERRM);
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
-- Test 6: Invalid strategy (should fail)
|
||||
PROMPT
|
||||
PROMPT Test 6: Invalid strategy (should fail)
|
||||
DECLARE
|
||||
vTestKey NUMBER := -1005;
|
||||
BEGIN
|
||||
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG (
|
||||
A_SOURCE_FILE_CONFIG_KEY,
|
||||
A_SOURCE_KEY,
|
||||
SOURCE_FILE_TYPE,
|
||||
SOURCE_FILE_ID,
|
||||
SOURCE_FILE_DESC,
|
||||
SOURCE_FILE_NAME_PATTERN,
|
||||
TABLE_ID,
|
||||
TEMPLATE_TABLE_NAME,
|
||||
ARCHIVAL_STRATEGY
|
||||
) VALUES (
|
||||
vTestKey,
|
||||
'TEST6',
|
||||
'INPUT',
|
||||
'TEST_INVALID',
|
||||
'Test invalid strategy',
|
||||
'test*.csv',
|
||||
'TEST_TABLE',
|
||||
'CT_ET_TEMPLATES.TEST',
|
||||
'INVALID_STRATEGY'
|
||||
);
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: Check constraint did not fire!');
|
||||
ROLLBACK;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
IF SQLCODE = -2290 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('SUCCESS: Check constraint prevented invalid strategy');
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('FAILED: Unexpected error: ' || SQLERRM);
|
||||
END IF;
|
||||
ROLLBACK;
|
||||
END;
|
||||
/
|
||||
|
||||
PROMPT
|
||||
PROMPT ========================================
|
||||
PROMPT MARS-828: Testing Complete
|
||||
PROMPT ========================================
|
||||
@@ -3,7 +3,7 @@
|
||||
-- ===================================================================
|
||||
-- Purpose: Track specified Oracle package versions
|
||||
-- Author: Grzegorz Michalski
|
||||
-- Date: 2025-12-04
|
||||
-- Date: 2026-01-27
|
||||
-- Version: 3.1.0 - List-Based Edition
|
||||
--
|
||||
-- USAGE:
|
||||
@@ -16,7 +16,7 @@ SET SERVEROUTPUT ON;
|
||||
DECLARE
|
||||
TYPE t_package_rec IS RECORD (
|
||||
owner VARCHAR2(50),
|
||||
name VARCHAR2(50),
|
||||
package_name VARCHAR2(50),
|
||||
version VARCHAR2(50)
|
||||
);
|
||||
TYPE t_packages IS TABLE OF t_package_rec;
|
||||
@@ -29,8 +29,8 @@ DECLARE
|
||||
-- Format: 'SCHEMA.PACKAGE_NAME'
|
||||
-- ===================================================================
|
||||
vPackageList t_string_array := t_string_array(
|
||||
'CT_MRDS.FILE_MANAGER',
|
||||
'ODS.FILE_MANAGER_ODS'
|
||||
'CT_MRDS.FILE_ARCHIVER',
|
||||
'CT_MRDS.FILE_MANAGER'
|
||||
);
|
||||
-- ===================================================================
|
||||
|
||||
@@ -52,39 +52,44 @@ BEGIN
|
||||
vOwner := SUBSTR(vPackageList(i), 1, vDotPos - 1);
|
||||
vPackageName := SUBSTR(vPackageList(i), vDotPos + 1);
|
||||
|
||||
-- Get package version
|
||||
BEGIN
|
||||
EXECUTE IMMEDIATE 'SELECT ' || vPackageList(i) || '.GET_VERSION() FROM DUAL'
|
||||
INTO vVersion;
|
||||
|
||||
EXECUTE IMMEDIATE 'SELECT ' || vOwner || '.' || vPackageName || '.GET_VERSION() FROM DUAL' INTO vVersion;
|
||||
vPackages.EXTEND;
|
||||
vPackages(vPackages.COUNT).owner := vOwner;
|
||||
vPackages(vPackages.COUNT).name := vPackageName;
|
||||
vPackages(vPackages.COUNT).package_name := vPackageName;
|
||||
vPackages(vPackages.COUNT).version := vVersion;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.TRACK_PACKAGE_VERSION(
|
||||
pPackageOwner => vOwner,
|
||||
pPackageName => vPackageName,
|
||||
pPackageVersion => vVersion,
|
||||
pPackageBuildDate => TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'),
|
||||
pPackageAuthor => 'Grzegorz Michalski'
|
||||
);
|
||||
vCount := vCount + 1;
|
||||
-- Track in ENV_MANAGER
|
||||
BEGIN
|
||||
CT_MRDS.ENV_MANAGER.TRACK_PACKAGE_VERSION(
|
||||
pPackageOwner => vOwner,
|
||||
pPackageName => vPackageName,
|
||||
pPackageVersion => vVersion,
|
||||
pPackageBuildDate => TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS'),
|
||||
pPackageAuthor => 'Grzegorz Michalski'
|
||||
);
|
||||
vCount := vCount + 1;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN NULL; -- Continue even if tracking fails
|
||||
END;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
DBMS_OUTPUT.PUT_LINE('Error tracking ' || vPackageList(i) || ': ' || SQLERRM);
|
||||
WHEN OTHERS THEN NULL; -- Skip packages that fail
|
||||
END;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Display results
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Summary:');
|
||||
DBMS_OUTPUT.PUT_LINE('--------');
|
||||
DBMS_OUTPUT.PUT_LINE('Packages tracked: ' || vCount || '/' || vPackageList.COUNT);
|
||||
|
||||
IF vPackages.COUNT > 0 THEN
|
||||
DBMS_OUTPUT.PUT_LINE('Packages tracked: ' || vCount || ' of ' || vPackages.COUNT);
|
||||
DBMS_OUTPUT.PUT_LINE('');
|
||||
DBMS_OUTPUT.PUT_LINE('Tracked Packages:');
|
||||
FOR i IN 1..vPackages.COUNT LOOP
|
||||
DBMS_OUTPUT.PUT_LINE(' ' || vPackages(i).owner || '.' || vPackages(i).name ||
|
||||
' (v' || vPackages(i).version || ')');
|
||||
DBMS_OUTPUT.PUT_LINE(' ' || vPackages(i).owner || '.' || vPackages(i).package_name || ' v' || vPackages(i).version);
|
||||
END LOOP;
|
||||
ELSE
|
||||
DBMS_OUTPUT.PUT_LINE('No packages found in list');
|
||||
END IF;
|
||||
|
||||
DBMS_OUTPUT.PUT_LINE('========================================');
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user