admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-02-02 10:59:29 +01:00
commit ecd833f682
679 changed files with 122717 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_ET_TEMPLATES/CREATE_USER.sql Normal file
View File

@@ -0,0 +1,13 @@
-- ====================================================================
-- CT_ET_TEMPLATES User Creation Script
-- ====================================================================
-- Purpose: Create CT_ET_TEMPLATES schema for external table templates
-- ====================================================================
CREATE USER CT_ET_TEMPLATES IDENTIFIED BY "afa3343DDD___"
DEFAULT TABLESPACE DATA;
-- Grant basic privileges
GRANT CONNECT TO CT_ET_TEMPLATES;
GRANT RESOURCE TO CT_ET_TEMPLATES;
GRANT UNLIMITED TABLESPACE TO CT_ET_TEMPLATES;

39
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_ET_TEMPLATES/tables/IDS_DATA.sql Normal file
View File

@@ -0,0 +1,39 @@
-- ====================================================================
-- IDS_DATA Template Table
-- ====================================================================
-- Purpose: Template table for IDS weather data files
-- ====================================================================
CREATE TABLE CT_ET_TEMPLATES.IDS_DATA(
STATION VARCHAR2(2000),
DATE VARCHAR2(2000),
LATITUDE VARCHAR2(2000),
LONGITUDE VARCHAR2(2000),
ELEVATION VARCHAR2(2000),
NAME VARCHAR2(2000),
TEMP VARCHAR2(2000),
TEMP_ATTRIBUTES VARCHAR2(2000),
DEWP VARCHAR2(2000),
DEWP_ATTRIBUTES VARCHAR2(2000),
SLP VARCHAR2(2000),
SLP_ATTRIBUTES VARCHAR2(2000),
STP VARCHAR2(2000),
STP_ATTRIBUTES VARCHAR2(2000),
VISIB VARCHAR2(2000),
VISIB_ATTRIBUTES VARCHAR2(2000),
WDSP VARCHAR2(2000),
WDSP_ATTRIBUTES VARCHAR2(2000),
MXSPD VARCHAR2(2000),
GUST VARCHAR2(2000),
MAX VARCHAR2(2000),
MAX_ATTRIBUTES VARCHAR2(2000),
MIN VARCHAR2(2000),
MIN_ATTRIBUTES VARCHAR2(2000),
PRCP VARCHAR2(2000),
PRCP_ATTRIBUTES VARCHAR2(2000),
SNDP VARCHAR2(2000),
FRSHTT VARCHAR2(2000)
);
-- Grant access to CT_MRDS for FILE_MANAGER operations
GRANT SELECT ON CT_ET_TEMPLATES.IDS_DATA TO CT_MRDS;

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_ET_TEMPLATES/tables/ODS_LM_STANDING_FACILITIES.sql Normal file
View File

@@ -0,0 +1,22 @@
-- ====================================================================
-- ODS_LM_STANDING_FACILITIES Template Table
-- ====================================================================
-- Purpose: Template table for Liquidity Management Standing Facilities data
-- ====================================================================
CREATE TABLE CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES (
REFERENCE_DATE VARCHAR2(2000),
VERSION_NUMBER VARCHAR2(2000),
MFI_CODE VARCHAR2(2000),
BANK_NAME VARCHAR2(2000),
MARGINAL_LANDING VARCHAR2(2000),
DEPOSIT_FACILITY VARCHAR2(2000),
COUNTRY VARCHAR2(2000),
A_WORKFLOW_HISTORY_ID VARCHAR2(2000)
);
-- Grant access to ODS schema for external table creation
GRANT SELECT ON CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES TO ODS;
-- Grant access to CT_MRDS for FILE_MANAGER operations
GRANT SELECT ON CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES TO CT_MRDS;

21
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_ET_TEMPLATES/tables/ODS_LM_STANDING_FACILITIES_HEADER.sql Normal file
View File

@@ -0,0 +1,21 @@
-- ====================================================================
-- ODS_LM_STANDING_FACILITIES_HEADER Template Table
-- ====================================================================
-- Purpose: Template table for Liquidity Management Standing Facilities Header data
-- ====================================================================
CREATE TABLE CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES_HEADER (
REFERENCE_DATE VARCHAR2(2000),
VERSION_NUMBER VARCHAR2(2000),
MARGINAL_LENDINGS_BS_TOTAL VARCHAR2(2000),
DEPOSIT_FACILITY_BS_TOTAL VARCHAR2(2000),
MARGINAL_LENDINGS_SF_TOTAL VARCHAR2(2000),
DEPOSIT_FACILITY_SF_TOTAL VARCHAR2(2000),
A_WORKFLOW_HISTORY_ID VARCHAR2(2000)
);
-- Grant access to ODS schema for external table creation
GRANT SELECT ON CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES_HEADER TO ODS;
-- Grant access to CT_MRDS for FILE_MANAGER operations
GRANT SELECT ON CT_ET_TEMPLATES.ODS_LM_STANDING_FACILITIES_HEADER TO CT_MRDS;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
t.sql
temp/

0
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/.gitkeep Normal file
View File

36
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_acc.sql Normal file
View File

@@ -0,0 +1,36 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_acc_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload acc specific part
--=============================================================================================================================
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('default', 'EnvironmentID', 'acc', '');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'Region', 'eu-frankfurt-1', 'OCI region for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'NameSpace', 'frcnomajoc7v', 'OCI namespace for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'InboxBucketName', 'mrds_inbox_acc', 'Name of the inbox bucket for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'DataBucketName', 'mrds_data_acc', 'Name of the data bucket for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'HistoryBucketName', 'mrds_hist_acc', 'Name of the history bucket for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'CredentialName', 'OCI$RESOURCE_PRINCIPAL', 'Name of the credential for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'LoggingEnabled', 'ON', 'Enable logging for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'MinLogLevel', 'DEBUG', 'Minimum logging level for acc environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('acc', 'DefaultDateFormat', 'DD/MM/YYYY HH24:MI:SS', 'Default date format for acc environment');
COMMIT;

123
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_create_ODS_package.sql Normal file
View File

@@ -0,0 +1,123 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_create_ODS_package_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 create ODS package
--=============================================================================================================================
CREATE OR REPLACE EDITIONABLE PACKAGE "ODS"."FILE_MANAGER_ODS"
AUTHID DEFINER
AS
/**
* FILE_MANAGER_ODS - Wrapper package for CT_MRDS.FILE_MANAGER
*
* This package serves as a wrapper for CT_MRDS.FILE_MANAGER procedures.
* The key difference is that this package uses AUTHID DEFINER instead of
* AUTHID CURRENT_USER, which means all objects will be created in the ODS
* schema regardless of which user executes the procedures.
*
* This solves the execution context issue where users need to connect as
* ODS user to use CT_MRDS.FILE_MANAGER procedures.
*/
/**
* @name CREATE_EXTERNAL_TABLE
* @desc Creates external tables that can read data from Oracle Cloud Storage.
* This is a wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE
* but uses DEFINER rights to ensure objects are created in ODS schema.
* @param pTableName - Name of the external table to create
* @param pTemplateTableName - Template table defining the structure
* @param pPrefix - Storage path prefix in Oracle Cloud Storage
* @param pBucketUri - URI of the target bucket (default: ENV_MANAGER.gvInboxBucketUri)
* @param pFileName - Specific file name (optional)
* @param pDelimiter - Field delimiter (default: ',')
* @example EXEC ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE(
* 'C2D_A_UC_DISSEM_METADATA_LOADS_INBOX',
* 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
* 'INBOX/C2D/UC_DISSEM/A_UC_DISSEM_METADATA_LOADS',
* CT_MRDS.ENV_MANAGER.gvInboxBucketUri
* );
*/
PROCEDURE CREATE_EXTERNAL_TABLE (
pTableName IN VARCHAR2,
pTemplateTableName IN VARCHAR2,
pPrefix IN VARCHAR2,
pBucketUri IN VARCHAR2 DEFAULT CT_MRDS.ENV_MANAGER.gvInboxBucketUri,
pFileName IN VARCHAR2 DEFAULT NULL,
pDelimiter IN VARCHAR2 DEFAULT ','
);
END FILE_MANAGER_ODS;
/
CREATE OR REPLACE EDITIONABLE PACKAGE BODY "ODS"."FILE_MANAGER_ODS"
AS
/**
* CREATE_EXTERNAL_TABLE - Wrapper for CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE
*/
PROCEDURE CREATE_EXTERNAL_TABLE (
pTableName IN VARCHAR2,
pTemplateTableName IN VARCHAR2,
pPrefix IN VARCHAR2,
pBucketUri IN VARCHAR2 DEFAULT CT_MRDS.ENV_MANAGER.gvInboxBucketUri,
pFileName IN VARCHAR2 DEFAULT NULL,
pDelimiter IN VARCHAR2 DEFAULT ','
)
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
-- Log the start of the procedure
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(
SYS.ODCIVARCHAR2LIST(
'pTableName => ''' || pTableName || '''',
'pTemplateTableName => ''' || pTemplateTableName || '''',
'pPrefix => ''' || pPrefix || '''',
'pBucketUri => ''' || pBucketUri || '''',
'pFileName => ''' || NVL(pFileName, 'NULL') || '''',
'pDelimiter => ''' || pDelimiter || ''''
)
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE', 'INFO', vParameters);
-- Call the original CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE procedure
-- This ensures all logic remains centralized in the original package
-- and ODS wrapper simply delegates execution with DEFINER rights
CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE(
pTableName => pTableName,
pTemplateTableName => pTemplateTableName,
pPrefix => pPrefix,
pBucketUri => pBucketUri,
pFileName => pFileName,
pDelimiter => pDelimiter
);
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE', 'INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(
'Error in ODS.FILE_MANAGER_ODS.CREATE_EXTERNAL_TABLE: ' || SQLERRM,
'ERROR',
vParameters
);
RAISE;
END CREATE_EXTERNAL_TABLE;
END FILE_MANAGER_ODS;
/

39
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_create_ODS_user.sql Normal file
View File

@@ -0,0 +1,39 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_create_ODS_user_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 create ODS user
--=============================================================================================================================
GRANT CONNECT, CREATE TABLE TO ODS;
GRANT READ, WRITE ON DIRECTORY DATA_PUMP_DIR TO ODS;
GRANT EXECUTE ON DBMS_CLOUD TO ODS;
EXEC DBMS_CLOUD_ADMIN.ENABLE_RESOURCE_PRINCIPAL(username => 'ODS');
GRANT EXECUTE ON CT_MRDS.FILE_MANAGER TO ODS;
-- GRANT SELECT ANY TABLE ON SCHEMA CT_MRDS TO ODS;
-- GRANT DELETE ANY TABLE ON SCHEMA CT_MRDS TO ODS;
-- GRANT INSERT ANY TABLE ON SCHEMA CT_MRDS TO ODS;
-- GRANT UPDATE ANY TABLE ON SCHEMA CT_MRDS TO ODS;
-- GRANT EXECUTE ANY PROCEDURE ON SCHEMA CT_MRDS TO ODS;
GRANT SELECT ANY TABLE TO ODS;
GRANT DELETE ANY TABLE TO ODS;
GRANT INSERT ANY TABLE TO ODS;
GRANT UPDATE ANY TABLE TO ODS;
GRANT EXECUTE ANY PROCEDURE TO ODS;
GRANT SELECT_CATALOG_ROLE TO ODS;
GRANT SELECT ANY TABLE TO ODS;

37
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_create_user.sql Normal file
View File

@@ -0,0 +1,37 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_create_user_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 create user
--=============================================================================================================================
--CREATE USER CT_MRDS IDENTIFIED BY Start_1234567890
--DEFAULT TABLESPACE USERS TEMPORARY TABLESPACE TEMP;
ALTER USER CT_MRDS QUOTA UNLIMITED ON DATA;
EXEC DBMS_CLOUD_ADMIN.ENABLE_RESOURCE_PRINCIPAL(username => 'CT_MRDS');
GRANT EXECUTE ON DBMS_CLOUD TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER ADMIN TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER PDBADMIN TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER PDBSOFA TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER MRDS_LOADER TO CT_MRDS;

38
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_dev.sql Normal file
View File

@@ -0,0 +1,38 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_dev_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload dev specific part
--=============================================================================================================================
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('default', 'EnvironmentID', 'dev', '');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'Region', 'eu-frankfurt-1', 'OCI region for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'NameSpace', 'frcnomajoc7v', 'OCI namespace for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'InboxBucketName', 'mrds_inbox_dev', 'Name of the inbox bucket for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'DataBucketName', 'mrds_data_dev', 'Name of the data bucket for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'HistoryBucketName', 'mrds_hist_dev', 'Name of the history bucket for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'CredentialName', 'OCI$RESOURCE_PRINCIPAL', 'Name of the credential for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'LoggingEnabled', 'ON', 'Enable logging for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'MinLogLevel', 'DEBUG', 'Minimum logging level for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'DefaultDateFormat', 'DD/MM/YYYY HH24:MI:SS', 'Default date format for dev environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('dev', 'ConsoleLoggingEnabled', 'ON', 'Controls whether log messages are also displayed in console using DBMS_OUTPUT.PUT_LINE');
COMMIT;

218
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_file_manager.sql Normal file
View File

@@ -0,0 +1,218 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_file_manager_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 file manager relevant objects
--=============================================================================================================================
create or replace type CT_MRDS.T_FILENAME as object (
FILENAME varchar2(200)
,PATHNAME varchar2(800)
,YEAR varchar2(4)
,MONTH varchar2(2)
);
/
create type CT_MRDS.T_FILENAMES is table of T_FILENAME;
/
--
-- CT_MRDS.A_FILE_MANAGER_CONFIG
--
CREATE TABLE "CT_MRDS"."A_FILE_MANAGER_CONFIG" (
environment_id VARCHAR2(100),
config_variable VARCHAR2(200),
config_variable_value VARCHAR2(200),
config_variable_comment VARCHAR2(600),
PRIMARY KEY (environment_id, config_variable)
);
CREATE TABLE "CT_MRDS"."A_TABLE_STAT"
( "A_TABLE_STAT_KEY" NUMBER(38,0) PRIMARY KEY,
"A_SOURCE_FILE_CONFIG_KEY" NUMBER(38,0) NOT NULL,
"TABLE_NAME" VARCHAR2(200) NOT NULL,
"FILE_COUNT" NUMBER(38,0),
"OVER_HIST_THRESOLD_FILE_COUNT" NUMBER(38,0),
"ROW_COUNT" NUMBER(38,0),
"OVER_HIST_THRESOLD_ROW_COUNT" NUMBER(38,0),
"SIZE" NUMBER(38,0),
"OVER_HIST_THRESOLD_SIZE" NUMBER(38,0),
"HIST_THRESHOLD_DAYS" NUMBER(4,0),
"CREATED" TIMESTAMP (6) default systimestamp
)
TABLESPACE "DATA" ;
alter table "CT_MRDS"."A_TABLE_STAT" add constraint A_TABLE_STAT_UK1 unique("A_SOURCE_FILE_CONFIG_KEY");
CREATE SEQUENCE CT_MRDS.A_TABLE_STAT_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_TABLE_STAT_KEY_SEQ TO MRDS_LOADER_ROLE;
---
CREATE TABLE "CT_MRDS"."A_TABLE_STAT_HIST"
( "A_TABLE_STAT_HIST_KEY" NUMBER(38,0) PRIMARY KEY,
"A_SOURCE_FILE_CONFIG_KEY" NUMBER(38,0) NOT NULL,
"TABLE_NAME" VARCHAR2(200) NOT NULL,
"FILE_COUNT" NUMBER(38,0),
"OVER_HIST_THRESOLD_FILE_COUNT" NUMBER(38,0),
"ROW_COUNT" NUMBER(38,0),
"OVER_HIST_THRESOLD_ROW_COUNT" NUMBER(38,0),
"SIZE" NUMBER(38,0),
"OVER_HIST_THRESOLD_SIZE" NUMBER(38,0),
"HIST_THRESHOLD_DAYS" NUMBER(4,0),
"CREATED" TIMESTAMP (6) default systimestamp
)
TABLESPACE "DATA" ;
--
-- CT_MRDS.A_SOURCE
--
CREATE TABLE "CT_MRDS"."A_SOURCE"
( "A_SOURCE_KEY" VARCHAR2(30),
"SOURCE_NAME" VARCHAR2(200)
)
TABLESPACE "DATA" ;
ALTER TABLE "CT_MRDS"."A_SOURCE"
ADD CONSTRAINT A_SOURCE_PK PRIMARY KEY (A_SOURCE_KEY);
GRANT SELECT, INSERT, UPDATE, DELETE ON "CT_MRDS"."A_SOURCE" TO MRDS_LOADER_ROLE;
--
-- CT_MRDS.A_SOURCE_FILE_CONFIG
--
CREATE SEQUENCE CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ TO MRDS_LOADER_ROLE;
-- DROP TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG";
CREATE TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
( "A_SOURCE_FILE_CONFIG_KEY" NUMBER(38,0) NOT NULL ENABLE,
"A_SOURCE_KEY" VARCHAR2(30) NOT NULL ENABLE,
"SOURCE_FILE_TYPE" VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG', the latter is for files that are mapped to multiple tables and split into different input files
"SOURCE_FILE_ID" VARCHAR2(200),
"SOURCE_FILE_DESC" VARCHAR2(2000),
"SOURCE_FILE_NAME_PATTERN" VARCHAR2(200),
"TABLE_ID" VARCHAR2(200),
"TEMPLATE_TABLE_NAME" VARCHAR2(200),
"CONTAINER_FILE_KEY" NUMBER(38,0),
"DAYS_FOR_HISTORY_THRESHOLD" NUMBER(4,0),
"FILES_COUNT_OVER_HISTORY_THRESHOLD" NUMBER(38,0),
"BYTES_SUM_OVER_HISTORY_THRESHOLD" NUMBER(38,0),
"ODS_SCHEMA_NAME" VARCHAR2(100),
"ROWS_COUNT_OVER_HISTORY_THRESHOLD" NUMBER(38,0),
"HOURS_TO_EXPIRE_STATISTICS" NUMBER(38,3)
)
TABLESPACE "DATA"
;
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
ADD CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY);
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
ADD CONSTRAINT SOURCE_FILE_TYPE_CHK
CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG'));
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
ADD CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY("A_SOURCE_KEY") REFERENCES "CT_MRDS"."A_SOURCE"("A_SOURCE_KEY");
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
ADD CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY("CONTAINER_FILE_KEY") REFERENCES "CT_MRDS"."A_SOURCE_FILE_CONFIG"("A_SOURCE_FILE_CONFIG_KEY");
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_CONFIG"
ADD CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE("SOURCE_FILE_TYPE","SOURCE_FILE_ID","TABLE_ID");
GRANT SELECT, INSERT, UPDATE, DELETE ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" TO MRDS_LOADER_ROLE;
--
-- CT_MRDS.A_SOURCE_FILE_RECEIVED
--
CREATE SEQUENCE CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ TO MRDS_LOADER_ROLE;
-- DROP TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED";
CREATE TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED"
( "A_SOURCE_FILE_RECEIVED_KEY" NUMBER(38,0) NOT NULL ENABLE,
"A_SOURCE_FILE_CONFIG_KEY" NUMBER(38,0) NOT NULL ENABLE,
"SOURCE_FILE_NAME" VARCHAR2(1000) NOT NULL,
"CHECKSUM" VARCHAR2(128),
"CREATED" TIMESTAMP (6) WITH TIME ZONE,
"BYTES" NUMBER,
"RECEPTION_DATE" DATE NOT NULL,
"PROCESSING_STATUS" VARCHAR2(200),
"EXTERNAL_TABLE_NAME" VARCHAR2(200),
"PARTITION_YEAR" VARCHAR2(4),
"PARTITION_MONTH" VARCHAR2(2),
"HIST_FILE_NAME" VARCHAR2(1000)
)
TABLESPACE "DATA" ;
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED"
ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY);
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED"
ADD CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY("A_SOURCE_FILE_CONFIG_KEY") REFERENCES "CT_MRDS"."A_SOURCE_FILE_CONFIG"("A_SOURCE_FILE_CONFIG_KEY");
ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED"
ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK
CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'VALIDATION_FAILED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED'));
-- Because "CREATED" column is type: TIMESTAMP WITH TIMEZONE, the unique constraint cannot be created.
-- As a workaround the unique index is created: A_SOURCE_FILE_RECEIVED_UK1
--ALTER TABLE "CT_MRDS"."A_SOURCE_FILE_RECEIVED"
--ADD CONSTRAINT A_SOURCE_FILE_RECEIVED_UK1 UNIQUE("CHECKSUM","CREATED","BYTES");
create unique index CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1 on CT_MRDS.A_SOURCE_FILE_RECEIVED("CHECKSUM","CREATED","BYTES");
GRANT SELECT, INSERT, UPDATE, DELETE ON "CT_MRDS"."A_SOURCE_FILE_RECEIVED" TO MRDS_LOADER_ROLE;
--
-- CT_MRDS.A_COLUMN_DATE_FORMAT
--
-- DROP TABLE "CT_MRDS"."A_COLUMN_DATE_FORMAT";
CREATE TABLE "CT_MRDS"."A_COLUMN_DATE_FORMAT"
( "TEMPLATE_TABLE_NAME" VARCHAR2(200) NOT NULL ENABLE,
"COLUMN_NAME" VARCHAR2(200) NOT NULL,
"DATE_FORMAT" VARCHAR2(200) NOT NULL
)
TABLESPACE "DATA" ;
ALTER TABLE "CT_MRDS"."A_COLUMN_DATE_FORMAT"
ADD CONSTRAINT A_COLUMN_DATE_FORMAT_PK PRIMARY KEY (TEMPLATE_TABLE_NAME, COLUMN_NAME);
GRANT SELECT, INSERT, UPDATE, DELETE ON "CT_MRDS"."A_COLUMN_DATE_FORMAT" TO MRDS_LOADER_ROLE;
--
-- CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY
--
CREATE TABLE CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY
(
"A_WORKFLOW_HISTORY_KEY" NUMBER,
"SERVICE_NAME" VARCHAR2(200),
"PROPERTY" VARCHAR2(200),
"VALUE" VARCHAR2(4000 CHAR)
)
TABLESPACE "DATA" ;
--ALTER TABLE "CT_MRDS"."A_WORKFLOW_HISTORY_PROPERTY"
--ADD CONSTRAINT A_WORKFLOW_HISTORY_FK FOREIGN KEY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME) REFERENCES CT_MRDS.A_WORKFLOW_HISTORY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME)
--;
CREATE INDEX CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY_IDX1 ON CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME);

75
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_grants.sql Normal file
View File

@@ -0,0 +1,75 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_grants_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload grants specific part
--=============================================================================================================================
-- GRANT SELECT ANY TABLE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT INSERT ANY TABLE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT UPDATE ANY TABLE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT DELETE ANY TABLE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT EXECUTE ANY PROCEDURE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT SELECT ANY SEQUENCE ON SCHEMA CT_MRDS TO MRDS_LOADER;
-- GRANT SELECT ANY TABLE ON SCHEMA CT_MRDS TO MOPDB_PUBLIC_GL;
-- GRANT SELECT ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT INSERT ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT UPDATE ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT DELETE ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT CREATE ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT DROP ANY TABLE ON SCHEMA CT_ET_TEMPLATES TO MRDS_LOADER;
-- GRANT SELECT ANY TABLE ON SCHEMA OU_RQSD TO RQSD;
-- BEGIN
-- FOR user_rec IN (SELECT username FROM dba_users WHERE oracle_maintained != 'Y' AND common != 'YES') LOOP
-- DECLARE
-- schema_name VARCHAR2(128) := user_rec.username;
-- BEGIN
-- EXECUTE IMMEDIATE 'GRANT SELECT ON CT_MRDS.A_WORKFLOW_HISTORY TO "'|| schema_name ||'" WITH GRANT OPTION';
-- EXECUTE IMMEDIATE 'GRANT SELECT ANY TABLE ON SCHEMA ODS TO '|| schema_name ;
-- EXCEPTION
-- WHEN OTHERS THEN
-- DBMS_OUTPUT.PUT_LINE('Error processing schema ' || schema_name || ': ' || SQLERRM);
-- END;
-- END LOOP;
-- END;
GRANT SELECT ANY TABLE TO OU_RQSD;
GRANT SELECT ANY TABLE TO RQSD;
BEGIN
FOR user_rec IN (SELECT username FROM dba_users WHERE oracle_maintained != 'Y' AND common != 'YES') LOOP
DECLARE
schema_name VARCHAR2(128) := user_rec.username;
BEGIN
EXECUTE IMMEDIATE 'GRANT SELECT ON CT_MRDS.A_WORKFLOW_HISTORY TO "'|| schema_name ||'" WITH GRANT OPTION';
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('Error processing schema ' || schema_name || ': ' || SQLERRM);
END;
END LOOP;
END;

797
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_packages_1.sql Normal file
View File

@@ -0,0 +1,797 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_packages_1_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload CT_MRDS.ENV_MANAGER package
--=============================================================================================================================
CREATE OR REPLACE EDITIONABLE PACKAGE "CT_MRDS"."ENV_MANAGER"
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select ENV_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
TYPE Error_Record IS RECORD (
code PLS_INTEGER,
message VARCHAR2(4000)
);
TYPE tErrorList IS TABLE OF Error_Record INDEX BY PLS_INTEGER;
Errors tErrorList;
guid VARCHAR2(32);
gvEnv VARCHAR2(200);
gvUsername VARCHAR2(128);
gvOsuser VARCHAR2(128);
gvMachine VARCHAR2(64);
gvModule VARCHAR2(64);
gvNameSpace VARCHAR2(200);
gvRegion VARCHAR2(200);
gvDataBucketName VARCHAR2(200);
gvInboxBucketName VARCHAR2(200);
gvHistoryBucketName VARCHAR2(200);
gvDataBucketUri VARCHAR2(200);
gvInboxBucketUri VARCHAR2(200);
gvHistoryBucketUri VARCHAR2(200);
gvCredentialName VARCHAR2(200);
-- Overwritten by variable "LoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF'
-- Overwritten by variable "MinLogLevel" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
-- Possible values: DEBUG ,INFO ,WARNING ,ERROR
gvMinLogLevel VARCHAR2(10) := 'DEBUG';
-- Overwritten by variable "DefaultDateFormat" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvDefaultDateFormat VARCHAR2(200) := 'DD/MM/YYYY HH24:MI:SS';
-- Overwritten by variable "ConsoleLoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvConsoleLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF'
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgSourceFileConfigKey PLS_INTEGER;
vgMsgTmp VARCHAR2(32000);
--Exceptions
ERR_EMPTY_FILEURI_AND_RECKEY EXCEPTION;
CODE_EMPTY_FILEURI_AND_RECKEY CONSTANT PLS_INTEGER := -20001;
MSG_EMPTY_FILEURI_AND_RECKEY_C VARCHAR2(4000) := 'Either pFileUri or pSourceFileReceivedKey must be not null';
MSG_EMPTY_FILEURI_AND_RECKEY VARCHAR2(4000) := MSG_EMPTY_FILEURI_AND_RECKEY_C;
PRAGMA EXCEPTION_INIT( ERR_EMPTY_FILEURI_AND_RECKEY
,CODE_EMPTY_FILEURI_AND_RECKEY);
ERR_NO_CONFIG_MATCH_FOR_FILEURI EXCEPTION;
CODE_NO_CONFIG_MATCH_FOR_FILEURI CONSTANT PLS_INTEGER := -20002;
MSG_NO_CONFIG_MATCH_FOR_FILEURI_C VARCHAR2(4000) := 'No match for source file in A_SOURCE_FILE_CONFIG table'
||cgBL||' The file provided in parameter: pFileUri does not have '
||cgBL||' coresponding configuration in A_SOURCE_FILE_CONFIG table';
MSG_NO_CONFIG_MATCH_FOR_FILEURI VARCHAR2(4000) := MSG_NO_CONFIG_MATCH_FOR_FILEURI_C;
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH_FOR_FILEURI
,CODE_NO_CONFIG_MATCH_FOR_FILEURI);
ERR_MULTIPLE_MATCH_FOR_SRCFILE EXCEPTION;
CODE_MULTIPLE_MATCH_FOR_SRCFILE CONSTANT PLS_INTEGER := -20003;
MSG_MULTIPLE_MATCH_FOR_SRCFILE_C VARCHAR2(4000) := 'Multiple match for source file in A_SOURCE_FILE_CONFIG table';
MSG_MULTIPLE_MATCH_FOR_SRCFILE VARCHAR2(4000) := MSG_MULTIPLE_MATCH_FOR_SRCFILE_C;
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_MATCH_FOR_SRCFILE
,CODE_MULTIPLE_MATCH_FOR_SRCFILE);
ERR_MISSING_COLUMN_DATE_FORMAT EXCEPTION;
CODE_MISSING_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20004;
MSG_MISSING_COLUMN_DATE_FORMAT_C VARCHAR2(4000) := 'Missing entry in config table: A_COLUMN_DATE_FORMAT primary key(A_SOURCE_FILE_CONFIG_KEY, COLUMN_NAME)'
||cgBL||' Remember: each column which data_type IN (''DATE'', ''TIMESTAMP'')'
||cgBL||' should have DateFormat specified in A_COLUMN_DATE_FORMAT table '
||cgBL||' for example: ''YYYY-MM-DD''';
MSG_MISSING_COLUMN_DATE_FORMAT VARCHAR2(4000) := MSG_MISSING_COLUMN_DATE_FORMAT_C;
PRAGMA EXCEPTION_INIT( ERR_MISSING_COLUMN_DATE_FORMAT
,CODE_MISSING_COLUMN_DATE_FORMAT);
ERR_MULTIPLE_COLUMN_DATE_FORMAT EXCEPTION;
CODE_MULTIPLE_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20005;
MSG_MULTIPLE_COLUMN_DATE_FORMAT_C VARCHAR2(4000) := 'Multiple records for date format in A_COLUMN_DATE_FORMAT table'
||cgBL||' There should be only one format specified for each DAT/TIMESTAMP column';
MSG_MULTIPLE_COLUMN_DATE_FORMAT VARCHAR2(4000) := MSG_MULTIPLE_COLUMN_DATE_FORMAT_C;
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_COLUMN_DATE_FORMAT
,CODE_MULTIPLE_COLUMN_DATE_FORMAT);
ERR_DIDNT_GET_LOAD_OPERATION_ID EXCEPTION;
CODE_DIDNT_GET_LOAD_OPERATION_ID CONSTANT PLS_INTEGER := -20006;
MSG_DIDNT_GET_LOAD_OPERATION_ID_C VARCHAR2(4000) := 'Didnt get load operation id from external table validation';
MSG_DIDNT_GET_LOAD_OPERATION_ID VARCHAR2(4000) := MSG_DIDNT_GET_LOAD_OPERATION_ID_C;
PRAGMA EXCEPTION_INIT( ERR_DIDNT_GET_LOAD_OPERATION_ID
,CODE_DIDNT_GET_LOAD_OPERATION_ID);
ERR_NO_CONFIG_FOR_RECEIVED_FILE EXCEPTION;
CODE_NO_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20007;
MSG_NO_CONFIG_FOR_RECEIVED_FILE_C VARCHAR2(4000) := 'No match for received source file in A_SOURCE_FILE_CONFIG '
||cgBL||' or missing data in A_SOURCE_FILE_RECEIVED table for provided pSourceFileReceivedKey parameter';
MSG_NO_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := MSG_NO_CONFIG_FOR_RECEIVED_FILE_C;
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_FOR_RECEIVED_FILE
,CODE_NO_CONFIG_FOR_RECEIVED_FILE);
ERR_MULTI_CONFIG_FOR_RECEIVED_FILE EXCEPTION;
CODE_MULTI_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20008;
MSG_MULTI_CONFIG_FOR_RECEIVED_FILE_C VARCHAR2(4000) := 'Multiple matchs for received source file in A_SOURCE_FILE_CONFIG';
MSG_MULTI_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := MSG_MULTI_CONFIG_FOR_RECEIVED_FILE_C;
PRAGMA EXCEPTION_INIT( ERR_MULTI_CONFIG_FOR_RECEIVED_FILE
,CODE_MULTI_CONFIG_FOR_RECEIVED_FILE);
ERR_FILE_NOT_FOUND_ON_CLOUD EXCEPTION;
CODE_FILE_NOT_FOUND_ON_CLOUD CONSTANT PLS_INTEGER := -20009;
MSG_FILE_NOT_FOUND_ON_CLOUD_C VARCHAR2(4000) := 'File not found on the cloud';
MSG_FILE_NOT_FOUND_ON_CLOUD VARCHAR2(4000) := MSG_FILE_NOT_FOUND_ON_CLOUD_C;
PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_FOUND_ON_CLOUD
,CODE_FILE_NOT_FOUND_ON_CLOUD);
ERR_FILE_VALIDATION_FAILED EXCEPTION;
CODE_FILE_VALIDATION_FAILED CONSTANT PLS_INTEGER := -20010;
MSG_FILE_VALIDATION_FAILED_C VARCHAR2(4000) := 'File validation failed';
MSG_FILE_VALIDATION_FAILED VARCHAR2(4000) := MSG_FILE_VALIDATION_FAILED_C;
PRAGMA EXCEPTION_INIT( ERR_FILE_VALIDATION_FAILED
,CODE_FILE_VALIDATION_FAILED);
ERR_NO_CONFIG_MATCH EXCEPTION;
CODE_NO_CONFIG_MATCH CONSTANT PLS_INTEGER := -20011;
MSG_NO_CONFIG_MATCH_C VARCHAR2(4000) := 'No match for specified parameters in A_SOURCE_FILE_CONFIG table';
MSG_NO_CONFIG_MATCH VARCHAR2(4000) := MSG_NO_CONFIG_MATCH_C;
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH
,CODE_NO_CONFIG_MATCH);
ERR_UNKNOWN_PREFIX EXCEPTION;
CODE_UNKNOWN_PREFIX CONSTANT PLS_INTEGER := -20012;
MSG_UNKNOWN_PREFIX_C VARCHAR2(4000) := 'Unknown prefix';
MSG_UNKNOWN_PREFIX VARCHAR2(4000) := MSG_UNKNOWN_PREFIX_C;
PRAGMA EXCEPTION_INIT( ERR_UNKNOWN_PREFIX
,CODE_UNKNOWN_PREFIX);
ERR_TABLE_NOT_EXISTS EXCEPTION;
CODE_TABLE_NOT_EXISTS CONSTANT PLS_INTEGER := -20013;
MSG_TABLE_NOT_EXISTS_C VARCHAR2(4000) := 'Table does not exist';
MSG_TABLE_NOT_EXISTS VARCHAR2(4000) := MSG_TABLE_NOT_EXISTS_C;
PRAGMA EXCEPTION_INIT( ERR_TABLE_NOT_EXISTS
,CODE_TABLE_NOT_EXISTS);
ERR_COLUMN_NOT_EXISTS EXCEPTION;
CODE_COLUMN_NOT_EXISTS CONSTANT PLS_INTEGER := -20014;
MSG_COLUMN_NOT_EXISTS_C VARCHAR2(4000) := 'Column does not exist in table';
MSG_COLUMN_NOT_EXISTS VARCHAR2(4000) := MSG_COLUMN_NOT_EXISTS_C;
PRAGMA EXCEPTION_INIT( ERR_COLUMN_NOT_EXISTS
,CODE_COLUMN_NOT_EXISTS);
ERR_UNSUPPORTED_DATA_TYPE EXCEPTION;
CODE_UNSUPPORTED_DATA_TYPE CONSTANT PLS_INTEGER := -20015;
MSG_UNSUPPORTED_DATA_TYPE_C VARCHAR2(4000) := 'Unsupported data type';
MSG_UNSUPPORTED_DATA_TYPE VARCHAR2(4000) := MSG_UNSUPPORTED_DATA_TYPE_C;
PRAGMA EXCEPTION_INIT( ERR_UNSUPPORTED_DATA_TYPE
,CODE_UNSUPPORTED_DATA_TYPE);
ERR_MISSING_SOURCE_KEY EXCEPTION;
CODE_MISSING_SOURCE_KEY CONSTANT PLS_INTEGER := -20016;
MSG_MISSING_SOURCE_KEY_C VARCHAR2(4000) := 'The Source was not found in parent table A_SOURCE';
MSG_MISSING_SOURCE_KEY VARCHAR2(4000) := MSG_MISSING_SOURCE_KEY_C;
PRAGMA EXCEPTION_INIT( ERR_MISSING_SOURCE_KEY
,CODE_MISSING_SOURCE_KEY);
ERR_NULL_SOURCE_FILE_CONFIG_KEY EXCEPTION;
CODE_NULL_SOURCE_FILE_CONFIG_KEY CONSTANT PLS_INTEGER := -20017;
MSG_NULL_SOURCE_FILE_CONFIG_KEY_C VARCHAR2(4000) := 'No entry in A_SOURCE_FILE_CONFIG table for specified A_SOURCE_FILE_CONFIG_KEY';
MSG_NULL_SOURCE_FILE_CONFIG_KEY VARCHAR2(4000) := MSG_NULL_SOURCE_FILE_CONFIG_KEY_C;
PRAGMA EXCEPTION_INIT( ERR_NULL_SOURCE_FILE_CONFIG_KEY
,CODE_NULL_SOURCE_FILE_CONFIG_KEY);
ERR_DUPLICATED_SOURCE_KEY EXCEPTION;
CODE_DUPLICATED_SOURCE_KEY CONSTANT PLS_INTEGER := -20018;
MSG_DUPLICATED_SOURCE_KEY_C VARCHAR2(4000) := 'The Source already exists in the A_SOURCE table';
MSG_DUPLICATED_SOURCE_KEY VARCHAR2(4000) := MSG_DUPLICATED_SOURCE_KEY_C;
PRAGMA EXCEPTION_INIT( ERR_DUPLICATED_SOURCE_KEY
,CODE_DUPLICATED_SOURCE_KEY);
ERR_MISSING_CONTAINER_CONFIG EXCEPTION;
CODE_MISSING_CONTAINER_CONFIG CONSTANT PLS_INTEGER := -20019;
MSG_MISSING_CONTAINER_CONFIG_C VARCHAR2(4000) := 'No match in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID';
MSG_MISSING_CONTAINER_CONFIG VARCHAR2(4000) := MSG_MISSING_CONTAINER_CONFIG_C;
PRAGMA EXCEPTION_INIT( ERR_MISSING_CONTAINER_CONFIG
,CODE_MISSING_CONTAINER_CONFIG);
ERR_MULTIPLE_CONTAINER_ENTRIES EXCEPTION;
CODE_MULTIPLE_CONTAINER_ENTRIES CONSTANT PLS_INTEGER := -20020;
MSG_MULTIPLE_CONTAINER_ENTRIES_C VARCHAR2(4000) := 'Multiple matches in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID';
MSG_MULTIPLE_CONTAINER_ENTRIES VARCHAR2(4000) := MSG_MULTIPLE_CONTAINER_ENTRIES_C;
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_CONTAINER_ENTRIES
,CODE_MULTIPLE_CONTAINER_ENTRIES);
ERR_WRONG_DESTINATION_PARAM EXCEPTION;
CODE_WRONG_DESTINATION_PARAM CONSTANT PLS_INTEGER := -20021;
MSG_WRONG_DESTINATION_PARAM_C VARCHAR2(4000) := 'Wrong destination parameter provided.';
MSG_WRONG_DESTINATION_PARAM VARCHAR2(4000) := MSG_WRONG_DESTINATION_PARAM_C;
PRAGMA EXCEPTION_INIT( ERR_WRONG_DESTINATION_PARAM
,CODE_WRONG_DESTINATION_PARAM);
ERR_FILE_NOT_EXISTS_ON_CLOUD EXCEPTION;
CODE_FILE_NOT_EXISTS_ON_CLOUD CONSTANT PLS_INTEGER := -20022;
MSG_FILE_NOT_EXISTS_ON_CLOUD_C VARCHAR2(4000) := 'File not exists on cloud.';
MSG_FILE_NOT_EXISTS_ON_CLOUD VARCHAR2(4000) := MSG_FILE_NOT_EXISTS_ON_CLOUD_C;
PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_EXISTS_ON_CLOUD
,CODE_FILE_NOT_EXISTS_ON_CLOUD);
ERR_FILE_ALREADY_REGISTERED EXCEPTION;
CODE_FILE_ALREADY_REGISTERED CONSTANT PLS_INTEGER := -20023;
MSG_FILE_ALREADY_REGISTERED_C VARCHAR2(4000) := 'File already registered in A_SOURCE_FILE_RECEIVED table.';
MSG_FILE_ALREADY_REGISTERED VARCHAR2(4000) := MSG_FILE_ALREADY_REGISTERED_C;
PRAGMA EXCEPTION_INIT( ERR_FILE_ALREADY_REGISTERED
,CODE_FILE_ALREADY_REGISTERED);
ERR_WRONG_DATE_TIMESTAMP_FORMAT EXCEPTION;
CODE_WRONG_DATE_TIMESTAMP_FORMAT CONSTANT PLS_INTEGER := -20024;
MSG_WRONG_DATE_TIMESTAMP_FORMAT_C VARCHAR2(4000) := 'Provided DATE or TIMESTAMP format has errors (possible duplicated codes, ex: ''DD'').';
MSG_WRONG_DATE_TIMESTAMP_FORMAT VARCHAR2(4000) := MSG_WRONG_DATE_TIMESTAMP_FORMAT_C;
PRAGMA EXCEPTION_INIT( ERR_WRONG_DATE_TIMESTAMP_FORMAT
,CODE_WRONG_DATE_TIMESTAMP_FORMAT);
ERR_ENVIRONMENT_NOT_SET EXCEPTION;
CODE_ENVIRONMENT_NOT_SET CONSTANT PLS_INTEGER := -20025;
MSG_ENVIRONMENT_NOT_SET_C VARCHAR2(4000) := 'EnvironmentID not set'
||cgBL||' Information about environment is needed to get proper configuration values.'
||cgBL||' It can be set up in two different ways:'
||cgBL||' 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => ''dev'')'
||cgBL||' 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values (''default'',''environment_id'',''dev'')'
||cgBL||' Session level setup (1.) takes precedence over configuration level one (2.)'
;
MSG_ENVIRONMENT_NOT_SET VARCHAR2(4000) := MSG_ENVIRONMENT_NOT_SET_C;
PRAGMA EXCEPTION_INIT( ERR_ENVIRONMENT_NOT_SET
,CODE_ENVIRONMENT_NOT_SET);
ERR_CONFIG_VARIABLE_NOT_SET EXCEPTION;
CODE_CONFIG_VARIABLE_NOT_SET CONSTANT PLS_INTEGER := -20026;
MSG_CONFIG_VARIABLE_NOT_SET_C VARCHAR2(4000) := 'Missing configuration value in A_FILE_MANAGER_CONFIG';
MSG_CONFIG_VARIABLE_NOT_SET VARCHAR2(4000) := MSG_CONFIG_VARIABLE_NOT_SET_C;
PRAGMA EXCEPTION_INIT( ERR_CONFIG_VARIABLE_NOT_SET
,CODE_CONFIG_VARIABLE_NOT_SET);
ERR_NOT_INPUT_SOURCE_FILE_TYPE EXCEPTION;
CODE_NOT_INPUT_SOURCE_FILE_TYPE CONSTANT PLS_INTEGER := -20027;
MSG_NOT_INPUT_SOURCE_FILE_TYPE_C VARCHAR2(4000) := 'Historization can be executed only for A_SOURCE_FILE_CONFIG_KEY where SOURCE_FILE_TYPE=''INPUT''';
MSG_NOT_INPUT_SOURCE_FILE_TYPE VARCHAR2(4000) := MSG_NOT_INPUT_SOURCE_FILE_TYPE_C;
PRAGMA EXCEPTION_INIT( ERR_NOT_INPUT_SOURCE_FILE_TYPE
,CODE_NOT_INPUT_SOURCE_FILE_TYPE);
ERR_EXP_DATA_FOR_HIST_FAILED EXCEPTION;
CODE_EXP_DATA_FOR_HIST_FAILED CONSTANT PLS_INTEGER := -20028;
MSG_EXP_DATA_FOR_HIST_FAILED_C VARCHAR2(4000) := 'Export data for historization failed.';
MSG_EXP_DATA_FOR_HIST_FAILED VARCHAR2(4000) := MSG_EXP_DATA_FOR_HIST_FAILED_C;
PRAGMA EXCEPTION_INIT( ERR_EXP_DATA_FOR_HIST_FAILED
,CODE_EXP_DATA_FOR_HIST_FAILED);
ERR_RESTORE_FILE_FROM_TRASH EXCEPTION;
CODE_RESTORE_FILE_FROM_TRASH CONSTANT PLS_INTEGER := -20029;
MSG_RESTORE_FILE_FROM_TRASH_C VARCHAR2(4000) := 'Unexpected issues occured while historization process. Restoration of exported files failed.';
MSG_RESTORE_FILE_FROM_TRASH VARCHAR2(4000) := MSG_RESTORE_FILE_FROM_TRASH_C;
PRAGMA EXCEPTION_INIT( ERR_RESTORE_FILE_FROM_TRASH
,CODE_RESTORE_FILE_FROM_TRASH);
ERR_CHANGE_STAT_TO_HISTORISED_FAILED EXCEPTION;
CODE_CHANGE_STAT_TO_HISTORISED_FAILED CONSTANT PLS_INTEGER := -20030;
MSG_CHANGE_STAT_TO_HISTORISED_FAILED_C VARCHAR2(4000) := 'Failed to change file status to: HISTORISED in A_SOURCE_FILE_RECEIVED table.';
MSG_CHANGE_STAT_TO_HISTORISED_FAILED VARCHAR2(4000) := MSG_CHANGE_STAT_TO_HISTORISED_FAILED_C;
PRAGMA EXCEPTION_INIT( ERR_CHANGE_STAT_TO_HISTORISED_FAILED
,CODE_CHANGE_STAT_TO_HISTORISED_FAILED);
ERR_MOVE_FILE_TO_TRASH_FAILED EXCEPTION;
CODE_MOVE_FILE_TO_TRASH_FAILED CONSTANT PLS_INTEGER := -20031;
MSG_MOVE_FILE_TO_TRASH_FAILED_C VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.';
MSG_MOVE_FILE_TO_TRASH_FAILED VARCHAR2(4000) := MSG_MOVE_FILE_TO_TRASH_FAILED_C;
PRAGMA EXCEPTION_INIT( ERR_MOVE_FILE_TO_TRASH_FAILED
,CODE_MOVE_FILE_TO_TRASH_FAILED);
ERR_DROP_EXPORTED_FILES_FAILED EXCEPTION;
CODE_DROP_EXPORTED_FILES_FAILED CONSTANT PLS_INTEGER := -20032;
MSG_DROP_EXPORTED_FILES_FAILED_C VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.';
MSG_DROP_EXPORTED_FILES_FAILED VARCHAR2(4000) := MSG_DROP_EXPORTED_FILES_FAILED_C;
PRAGMA EXCEPTION_INIT( ERR_DROP_EXPORTED_FILES_FAILED
,CODE_DROP_EXPORTED_FILES_FAILED);
ERR_UNKNOWN EXCEPTION;
CODE_UNKNOWN CONSTANT PLS_INTEGER := -20999;
MSG_UNKNOWN_C VARCHAR2(4000) := 'Unknown Error Occured';
MSG_UNKNOWN VARCHAR2(4000) := MSG_UNKNOWN_C;
PRAGMA EXCEPTION_INIT( ERR_UNKNOWN
,CODE_UNKNOWN);
---------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------
/**
* @name LOG_PROCESS_EVENT
* @desc Insert a new log record into A_PROCESS_LOG table.
* Also outputs to console if gvConsoleLoggingEnabled = 'ON'.
* Respects logging level configuration (gvMinLogLevel).
* @example ENV_MANAGER.LOG_PROCESS_EVENT('Process completed successfully', 'INFO', 'pParam1=value1');
* @ex_rslt Record inserted into A_PROCESS_LOG table and optionally displayed in console output
**/
PROCEDURE LOG_PROCESS_EVENT (
pLogMessage VARCHAR2
,pLogLevel VARCHAR2 DEFAULT 'ERROR'
,pParameters VARCHAR2 DEFAULT NULL
,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER'
);
/**
* @name INIT_ERRORS
* @desc Loads data into Errors array.
* Errors array is a list of Record(Error_Code, Error_Message) index by Error_Code.
* Called automatically during package initialization.
* @example Called automatically when package is first referenced
* @ex_rslt Errors array populated with all error codes and messages
**/
PROCEDURE INIT_ERRORS;
/**
* @name GET_DEFAULT_ENV
* @desc It returns string with name of default environment.
* Return string is A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID value.
* @example select ENV_MANAGER.GET_DEFAULT_ENV() from dual;
* @ex_rslt dev
**/
FUNCTION GET_DEFAULT_ENV
RETURN VARCHAR2;
/**
* @name INIT_VARIABLES
* @desc For specified pEnv parameter (A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID)
* Assign values to following global package variables:
* - gvNameSpace
* - gvRegion
* - gvCredentialName
* - gvInboxBucketName
* - gvDataBucketName
* - gvHistoryBucketName
* - gvInboxBucketUri
* - gvDataBucketUri
* - gvHistoryBucketUri
* - gvLoggingEnabled
* - gvMinLogLevel
* - gvDefaultDateFormat
* - gvConsoleLoggingEnabled
**/
PROCEDURE INIT_VARIABLES(
pEnv VARCHAR2
);
/**
* @name GET_ERROR_MESSAGE
* @desc It returns string with error message for specified pCode (Error_Code).
* Error message is take from Errors Array loaded by INIT_ERRORS procedure
* @example select ENV_MANAGER.GET_ERROR_MESSAGE(pCode => -20009) from dual;
* @ex_rslt File not found on the cloud
**/
FUNCTION GET_ERROR_MESSAGE(
pCode PLS_INTEGER
) RETURN VARCHAR2;
/**
* @name GET_ERROR_STACK
* @desc It returns string with all possible error stack info.
* Error message is take from Errors Array loaded by INIT_ERRORS procedure
* @example
* select ENV_MANAGER.GET_ERROR_STACK(
* pFormat => 'OUTPUT'
* ,pCode => -20009
* ,pSourceFileReceivedKey => NULL)
* from dual
* @ex_rslt
* ------------------------------------------------------+
* Error Message:
* ORA-0000: normal, successful completion
* -------------------------------------------------------
* Error Stack:
* -------------------------------------------------------
* Error Backtrace:
* ------------------------------------------------------+
**/
FUNCTION GET_ERROR_STACK(
pFormat VARCHAR2
,pCode PLS_INTEGER
,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL
) RETURN VARCHAR2;
/**
* @name FORMAT_PARAMETERS
* @desc Formats parameter list for logging purposes.
* Converts SYS.ODCIVARCHAR2LIST to formatted string with proper NULL handling.
* @example select ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('param1=value1', 'param2=NULL')) from dual;
* @ex_rslt param1=value1 ,
* param2=NULL
**/
FUNCTION FORMAT_PARAMETERS(
pParameterList SYS.ODCIVARCHAR2LIST
) RETURN VARCHAR2;
END ENV_MANAGER;
/
CREATE OR REPLACE EDITIONABLE PACKAGE BODY "CT_MRDS"."ENV_MANAGER"
AS
----------------------------------------------------------------------------------------------------
PROCEDURE INIT_ERRORS IS
BEGIN
Errors(CODE_EMPTY_FILEURI_AND_RECKEY) := Error_Record(CODE_EMPTY_FILEURI_AND_RECKEY, MSG_EMPTY_FILEURI_AND_RECKEY); -- -20001
Errors(CODE_NO_CONFIG_MATCH_FOR_FILEURI) := Error_Record(CODE_NO_CONFIG_MATCH_FOR_FILEURI, MSG_NO_CONFIG_MATCH_FOR_FILEURI); -- -20002
Errors(CODE_MULTIPLE_MATCH_FOR_SRCFILE) := Error_Record(CODE_MULTIPLE_MATCH_FOR_SRCFILE, MSG_MULTIPLE_MATCH_FOR_SRCFILE); -- -20003
Errors(CODE_MISSING_COLUMN_DATE_FORMAT) := Error_Record(CODE_MISSING_COLUMN_DATE_FORMAT, MSG_MISSING_COLUMN_DATE_FORMAT); -- -20004
Errors(CODE_MULTIPLE_COLUMN_DATE_FORMAT) := Error_Record(CODE_MULTIPLE_COLUMN_DATE_FORMAT, MSG_MULTIPLE_COLUMN_DATE_FORMAT); -- -20005
Errors(CODE_DIDNT_GET_LOAD_OPERATION_ID) := Error_Record(CODE_DIDNT_GET_LOAD_OPERATION_ID, MSG_DIDNT_GET_LOAD_OPERATION_ID); -- -20006
Errors(CODE_NO_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_NO_CONFIG_FOR_RECEIVED_FILE, MSG_NO_CONFIG_FOR_RECEIVED_FILE); -- -20007
Errors(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE) := Error_Record(CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); -- -20008
Errors(CODE_FILE_NOT_FOUND_ON_CLOUD) := Error_Record(CODE_FILE_NOT_FOUND_ON_CLOUD, MSG_FILE_NOT_FOUND_ON_CLOUD); -- -20009
Errors(CODE_FILE_VALIDATION_FAILED) := Error_Record(CODE_FILE_VALIDATION_FAILED, MSG_FILE_VALIDATION_FAILED); -- -20010
Errors(CODE_NO_CONFIG_MATCH) := Error_Record(CODE_NO_CONFIG_MATCH, MSG_NO_CONFIG_MATCH); -- -20011
Errors(CODE_UNKNOWN_PREFIX) := Error_Record(CODE_UNKNOWN_PREFIX, MSG_UNKNOWN_PREFIX); -- -20012
Errors(CODE_TABLE_NOT_EXISTS) := Error_Record(CODE_TABLE_NOT_EXISTS, MSG_TABLE_NOT_EXISTS); -- -20013
Errors(CODE_COLUMN_NOT_EXISTS) := Error_Record(CODE_COLUMN_NOT_EXISTS, MSG_COLUMN_NOT_EXISTS); -- -20014
Errors(CODE_UNSUPPORTED_DATA_TYPE) := Error_Record(CODE_UNSUPPORTED_DATA_TYPE, MSG_UNSUPPORTED_DATA_TYPE); -- -20015
Errors(CODE_MISSING_SOURCE_KEY) := Error_Record(CODE_MISSING_SOURCE_KEY, MSG_MISSING_SOURCE_KEY); -- -20016
Errors(CODE_NULL_SOURCE_FILE_CONFIG_KEY) := Error_Record(CODE_NULL_SOURCE_FILE_CONFIG_KEY, MSG_NULL_SOURCE_FILE_CONFIG_KEY); -- -20017
Errors(CODE_DUPLICATED_SOURCE_KEY) := Error_Record(CODE_DUPLICATED_SOURCE_KEY, MSG_DUPLICATED_SOURCE_KEY); -- -20018
Errors(CODE_MISSING_CONTAINER_CONFIG) := Error_Record(CODE_MISSING_CONTAINER_CONFIG, MSG_MISSING_CONTAINER_CONFIG); -- -20019
Errors(CODE_MULTIPLE_CONTAINER_ENTRIES) := Error_Record(CODE_MULTIPLE_CONTAINER_ENTRIES, MSG_MULTIPLE_CONTAINER_ENTRIES); -- -20020
Errors(CODE_WRONG_DESTINATION_PARAM) := Error_Record(CODE_WRONG_DESTINATION_PARAM, MSG_WRONG_DESTINATION_PARAM); -- -20021
Errors(CODE_FILE_NOT_EXISTS_ON_CLOUD) := Error_Record(CODE_FILE_NOT_EXISTS_ON_CLOUD, MSG_FILE_NOT_EXISTS_ON_CLOUD); -- -20022
Errors(CODE_FILE_ALREADY_REGISTERED) := Error_Record(CODE_FILE_ALREADY_REGISTERED, MSG_FILE_ALREADY_REGISTERED); -- -20023
Errors(CODE_WRONG_DATE_TIMESTAMP_FORMAT) := Error_Record(CODE_WRONG_DATE_TIMESTAMP_FORMAT, MSG_WRONG_DATE_TIMESTAMP_FORMAT); -- -20024
Errors(CODE_ENVIRONMENT_NOT_SET) := Error_Record(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET); -- -20025
Errors(CODE_CONFIG_VARIABLE_NOT_SET) := Error_Record(CODE_CONFIG_VARIABLE_NOT_SET, MSG_CONFIG_VARIABLE_NOT_SET); -- -20026
Errors(CODE_NOT_INPUT_SOURCE_FILE_TYPE) := Error_Record(CODE_NOT_INPUT_SOURCE_FILE_TYPE, MSG_NOT_INPUT_SOURCE_FILE_TYPE); -- -20027
Errors(CODE_EXP_DATA_FOR_HIST_FAILED) := Error_Record(CODE_EXP_DATA_FOR_HIST_FAILED, MSG_EXP_DATA_FOR_HIST_FAILED); -- -20028
Errors(CODE_RESTORE_FILE_FROM_TRASH) := Error_Record(CODE_RESTORE_FILE_FROM_TRASH, MSG_RESTORE_FILE_FROM_TRASH); -- -20029
Errors(CODE_CHANGE_STAT_TO_HISTORISED_FAILED):= Error_Record(CODE_CHANGE_STAT_TO_HISTORISED_FAILED, MSG_CHANGE_STAT_TO_HISTORISED_FAILED); -- -20030
Errors(CODE_MOVE_FILE_TO_TRASH_FAILED) := Error_Record(CODE_MOVE_FILE_TO_TRASH_FAILED, MSG_MOVE_FILE_TO_TRASH_FAILED); -- -20031
Errors(CODE_DROP_EXPORTED_FILES_FAILED) := Error_Record(CODE_DROP_EXPORTED_FILES_FAILED, MSG_DROP_EXPORTED_FILES_FAILED); -- -20032
Errors(CODE_UNKNOWN) := Error_Record(CODE_UNKNOWN, MSG_UNKNOWN); -- -20999
END INIT_ERRORS;
----------------------------------------------------------------------------------------------------
FUNCTION GET_DEFAULT_ENV
RETURN VARCHAR2
IS
vDefaultEnv CT_MRDS.a_file_manager_config.config_variable_value%TYPE;
BEGIN
select config_variable_value
into vDefaultEnv
from CT_MRDS.a_file_manager_config
where lower(environment_id)='default'
and lower(config_variable)='environmentid';
RETURN vDefaultEnv;
EXCEPTION
WHEN NO_DATA_FOUND THEN
RETURN NULL;
END;
----------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------
PROCEDURE INIT_VARIABLES(
pEnv VARCHAR2
) IS
BEGIN
for rec in (
select
ENVIRONMENT_ID
,REGION
,NAMESPACE
,INBOXBUCKETNAME
,DATABUCKETNAME
,HISTORYBUCKETNAME
,CREDENTIALNAME
,LOGGINGENABLED
,MINLOGLEVEL
,DEFAULTDATEFORMAT
,CONSOLELOGGINGENABLED
from (
select environment_id, config_variable, config_variable_value from CT_MRDS.A_FILE_MANAGER_CONFIG
where environment_id=pEnv
)
pivot (
min(config_variable_value)
for config_variable in (
'Region' as Region
,'NameSpace' as NameSpace
,'InboxBucketName' as InboxBucketName
,'DataBucketName' as DataBucketName
,'HistoryBucketName' as HistoryBucketName
,'CredentialName' as CredentialName
,'LoggingEnabled' as LoggingEnabled
,'MinLogLevel' as MinLogLevel
,'DefaultDateFormat' as DefaultDateFormat
,'ConsoleLoggingEnabled' as ConsoleLoggingEnabled)
)
) loop
if (rec.NAMESPACE is NULL
or rec.REGION is NULL
or rec.NAMESPACE is NULL
or rec.INBOXBUCKETNAME is NULL
or rec.DATABUCKETNAME is NULL
or rec.HISTORYBUCKETNAME is NULL
or rec.CREDENTIALNAME is NULL
) THEN
MSG_CONFIG_VARIABLE_NOT_SET := MSG_CONFIG_VARIABLE_NOT_SET
||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': '
||cgBL||' '||'-------------------------'
||cgBL||' '||'Region = '||rec.Region
||cgBL||' '||'NameSpace = '||rec.NameSpace
||cgBL||' '||'InboxBucketName = '||rec.InboxBucketName
||cgBL||' '||'DataBucketName = '||rec.DataBucketName
||cgBL||' '||'HistoryBucketName = '||rec.HistoryBucketName
||cgBL||' '||'CredentialName = '||rec.CredentialName
;
LOG_PROCESS_EVENT(MSG_CONFIG_VARIABLE_NOT_SET, 'ERROR');
RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, MSG_CONFIG_VARIABLE_NOT_SET);
elsif (rec.LOGGINGENABLED is NULL
or rec.MINLOGLEVEL is NULL
or rec.DEFAULTDATEFORMAT is NULL
) THEN
vgMsgTmp := 'Missing configuration variables'
||cgBL||' '||'Details about existing Configuration Variables where environment_id='||pEnv||': '
||cgBL||' '||'-------------------------'
||cgBL||' '||'LoggingEnabled = '||rec.LoggingEnabled
||cgBL||' '||'MinLogLevel = '||rec.MinLogLevel
||cgBL||' '||'DefaultDateFormat = '||rec.DefaultDateFormat
;
LOG_PROCESS_EVENT(vgMsgTmp, 'WARNING');
else
gvNameSpace := rec.NAMESPACE;
gvRegion := rec.REGION;
gvInboxBucketName := rec.INBOXBUCKETNAME;
gvDataBucketName := rec.DATABUCKETNAME;
gvHistoryBucketName := rec.HISTORYBUCKETNAME;
gvCredentialName := rec.CREDENTIALNAME;
gvInboxBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.INBOXBUCKETNAME||'/o/';
gvDataBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.DATABUCKETNAME||'/o/';
gvHistoryBucketUri := 'https://objectstorage.'||rec.REGION||'.oraclecloud.com/n/'||rec.NAMESPACE||'/b/'||rec.HISTORYBUCKETNAME||'/o/';
gvLoggingEnabled := rec.LOGGINGENABLED;
gvMinLogLevel := rec.MINLOGLEVEL;
gvDefaultDateFormat := rec.DEFAULTDATEFORMAT;
gvConsoleLoggingEnabled := NVL(rec.CONSOLELOGGINGENABLED, 'ON');
end if;
end loop;
EXCEPTION
WHEN NO_DATA_FOUND THEN
MSG_CONFIG_VARIABLE_NOT_SET := MSG_CONFIG_VARIABLE_NOT_SET_C
||cgBL||' '||'No configuration found for environment_id='||pEnv||' in A_FILE_MANAGER_CONFIG table';
LOG_PROCESS_EVENT(MSG_CONFIG_VARIABLE_NOT_SET, 'ERROR', 'pEnv='||pEnv);
RAISE_APPLICATION_ERROR(CODE_CONFIG_VARIABLE_NOT_SET, MSG_CONFIG_VARIABLE_NOT_SET);
WHEN OTHERS THEN
vgMsgTmp := 'Unexpected error while initializing variables for environment: '||pEnv
||cgBL||' '||'SQLERRM: '||SQLERRM;
LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', 'pEnv='||pEnv);
RAISE;
END INIT_VARIABLES;
----------------------------------------------------------------------------------------------------
FUNCTION GET_ERROR_MESSAGE(
pCode PLS_INTEGER
) RETURN VARCHAR2
IS
BEGIN
RETURN Errors(pCode).message;
EXCEPTION
WHEN NO_DATA_FOUND THEN
LOG_PROCESS_EVENT('No error message found for pCode='||pCode , 'WARNING', 'pCode='||pCode);
LOG_PROCESS_EVENT('Update ENV_MANAGER package header with new code.' , 'WARNING', 'pCode='||pCode);
RETURN NULL;
WHEN OTHERS THEN
LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pCode='||pCode);
RAISE ERR_UNKNOWN;
END GET_ERROR_MESSAGE;
----------------------------------------------------------------------------------------------------
FUNCTION GET_ERROR_STACK(
pFormat VARCHAR2
,pCode PLS_INTEGER
,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL
) RETURN VARCHAR2
IS
vFullErrorCore VARCHAR2(32000);
vFullErrorMsg VARCHAR2(32000);
BEGIN
-- vgErrorMessage := SQLERRM|| cgBL;
-- vgErrorStack := DBMS_UTILITY.FORMAT_ERROR_STACK;
-- vgErrorBacktrace := DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
vFullErrorCore :='Error Message:'
||cgBL|| SQLERRM|| cgBL
||'-------------------------------------------------------'
||cgBL||'Error Stack:'
||cgBL|| DBMS_UTILITY.FORMAT_ERROR_STACK
||'-------------------------------------------------------'
||cgBL||'Error Backtrace:'
||cgBL|| DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;
-- vFullErrorCore := REGEXP_REPLACE (vFullErrorCore, pCode||': ', pCode||': '||GET_ERROR_MESSAGE(pCode) , 1, 1);
IF (pFormat = 'TABLE') THEN
vFullErrorMsg := vFullErrorCore;
ELSE
vFullErrorMsg := cgBL||'------------------------------------------------------+'
||cgBL||vFullErrorCore
||'------------------------------------------------------+';
END IF;
-- IF pSourceFileReceivedKey is not null THEN
-- vFullErrorMsg := vFullErrorMsg ||cgBL||GET_DET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey,1,1,1);
-- END IF;
RETURN vFullErrorMsg;
EXCEPTION
WHEN OTHERS THEN
LOG_PROCESS_EVENT(MSG_UNKNOWN , 'ERROR', 'pFormat='||pFormat);
RETURN NULL;
END GET_ERROR_STACK;
----------------------------------------------------------------------------------------------------
FUNCTION FORMAT_PARAMETERS(
pParameterList SYS.ODCIVARCHAR2LIST
) RETURN VARCHAR2 IS
vResult VARCHAR2(10000);
BEGIN
FOR i IN 1 .. pParameterList.COUNT LOOP
-- dbms_output.put_line('pParameterList(i): '||pParameterList(i));
if i < pParameterList.COUNT then vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL') ||' ,'|| cgBL;
else vResult := vResult || replace(pParameterList(i), '''NULL''', 'NULL');
end if;
END LOOP;
RETURN vResult;
EXCEPTION
WHEN OTHERS THEN
LOG_PROCESS_EVENT('Error while formating parameters.' , 'WARNING');
RETURN NULL;
END FORMAT_PARAMETERS;
----------------------------------------------------------------------------------------------------
PROCEDURE LOG_PROCESS_EVENT (
pLogMessage VARCHAR2
,pLogLevel VARCHAR2 DEFAULT 'ERROR'
,pParameters VARCHAR2 DEFAULT NULL
,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER'
) IS
PRAGMA AUTONOMOUS_TRANSACTION;
vLoggingEnabled VARCHAR2(10);
vMinLogLevel VARCHAR2(10);
vCallStack VARCHAR2(10000);
vProcedureName VARCHAR2(100);
vProcedureLevel PLS_INTEGER;
vTotalLines PLS_INTEGER;
vCurrentLine PLS_INTEGER;
-- Map of priority level
TYPE logLevelMap IS TABLE OF NUMBER INDEX BY VARCHAR2(10);
vLogLevels logLevelMap;
BEGIN
-- Prority logging level (higher -> more important)
vLogLevels('DEBUG') := 1;
vLogLevels('INFO') := 2;
vLogLevels('WARNING') := 3;
vLogLevels('ERROR') := 4;
-- Check id logging is TURN-OFF
IF gvLoggingEnabled = 'OFF' THEN
RETURN;
END IF;
-- Check logging level
IF vLogLevels(pLogLevel) < vLogLevels(gvMinLogLevel) THEN
RETURN;
END IF;
vCallStack := DBMS_UTILITY.FORMAT_CALL_STACK;
vProcedureName := REGEXP_SUBSTR(vCallStack, 'package body\s+\w+\.(\w+\.\w+)', 1, 2, NULL, 1);
vTotalLines := REGEXP_COUNT(vCallStack, CHR(10)) + 1;
vCurrentLine := REGEXP_COUNT(SUBSTR(vCallStack, 1, INSTR(vCallStack, vProcedureName) - 1), CHR(10)) + 1;
vProcedureLevel := (vTotalLines - vCurrentLine + 1) - 3;
vProcedureName := LPAD(vProcedureName, LENGTH(vProcedureName) + 2*vProcedureLevel, ' ');
INSERT INTO CT_MRDS.A_PROCESS_LOG (guid, username, osuser, machine, module, process_name, procedure_name, procedure_parameters, log_level, log_message)
VALUES (guid, gvUsername, gvOsuser, gvMachine, gvModule, pProcessName, vProcedureName, pParameters, pLogLevel, pLogMessage);
COMMIT;
-- Also output to console for immediate visibility (if enabled)
IF gvConsoleLoggingEnabled = 'ON' THEN
DBMS_OUTPUT.PUT_LINE('[' || TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') || '] [' || pLogLevel || '] ' || vProcedureName || ': ' || pLogMessage);
END IF;
END LOG_PROCESS_EVENT;
----------------------------------------------------------------------------------------------------
BEGIN
INIT_ERRORS;
guid := sys_guid();
gvUsername := SYS_CONTEXT('USERENV', 'SESSION_USER');
gvOsuser := SYS_CONTEXT('USERENV', 'OS_USER');
gvMachine := SYS_CONTEXT('USERENV', 'HOST');
gvModule := SYS_CONTEXT('USERENV', 'MODULE');
-- Get info about EnvironmentID. Without it package cannot proceed further.
-- Information about environment is needed to get proper configuration values
-- It can be set up in two different ways :
-- 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => 'dev');
-- 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values ('default','environment_id','dev');
-- Session level setup (1.) takes precedence over configuration level one (2.)
gvEnv := nvl(SYS_CONTEXT ('USERENV', 'CLIENT_IDENTIFIER'), GET_DEFAULT_ENV());
if gvEnv is null then
dbms_output.put_line(MSG_ENVIRONMENT_NOT_SET);
LOG_PROCESS_EVENT(MSG_ENVIRONMENT_NOT_SET, 'ERROR');
RAISE_APPLICATION_ERROR(CODE_ENVIRONMENT_NOT_SET, MSG_ENVIRONMENT_NOT_SET);
else
dbms_output.put_line('EnvironmentID set to: '||gvEnv);
end if;
INIT_VARIABLES(pEnv => gvEnv);
END ENV_MANAGER;
/

178
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_packages_2.sql Normal file
View File

@@ -0,0 +1,178 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_packages_2_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload --CT_MRDS.WORKFLOW_MANAGER package
--=============================================================================================================================
CREATE OR REPLACE EDITIONABLE PACKAGE "CT_MRDS"."WORKFLOW_MANAGER"
IS
FUNCTION INIT_WORKFLOW(pServiceName IN VARCHAR2, pWorkflowRunId IN VARCHAR2, pWorkflowName in VARCHAR2)
RETURN NUMBER;
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pWorkflowStatus IN VARCHAR2);
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pServiceName IN VARCHAR2, pWorkflowStatus IN VARCHAR2);
FUNCTION INIT_TASK(pTaskRunId IN VARCHAR2, pTaskName in VARCHAR2, pWorkflowHistoryKey IN NUMBER)
RETURN NUMBER;
PROCEDURE FINALISE_TASK(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2);
PROCEDURE STORE_TASK_SOURCE_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2, pNumRows IN NUMBER);
PROCEDURE STORE_TASK_TARGET_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2,
pNumRowsApplied IN NUMBER, pNumRowsRejected IN NUMBER);
vpRunningStatus CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL%TYPE := 'Running';
--
-- Set and get information on workflow level
--
PROCEDURE SET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
,pValue IN VARCHAR2
);
FUNCTION GET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2)
RETURN VARCHAR2;
END WORKFLOW_MANAGER;
/
CREATE OR REPLACE EDITIONABLE PACKAGE BODY "CT_MRDS"."WORKFLOW_MANAGER"
IS
FUNCTION INIT_WORKFLOW(pServiceName IN VARCHAR2, pWorkflowRunId IN VARCHAR2, pWorkflowName in VARCHAR2)
RETURN NUMBER
IS
vWorkflowHistoryKey NUMBER;
BEGIN
vWorkflowHistoryKey := A_WORKFLOW_HISTORY_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_WORKFLOW_HISTORY (SERVICE_NAME, A_WORKFLOW_HISTORY_KEY, ORCHESTRATION_RUN_ID,
WORKFLOW_NAME, WORKFLOW_START, WORKFLOW_SUCCESSFUL)
VALUES (pServiceName, vWorkflowHistoryKey, pWorkflowRunId,
pWorkflowName, SYSTIMESTAMP, vpRunningStatus);
return vWorkflowHistoryKey;
END INIT_WORKFLOW;
--
-- Overload without service name for backward compatability, to be cleaned up later
--
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pWorkflowStatus IN VARCHAR2)
IS
BEGIN
FINALISE_WORKFLOW(pWorkflowHistoryKey, NULL, pWorkflowStatus);
END;
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pServiceName IN VARCHAR2, pWorkflowStatus IN VARCHAR2)
IS
BEGIN
UPDATE CT_MRDS.A_WORKFLOW_HISTORY SET WORKFLOW_SUCCESSFUL = pWorkflowStatus,
WORKFLOW_END = SYSTIMESTAMP
WHERE A_WORKFLOW_HISTORY_KEY = pWorkflowHistoryKey
AND SERVICE_NAME = NVL(pServiceName,SERVICE_NAME);
END FINALISE_WORKFLOW;
FUNCTION INIT_TASK(pTaskRunId IN VARCHAR2, pTaskName in VARCHAR2, pWorkflowHistoryKey IN NUMBER)
RETURN NUMBER
IS
vTaskHistoryKey NUMBER;
BEGIN
vTaskHistoryKey := A_TASK_HISTORY_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_TASK_HISTORY (A_TASK_HISTORY_KEY, TASK_RUN_ID, A_WORKFLOW_HISTORY_KEY,
TASK_NAME, TASK_START, TASK_SUCCESSFUL, SERVICE_NAME)
VALUES (vTaskHistoryKey, pTaskRunId, pWorkflowHistoryKey,
pTaskName, SYSTIMESTAMP, vpRunningStatus, 'ODS');
return vTaskHistoryKey;
END INIT_TASK;
PROCEDURE FINALISE_TASK(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2)
IS
BEGIN
UPDATE CT_MRDS.A_TASK_HISTORY SET TASK_SUCCESSFUL = pTaskStatus,
TASK_END = SYSTIMESTAMP
WHERE A_TASK_HISTORY_KEY = pTaskHistoryKey;
END FINALISE_TASK;
PROCEDURE STORE_TASK_SOURCE_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2, pNumRows IN NUMBER)
IS
BEGIN
INSERT INTO CT_MRDS.A_TASK_HISTORY_SOURCE (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY,
SOURCE_NAME, ROW_COUNT)
VALUES (A_TASK_HISTORY_SOURCE_KEY_SEQ.NEXTVAL, pTaskHistoryKey,
pSourceName, pNumRows);
COMMIT;
END;
PROCEDURE STORE_TASK_TARGET_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2,
pNumRowsApplied IN NUMBER, pNumRowsRejected IN NUMBER)
IS
BEGIN
INSERT INTO CT_MRDS.A_TASK_HISTORY_TARGET (A_TASK_HISTORY_TARGET_KEY, A_TASK_HISTORY_KEY,
TARGET_NAME, ROW_COUNT_APPLIED, ROW_COUNT_REJECTED)
VALUES (A_TASK_HISTORY_TARGET_KEY_SEQ.NEXTVAL, pTaskHistoryKey,
pSourceName, pNumRowsApplied, pNumRowsRejected);
COMMIT;
END;
PROCEDURE SET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
,pValue IN VARCHAR2
) IS
BEGIN
INSERT INTO CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME, PROPERTY, VALUE)
VALUES (pWorkflowHistoryKey, pServiceName, pProperty, pValue);
END;
FUNCTION GET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
) RETURN VARCHAR2
IS
vValue CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY.VALUE%TYPE;
BEGIN
SELECT VALUE
INTO vValue
FROM CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY
WHERE A_WORKFLOW_HISTORY_KEY = pWorkflowHistoryKey
AND SERVICE_NAME = pServiceName
AND PROPERTY = pProperty;
RETURN vValue;
END;
END WORKFLOW_MANAGER;

2851
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_packages_3.sql Normal file
View File

File diff suppressed because it is too large Load Diff

475
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_packages_4.sql Normal file
View File

@@ -0,0 +1,475 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_packages_4_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload --CT_MRDS.FILE_ARCHIVER package
--=============================================================================================================================
CREATE OR REPLACE EDITIONABLE PACKAGE "CT_MRDS"."FILE_ARCHIVER"
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
/**
* @name HISTORISE_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
**/
PROCEDURE HISTORISE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
);
/**
* @name GATHER_TABLE_STAT
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
**/
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
);
END;
/
CREATE OR REPLACE EDITIONABLE PACKAGE BODY "CT_MRDS"."FILE_ARCHIVER"
AS
----------------------------------------------------------------------------------------------------
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
IS
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vCount PLS_INTEGER;
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT count(*) , min(SOURCE_FILE_TYPE)
INTO vCount, vSourceFileType
FROM CT_MRDS.A_TABLE_STAT s
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
IF vCount=0 and vSourceFileType='INPUT' THEN
GATHER_TABLE_STAT(pSourceFileConfigKey);
END IF;
BEGIN
SELECT *
INTO vTableStat
FROM CT_MRDS.A_TABLE_STAT
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- EXCEPTION
-- WHEN NO_DATA_FOUND THEN
--
END;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vTableStat;
END GET_TABLE_STAT;
----------------------------------------------------------------------------------------------------
PROCEDURE HISTORISE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vQuery VARCHAR2(4000);
vTableName VARCHAR2(200);
vUri VARCHAR2(1000);
vfiles T_FILENAMES;
vFilename VARCHAR2(300);
vOperationId NUMBER := -1;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vHistorisationTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
vProcessControlStatus VARCHAR2(60) := 'OK';
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
end if;
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
end if;
if vTableStat.OVER_HIST_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_HISTORY_THRESHOLD then vHistorisationTriggeredBy := 'FILES_COUNT';
elsif vTableStat.OVER_HIST_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_HISTORY_THRESHOLD then vHistorisationTriggeredBy := vHistorisationTriggeredBy||', ROWS_COUNT';
elsif vTableStat.OVER_HIST_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_HISTORY_THRESHOLD then vHistorisationTriggeredBy := vHistorisationTriggeredBy||', BYTES_SUM';
else ENV_MANAGER.LOG_PROCESS_EVENT('Non of historisation triggers reached','INFO');
end if;
if LENGTH(vHistorisationTriggeredBy)>0 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Historisation Triggered By: '||vHistorisationTriggeredBy,'INFO');
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
vQuery := '
select t_filename(
file$name
,file$path
, to_char(h.workflow_start,''yyyy'')
, to_char(h.workflow_start,''mm'')
)
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
where extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD
;
-- Get all files that will be historised into "vfiles" collection ("regular data files")
execute immediate vQuery bulk collect into vfiles;
-- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
vQuery:=
'select
s.*
-- ,r.partition_year
-- ,r.partition_month
from '|| vTableName ||' s
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
on s.file$name = r.source_file_name
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
and r.partition_year='''||ym_loop.year||'''
and r.partition_month='''||ym_loop.month||'''
and r.PROCESSING_STATUS = ''INGESTED''
'
;
vUri := FILE_MANAGER.GET_BUCKET_URI('HISTORY')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/YEAR='||ym_loop.year||'/MONTH='||ym_loop.month||'/';
ENV_MANAGER.LOG_PROCESS_EVENT('Start Historization for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
BEGIN
DBMS_CLOUD.EXPORT_DATA(
credential_name => ENV_MANAGER.gvCredentialName,
file_uri_list => vUri||'d' ,
format => json_object('type' value 'parquet'),
query => vQuery,
operation_id => vOperationId
);
EXCEPTION
WHEN OTHERS THEN
vProcessControlStatus :='EXPORT_FAILURE';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);
END;
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
-- Get USER_LOAD_OPERATIONS info
select *
into vUserLoadOperations
from USER_LOAD_OPERATIONS
where id = vOperationId;
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED := ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED
||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS;
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED := ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED
||cgBL|| ' Zero rows were exported.';
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to historization file for YEAR_MONTH: 2025_01','INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
END IF;
SELECT
object_name
into vFilename
from DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vUri)
where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7'))
between vUserLoadOperations.START_TIME
and vUserLoadOperations.UPDATE_TIME
;
-- Try to drop EXPORTED FILES ("regular data files")
BEGIN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
-- first change of status
BEGIN
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'HISTORISED'
,HIST_FILE_NAME = vUri||vFilename
WHERE r.a_source_file_config_key= pSourceFileConfigKey
AND r.source_file_name = f.filename
AND r.processing_status = 'INGESTED'
;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'CHANGE_STATUS_TO_HISTORISED_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_HISTORISED_FAILURE';
-- move file to trash before dropping
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => f.pathname||'/'||f.filename,
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
rollback;
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
commit;
END LOOP;
--------------------------------------------------------------------
-- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
IF vProcessControlStatus = 'OK' THEN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
--Drop file from TRASH
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
END LOOP;
--ROLLBACK PART
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
FOR f in ( SELECT vf.filename, vf.pathname
FROM TABLE(vfiles) vf
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
ON r.source_file_name = vf.filename
AND r.a_source_file_config_key = pSourceFileConfigKey
AND r.PROCESSING_STATUS = 'HISTORISED'
AND vf.year = ym_loop.year
AND vf.month = ym_loop.month
) LOOP
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_object_uri => f.pathname||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'INGESTED'
,HIST_FILE_NAME = NULL
WHERE r.a_source_file_config_key = pSourceFileConfigKey
AND r.source_file_name = f.filename
;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
END;
END LOOP;
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => vUri||vFilename);
ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Historization PARQUET file dropped.','DEBUG', vUri||vFilename);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_HISTORISED_FAILURE' THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED, 'ERROR', vParameters);
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => vUri||vFilename);
ENV_MANAGER.LOG_PROCESS_EVENT('Historization PARQUET file dropped.','DEBUG', vUri||vFilename);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED);
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
END IF;
EXCEPTION
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_HISTORISED_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED);
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error during historization process','ERROR');
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
END;
-- END of "Try to drop EXPORTED FILES"
ENV_MANAGER.LOG_PROCESS_EVENT('All historized files had been dropped.','INFO');
ENV_MANAGER.LOG_PROCESS_EVENT('End Historization for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
END LOOP; --ym_loop end (YEAR_MONTH)
COMMIT;
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Non of historization thresholds reached. Skip historization.'||vHistorisationTriggeredBy,'INFO');
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_HIST_FAILED THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_HISTORISED_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END HISTORISE_TABLE_DATA;
----------------------------------------------------------------------------------------------------
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableName VARCHAR2(200);
vQuery VARCHAR2(32000);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
vQuery :=
'with tmp as (
select
s.*
,file$name as filename
,h.workflow_start
, to_char(h.workflow_start,''yyyy'') as year
, to_char(h.workflow_start,''mm'') as month
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
)
, tmp_gr as (
select
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
from tmp
group by filename
)
select
NULL as A_TABLE_STAT_KEY
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
,'''||vTableName||''' as TABLE_NAME
,count(*) as FILE_COUNT
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then 1 else 0 end) as OLD_FILE_COUNT
,sum (row_count_per_file) as ROW_COUNT
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then row_count_per_file else 0 end) as OLD_ROW_COUNT
,sum(r.bytes) as BYTES
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then r.bytes else 0 end) as OLD_BYTES
,'||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' as DAYS_FOR_HISTORY_THRESHOLD
,systimestamp as CREATED
from tmp_gr t
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
credential_name => '''||ENV_MANAGER.gvCredentialName||''',
location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/''
)
) r
on t.filename = r.object_name'
;
ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
execute immediate vQuery into vStats;
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
insert into A_TABLE_STAT_HIST values vStats;
delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
insert into A_TABLE_STAT values vStats;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GATHER_TABLE_STAT;
----------------------------------------------------------------------------------------------------
END;

25
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_packages_5.sql Normal file
View File

@@ -0,0 +1,25 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_packages_4_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload --CT_MRDS.FILE_ARCHIVER package
--=============================================================================================================================

36
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_prd.sql Normal file
View File

@@ -0,0 +1,36 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_prd_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload prd specific part
--=============================================================================================================================
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('default', 'EnvironmentID', 'prd', '');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'Region', 'eu-frankfurt-1', 'OCI region for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'NameSpace', 'frcnomajoc7v', 'OCI namespace for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'InboxBucketName', 'mrds_inbox_prd', 'Name of the inbox bucket for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'DataBucketName', 'mrds_data_prd', 'Name of the data bucket for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'HistoryBucketName', 'mrds_hist_prd', 'Name of the history bucket for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'CredentialName', 'OCI$RESOURCE_PRINCIPAL', 'Name of the credential for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'LoggingEnabled', 'ON', 'Enable logging for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'MinLogLevel', 'DEBUG', 'Minimum logging level for prd environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('prd', 'DefaultDateFormat', 'DD/MM/YYYY HH24:MI:SS', 'Default date format for prd environment');
COMMIT;

250
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_tables.sql Normal file
View File

@@ -0,0 +1,250 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_tables_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload
--=============================================================================================================================
CREATE TABLE "CT_MRDS"."A_PROCESS_LOG" (
a_process_log_key NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
guid VARCHAR2(32),
Username VARCHAR2(128),
Osuser VARCHAR2(128),
Machine VARCHAR2(64),
Module VARCHAR2(64),
process_name VARCHAR2(200),
procedure_name VARCHAR2(200),
procedure_parameters VARCHAR2(20000),
log_level VARCHAR2(10), -- DEBUG, INFO, WARNING, ERROR
log_message VARCHAR2(4000),
log_timestamp TIMESTAMP DEFAULT SYSTIMESTAMP
);
CREATE TABLE "CT_MRDS"."A_CASPER_FILEVAULT" (
"A_WORKFLOW_HISTORY_KEY" NUMBER(28,0),
"WLA_RUN_ID" NUMBER(28,0),
"WORKFLOW_NAME" CHAR(16 BYTE),
"FILENAME" VARCHAR2(250 CHAR),
"STATUS" VARCHAR2(30 CHAR),
"FILESIZE" NUMBER(28,0),
"FILE_ID" NUMBER(28,0),
"IS_TEST" VARCHAR2(5 CHAR),
"REFERENCE_TIME" TIMESTAMP (6)
);
CREATE TABLE "CT_MRDS"."A_TASK_HISTORY" (
"A_TASK_HISTORY_KEY" NUMBER(38,0),
"A_WORKFLOW_HISTORY_KEY" NUMBER(38,0),
"TASK_RUN_ID" VARCHAR2(255 BYTE),
"TASK_NAME" VARCHAR2(255 BYTE),
"TASK_START" TIMESTAMP (6),
"TASK_END" TIMESTAMP (6),
"TASK_SUCCESSFUL" VARCHAR2(200 BYTE),
"SERVICE_NAME" VARCHAR2(25 BYTE)
);
COMMENT ON TABLE CT_MRDS.A_TASK_HISTORY IS 'This table logs subprocesses and their execution details.';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.A_TASK_HISTORY_KEY IS 'Surrogate key for logged subprocesses';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.A_WORKFLOW_HISTORY_KEY IS 'Technical key to identify a load operation. Generated with Oracle sequence "CT_MRDS.SA_TASK_HISTORY_KEY_SEQ"';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_RUN_ID IS 'Unique identifier for the task run';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_NAME IS 'Mapping name of the logged subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_START IS 'Start time of the subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_END IS 'End time of the subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_SUCCESSFUL IS 'Indicates that the subprocess completed successfully. Allowed values are ''RUNNING'', ''FAILED'', and ''SUCCESS''.';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.SERVICE_NAME IS 'Database of reference';
CREATE TABLE "CT_MRDS"."A_TASK_HISTORY_SOURCE" (
"A_TASK_HISTORY_SOURCE_KEY" NUMBER(38,0),
"A_TASK_HISTORY_KEY" NUMBER(38,0),
"A_WORKFLOW_HISTORY_SOURCE_KEY" NUMBER(38,0),
"SOURCE_NAME" VARCHAR2(255 BYTE),
"ROW_COUNT" NUMBER(38,0),
"SERVICE_NAME" VARCHAR2(25 BYTE)
);
COMMENT ON TABLE CT_MRDS.A_TASK_HISTORY_SOURCE IS 'This table logs details of data sources used in load processes.';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.A_TASK_HISTORY_SOURCE_KEY IS 'Technical primary key';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.A_TASK_HISTORY_KEY IS 'Foreign key referencing the subprocess delivering to the target';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.A_WORKFLOW_HISTORY_SOURCE_KEY IS 'Technical key for the workflow history source';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.SOURCE_NAME IS 'Name of the source used in the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.ROW_COUNT IS 'Number of rows read from the source during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.SERVICE_NAME IS 'Database of reference';
CREATE TABLE "CT_MRDS"."A_TASK_HISTORY_TARGET" (
"A_TASK_HISTORY_TARGET_KEY" NUMBER(38,0),
"A_TASK_HISTORY_KEY" NUMBER(38,0),
"TARGET_NAME" VARCHAR2(255 BYTE),
"ROW_COUNT_APPLIED" NUMBER(38,0),
"ROW_COUNT_REJECTED" NUMBER(38,0),
"LOAD_SUCCESSFUL" CHAR(1 CHAR),
"SERVICE_NAME" VARCHAR2(25 BYTE)
);
COMMENT ON TABLE CT_MRDS.A_TASK_HISTORY_TARGET IS 'This table logs details of target data written during load processes.';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.A_TASK_HISTORY_TARGET_KEY IS 'Technical primary key';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.A_TASK_HISTORY_KEY IS 'Foreign key referencing the subprocess delivering to the target';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.TARGET_NAME IS 'Name of the target where data is written during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.ROW_COUNT_APPLIED IS 'Number of rows successfully applied (written to) the DWH target table during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.ROW_COUNT_REJECTED IS 'Number of rows rejected (NOT written to) the DWH target table during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.LOAD_SUCCESSFUL IS 'Indicates the success of the load process. Allowed values are ''RUNNING'', ''SUCCESS'', and ''FAILED''.';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.SERVICE_NAME IS 'Database of reference';
CREATE TABLE "CT_MRDS"."A_WORKFLOW_HISTORY" (
"A_WORKFLOW_HISTORY_KEY" NUMBER(38,0),
"ORCHESTRATION_RUN_ID" VARCHAR2(300 BYTE),
"WORKFLOW_NAME" VARCHAR2(255 BYTE),
"WORKFLOW_START" TIMESTAMP (6),
"WORKFLOW_END" TIMESTAMP (6),
"WORKFLOW_SUCCESSFUL" VARCHAR2(200 BYTE),
"DBT_INVOCATION_ID" VARCHAR2(200 BYTE),
"SERVICE_NAME" VARCHAR2(25 BYTE)
);
COMMENT ON TABLE CT_MRDS.A_WORKFLOW_HISTORY IS 'This table logs details of workflow executions, including start and end times, and overall workflow status.';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY IS 'Technical key to identify a load operation. Generated with Oracle sequence "CT_MRDS.SEQ_WORKFLOW_HISTORY"';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.ORCHESTRATION_RUN_ID IS 'Unique identifier for the orchestration run';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_NAME IS 'The name of the workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_START IS 'Start time of the Airflow / Informatica workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_END IS 'End time of the Airflow / Informatica workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL IS 'Overall status of the workflow. Allowed values are ''R'' (running), ''Y'' (yes), and ''N'' (no)';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.DBT_INVOCATION_ID IS 'Unique identifier for the DBT invocation';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.SERVICE_NAME IS 'Filter for the service: RAR, MOPDB, ODS';
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_MOPDB" AS
SELECT OWNER,
TABLE_NAME,
TABLE_ALIAS,
CAST(NULL AS VARCHAR2(200)) AS LAST_RUN_ID,
CAST(NULL AS DATE) AS LAST_START_TIME,
CAST(NULL AS DATE) AS LAST_END_TIME,
CAST(NULL AS VARCHAR2(200)) AS LAST_STATUS,
SAVE_MODE,
BUCKET_COLUMN AS PARTITION_COLUMN,
NUMBER_OF_BUCKETS AS MAX_THREADS,
CAST(NULL AS NUMBER(28,10)) AS LAST_PROCESSED_KEY
FROM CT_MOPDB.MOPDB_ON_DISC_REPLICA_MGMT;
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_RAR" AS
SELECT OWNER,
TABLE_NAME,
TABLE_ALIAS,
CAST(NULL AS VARCHAR2(200)) AS LAST_RUN_ID,
CAST(NULL AS DATE) AS LAST_START_TIME,
CAST(NULL AS DATE) AS LAST_END_TIME,
CAST(NULL AS VARCHAR2(200)) AS LAST_STATUS,
SAVE_MODE,
BUCKET_COLUMN AS PARTITION_COLUMN,
NUMBER_OF_BUCKETS AS MAX_THREADS,
CAST(NULL AS NUMBER(28,10)) AS LAST_PROCESSED_KEY
FROM CT_REF.RAR_ON_DISC_REPLICA_MGMT;
CREATE TABLE "CT_MRDS"."ERR_LOG"
("ERR_LOG_KEY" NUMBER(*,0),
"A_WORKFLOW_HISTORY_KEY" NUMBER(38,0),
"ERR_MESSAGE_FK" NUMBER,
"ERR_SEVERITY_FK" NUMBER,
"ERR_WORKFLOW_RUN_ID" CHAR(17 BYTE) COLLATE "USING_NLS_COMP",
"ERR_WORKFLOW_NAME" CHAR(18 BYTE) COLLATE "USING_NLS_COMP",
"ERR_MAPPING_NAME" CHAR(74 BYTE) COLLATE "USING_NLS_COMP",
"ERR_LOCATION" CHAR(74 BYTE) COLLATE "USING_NLS_COMP",
"ERR_SOURCE" CHAR(22 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_BUSINESS" CHAR(6 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_BUSINESS_VALUE" CHAR(14 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_TECHNICAL" CHAR(22 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_TECHNICAL_VALUE" VARCHAR2(40 BYTE) COLLATE "USING_NLS_COMP",
"ERR_DESCRIPTION" CHAR(94 BYTE) COLLATE "USING_NLS_COMP",
"ERR_MESSAGE" VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
"INDEP_SUBPROCESS_FK" NUMBER(38,0)
);
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_CEPH';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'MPEC' AND TABLE_NAME = 'T_MPEC';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CAPITAL_KEY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'CT_MOPDB' AND TABLE_NAME = 'MOPDB_METADATA_INVENTORY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CSDB_RATING_ACTION';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CURRENCY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'CLSLNK_SNPSHT_DT' WHERE OWNER = 'RIAD' AND TABLE_NAME = 'T_CLSLNK_PSTNZ_HIST';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_AGGREGATED_ALLOTMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'ELA' AND TABLE_NAME = 'T_ELA_INFO';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_ORIGINATOR';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_ADD_GUARANTOR';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_IAB';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_SERVICER';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_LSP';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_IRSP';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_FXSC';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_COLLATERAL';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_COUNTRY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_SNAPSHOT_DATE';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT_MODIFICATIONS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ANNOUNCEMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_FBL_ITEM';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_FULLBID_ARRAY_COMPILED';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REF_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_STANDING_FACILITIES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REFERENCE_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_CURRENT_ACCOUNTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REF_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_BALANCE_SHEET';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REFERENCE_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FORECAST';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'ADJUSTMENT_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_ADH_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_CSM_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_QRE_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_COLLATERAL_ALL_SOURCES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'TIME_PERIOD' WHERE OWNER = 'SDW' AND TABLE_NAME = 'T_EXCHANGE_RATES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT_MODIFICATIONS_AGGREGATED';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_INTRADAY_CREDIT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'MV_PUBLIC_ELIGIBLE_MARKETABLE_ASSETS';
commit;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR
SET SAVE_MODE = 'OVERWRITE' , PARTITION_COLUMN = 'ROWID';
commit;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_SNAPSHOT_DATE' WHERE TABLE_NAME = 'NH_HF_PRICE' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_PRICE' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_POSITION' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_PERFORMANCE' AND OWNER = 'CORR_RAR' ;
commit;
---------------------
------ SEQUENCES ----
---------------------
DECLARE
start_val NUMBER; -- Variable to hold the maximum value from the table
BEGIN
-- Create A_TASK_HISTORY_KEY_SEQ
EXECUTE IMMEDIATE 'CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_KEY_SEQ MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 10000000 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL';
-- Create A_TASK_HISTORY_SOURCE_KEY_SEQ
EXECUTE IMMEDIATE 'CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_SOURCE_KEY_SEQ MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 10000000 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL';
-- Create A_TASK_HISTORY_TARGET_KEY_SEQ
EXECUTE IMMEDIATE 'CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_TARGET_KEY_SEQ MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 10000000 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL';
-- Create A_WORKFLOW_HISTORY_KEY_SEQ
EXECUTE IMMEDIATE 'CREATE SEQUENCE CT_MRDS.A_WORKFLOW_HISTORY_KEY_SEQ MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 10000000 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL';
END;

59
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_templates.sql Normal file
View File

@@ -0,0 +1,59 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_templates_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload
--=============================================================================================================================
CREATE USER CT_ET_TEMPLATES IDENTIFIED BY "afa3343DDD___"
DEFAULT TABLESPACE DATA;
--DROP TABLE CT_ET_TEMPLATES.IDS_DATA;
CREATE TABLE CT_ET_TEMPLATES.IDS_DATA(
"STATION" VARCHAR2(2000),
"DATE" VARCHAR2(2000),
"LATITUDE" VARCHAR2(2000),
"LONGITUDE" VARCHAR2(2000),
"ELEVATION" VARCHAR2(2000),
"NAME" VARCHAR2(2000),
"TEMP" VARCHAR2(2000),
"TEMP_ATTRIBUTES" VARCHAR2(2000),
"DEWP" VARCHAR2(2000),
"DEWP_ATTRIBUTES" VARCHAR2(2000),
"SLP" VARCHAR2(2000),
"SLP_ATTRIBUTES" VARCHAR2(2000),
"STP" VARCHAR2(2000),
"STP_ATTRIBUTES" VARCHAR2(2000),
"VISIB" VARCHAR2(2000),
"VISIB_ATTRIBUTES" VARCHAR2(2000),
"WDSP" VARCHAR2(2000),
"WDSP_ATTRIBUTES" VARCHAR2(2000),
"MXSPD" VARCHAR2(2000),
"GUST" VARCHAR2(2000),
"MAX" VARCHAR2(2000),
"MAX_ATTRIBUTES" VARCHAR2(2000),
"MIN" VARCHAR2(2000),
"MIN_ATTRIBUTES" VARCHAR2(2000),
"PRCP" VARCHAR2(2000),
"PRCP_ATTRIBUTES" VARCHAR2(2000),
"SNDP" VARCHAR2(2000),
"FRSHTT" VARCHAR2(2000)
);

95
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_triggers.sql Normal file
View File

@@ -0,0 +1,95 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_triggers_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 triggers
--=============================================================================================================================
CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BIU_CHCK_TEMPLATE_TABLE_NAME
BEFORE INSERT OR UPDATE ON "CT_MRDS"."A_COLUMN_DATE_FORMAT"
FOR EACH ROW
DECLARE
vCount NUMBER;
BEGIN
SELECT COUNT(*)
INTO vCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE TEMPLATE_TABLE_NAME = :NEW.TEMPLATE_TABLE_NAME;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(-20001, 'TEMPLATE_TABLE_NAME = '''||:NEW.TEMPLATE_TABLE_NAME||''' not exists in A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME ');
END IF;
END;
/
CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BI_A_SOURCE_FILE_CONFIG_CHECK
BEFORE INSERT ON CT_MRDS.A_SOURCE_FILE_CONFIG
FOR EACH ROW
DECLARE
-- NO_CONTAINER_FOR_INPUT EXCEPTION;
WRONG_CONTAINER_FOR_INPUT EXCEPTION;
CONTAINER_ALREADY_EXISTS EXCEPTION;
vCount PLS_INTEGER;
vContainerFileKey CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE;
BEGIN
--Check if there is CONTAINER entity in A_SOURCE_FILE_CONFIG table
IF (:NEW.SOURCE_FILE_TYPE = 'INPUT' AND :NEW.CONTAINER_FILE_KEY IS NULL) THEN
:NEW.CONTAINER_FILE_KEY := CT_MRDS.FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(:NEW.SOURCE_FILE_ID);
-- IF :NEW.CONTAINER_FILE_KEY is null THEN
-- RAISE NO_CONTAINER_FOR_INPUT;
-- END IF;
-- Check if provided CONTAINER_FILE_KEY is the one which exists in the A_SOURCE_FILE_CONFIG table
ELSIF (:NEW.SOURCE_FILE_TYPE = 'INPUT' AND :NEW.CONTAINER_FILE_KEY IS NOT NULL) THEN
vContainerFileKey := CT_MRDS.FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(:NEW.SOURCE_FILE_ID);
IF :NEW.CONTAINER_FILE_KEY <> vContainerFileKey THEN
RAISE WRONG_CONTAINER_FOR_INPUT;
END IF;
-- Check if there is already a CONTAINER for provided SOURCE_FILE_ID
ELSIF (:NEW.SOURCE_FILE_TYPE = 'CONTAINER') THEN
SELECT count(*)
INTO vCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE :NEW.SOURCE_FILE_TYPE = 'CONTAINER'
AND SOURCE_FILE_ID = :NEW.SOURCE_FILE_ID;
IF vCount > 0 THEN
RAISE CONTAINER_ALREADY_EXISTS;
END IF;
END IF;
SELECT CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ.NEXTVAL INTO :NEW.A_SOURCE_FILE_CONFIG_KEY FROM DUAL;
EXCEPTION
-- WHEN NO_CONTAINER_FOR_INPUT THEN
-- RAISE_APPLICATION_ERROR(-20011, 'Error while adding entity where SOURCE_FILE_TYPE = ''INPUT'''
-- ||CHR(13)||CHR(10)||'There is no A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER'''
-- ||CHR(13)||CHR(10)||'Add ''CONTAINER'' config entity before ''INPUT'' one');
WHEN WRONG_CONTAINER_FOR_INPUT THEN
RAISE_APPLICATION_ERROR(-20012, 'Provided CONTAINER_FILE_KEY='||:NEW.CONTAINER_FILE_KEY||' is wrong'
||CHR(13)||CHR(10)||'There is A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER'''
||CHR(13)||CHR(10)||'Existing CONTAINER_FILE_KEY='||vContainerFileKey);
WHEN CONTAINER_ALREADY_EXISTS THEN
RAISE_APPLICATION_ERROR(-20013, 'Such CONTAINER already exists!'
||CHR(13)||CHR(10)||'There is A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER''');
END;

36
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_tst.sql Normal file
View File

@@ -0,0 +1,36 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_tst_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload tst specific part
--=============================================================================================================================
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('default', 'EnvironmentID', 'tst', '');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'Region', 'eu-frankfurt-1', 'OCI region for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'NameSpace', 'frcnomajoc7v', 'OCI namespace for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'InboxBucketName', 'mrds_inbox_tst', 'Name of the inbox bucket for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'DataBucketName', 'mrds_data_tst', 'Name of the data bucket for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'HistoryBucketName', 'mrds_history_tst', 'Name of the history bucket for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'CredentialName', 'OCI$RESOURCE_PRINCIPAL', 'Name of the credential for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'LoggingEnabled', 'ON', 'Enable logging for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'MinLogLevel', 'DEBUG', 'Minimum logging level for test environment');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID, CONFIG_VARIABLE, CONFIG_VARIABLE_VALUE, CONFIG_VARIABLE_COMMENT) VALUES ('tst', 'DefaultDateFormat', 'DD/MM/YYYY HH24:MI:SS', 'Default date format for test environment');
COMMIT;

116
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/INSTALL/install_package_MarsGeneralRelease_MARS770_views.sql Normal file
View File

@@ -0,0 +1,116 @@
--MARS-770
set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;
column owner format a15
column object_name format a30
column spoolname new_value spoolname
select 'install_package_MarsGeneralRelease_MARS770_views_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'
prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;
--set echo on
--=============================================================================================================================
-- start install --MARS-770 views
--=============================================================================================================================
create or replace view CT_MRDS.VW_A_PROCESS_LOG as
select * from (
select
l.a_process_log_key
,l.guid
,l.process_name
,l.procedure_name
,l.log_level
,l.log_message
,l.log_timestamp
,lead(log_message) over (partition by l.process_name, l.procedure_name, guid order by l.a_process_log_key) as next_log_message
,lead(log_timestamp) over (partition by l.process_name, l.procedure_name, guid order by l.a_process_log_key) as next_log_timestamp
from CT_MRDS.a_process_log l
) where 1=1
and log_message='Start'
order by log_timestamp
;
create or replace view CT_MRDS.VW_FILE_RECONCILIATION as
select
case
when c.bytes is null then 'RECEIVED_MISSING_ON_CLOUD'
when r.bytes is null then 'NOT_RECEIVED'
when r.bytes is not null and c.bytes is not null then 'RECEIVED'
else 'UNKNOWN'
end as status
,c.object_name as cloud_file
,r.*
from DBMS_CLOUD.LIST_OBJECTS('OCI$RESOURCE_PRINCIPAL', CT_MRDS.FILE_MANAGER.GET_INBOX_BUCKET_URI()) c
full join a_source_file_received r
on c.bytes = r.bytes
and c.checksum = r.checksum
and c.created = r.created
;
create or replace view CT_MRDS.VW_FILE_RECONCILIATION_SUMMARY as
select
all_statuses.status
,nvl(recon_stats.cnt, 0) as number_of_files
from (
select 'RECEIVED_MISSING_ON_CLOUD' as status from dual
union all select 'NOT_RECEIVED' from dual
union all select 'RECEIVED' from dual
union all select 'UNKNOWN' from dual
) all_statuses
left join (select status, count(*) as cnt from CT_MRDS.VW_FILE_RECONCILIATION group by status
) recon_stats
on all_statuses.status = recon_stats.status
order by 1
;
CREATE OR REPLACE FORCE EDITIONABLE VIEW "CT_MRDS"."A_MOPDB_LOAD_HISTORY" ("A_WORKFLOW_HISTORY_KEY", "A_WORKFLOW_HISTORY_SOURCE_KEY", "A_TASK_HISTORY_KEY", "TASK_NAME", "TASK_START", "TASK_END", "WORKFLOW_SUCCESSFUL") DEFAULT COLLATION "USING_NLS_COMP" AS
SELECT distinct wh.a_workflow_history_key,
ths.a_workflow_history_source_key,
th.a_task_history_key,
th.task_name,
th.task_start,
th.task_end,
wh.workflow_successful
FROM CT_MRDS.A_WORKFLOW_HISTORY wh
JOIN CT_MRDS.A_TASK_HISTORY th
ON wh.a_workflow_history_key = th.a_workflow_history_key
AND wh.service_name = th.service_name
JOIN CT_MRDS.A_TASK_HISTORY_SOURCE ths
ON th.a_task_history_key = ths.a_task_history_key
AND th.service_name = ths.service_name
WHERE wh.service_name = 'MOPDB'
AND wh.workflow_successful = 'Y'
AND lower(th.task_name) not like '%_sq'
AND ths.a_workflow_history_source_key is not null;
CREATE OR REPLACE FORCE EDITIONABLE VIEW "CT_MRDS"."A_DWH_LOAD_HISTORY" ("A_WORKFLOW_HISTORY_KEY", "A_WORKFLOW_HISTORY_SOURCE_KEY", "A_TASK_HISTORY_KEY", "TASK_NAME", "TASK_START", "TASK_END", "WORKFLOW_SUCCESSFUL") DEFAULT COLLATION "USING_NLS_COMP" AS
SELECT distinct wh.a_workflow_history_key,
ths.a_workflow_history_source_key,
th.a_task_history_key,
th.task_name,
th.task_start,
th.task_end,
wh.workflow_successful
FROM CT_MRDS.A_WORKFLOW_HISTORY wh
JOIN CT_MRDS.A_TASK_HISTORY th
ON wh.a_workflow_history_key = th.a_workflow_history_key
AND wh.service_name = th.service_name
JOIN CT_MRDS.A_TASK_HISTORY_SOURCE ths
ON th.a_task_history_key = ths.a_task_history_key
AND th.service_name = ths.service_name
WHERE wh.service_name = 'RAR'
AND wh.workflow_successful = 'Y'
AND lower(th.task_name) not like '%_sq'
AND ths.a_workflow_history_source_key is not null;

0
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/.gitkeep Normal file
View File

24
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/CREATE_USER.sql Normal file
View File

@@ -0,0 +1,24 @@
DROP USER CT_MRDS CASCADE;
CREATE USER CT_MRDS IDENTIFIED BY Start_1234567890
DEFAULT TABLESPACE USERS TEMPORARY TABLESPACE TEMP;
ALTER USER CT_MRDS QUOTA UNLIMITED ON DATA;
EXEC DBMS_CLOUD_ADMIN.ENABLE_RESOURCE_PRINCIPAL(username => 'CT_MRDS');
-- Cloud and database privileges (MARS-770)
GRANT EXECUTE ON DBMS_CLOUD TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER ADMIN TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER PDBADMIN TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER PDBSOFA TO CT_MRDS;
GRANT INHERIT PRIVILEGES ON USER MRDS_LOADER TO CT_MRDS;
-- Grant DBMS_CRYPTO for package hash calculation (SHA256)
GRANT EXECUTE ON DBMS_CRYPTO TO CT_MRDS;
-- Grant SELECT ANY DICTIONARY for cross-schema package hash tracking (MARS-1049)
-- Allows ENV_MANAGER to read ALL_SOURCE for packages in ODS and other schemas
-- Required for CALCULATE_PACKAGE_HASH to work across schemas
GRANT SELECT ANY DICTIONARY TO CT_MRDS;

25
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/grants/MARS770_additional_grants.sql Normal file
View File

@@ -0,0 +1,25 @@
-- ====================================================================
-- Additional Grants from MARS-770
-- ====================================================================
-- Purpose: Additional grants and dynamic privilege assignments
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_grants.sql
-- ====================================================================
-- Grant SELECT ANY TABLE to specific schemas
GRANT SELECT ANY TABLE TO OU_RQSD;
GRANT SELECT ANY TABLE TO RQSD;
-- Dynamic grants to all non-Oracle schemas for A_WORKFLOW_HISTORY access
BEGIN
FOR user_rec IN (SELECT username FROM dba_users WHERE oracle_maintained != 'Y' AND common != 'YES') LOOP
DECLARE
schema_name VARCHAR2(128) := user_rec.username;
BEGIN
EXECUTE IMMEDIATE 'GRANT SELECT ON CT_MRDS.A_WORKFLOW_HISTORY TO "'|| schema_name ||'" WITH GRANT OPTION';
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.PUT_LINE('Error processing schema ' || schema_name || ': ' || SQLERRM);
END;
END LOOP;
END;
/

733
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/DATA_EXPORTER.pkb Normal file
View File

@@ -0,0 +1,733 @@
create or replace PACKAGE BODY CT_MRDS.DATA_EXPORTER
AS
-- Internal shared function to process column list with T. prefix and key column mapping
FUNCTION processColumnList(pColumnList IN VARCHAR2, pTableName IN VARCHAR2, pSchemaName IN VARCHAR2, pKeyColumnName IN VARCHAR2) RETURN VARCHAR2 IS
vResult VARCHAR2(32767);
vColumns VARCHAR2(32767);
vPos PLS_INTEGER;
vNextPos PLS_INTEGER;
vCurrentCol VARCHAR2(128);
vAllCols VARCHAR2(32767);
BEGIN
IF pColumnList IS NULL THEN
-- Build list of all columns
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
INTO vAllCols
FROM all_tab_columns
WHERE table_name = pTableName
AND owner = pSchemaName;
-- Add T. prefix to all columns
vResult := 'T.' || REPLACE(vAllCols, ', ', ', T.');
-- Replace key column with aliased version (e.g., T.A_ETL_LOAD_SET_KEY_FK AS A_WORKFLOW_HISTORY_KEY)
vResult := REPLACE(vResult, 'T.' || pKeyColumnName, 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY');
RETURN vResult;
END IF;
-- Remove extra spaces and convert to uppercase
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
vPos := 1;
vResult := '';
-- Parse comma-separated column list and add T. prefix
WHILE vPos <= LENGTH(vColumns) LOOP
vNextPos := INSTR(vColumns, ',', vPos);
IF vNextPos = 0 THEN
vNextPos := LENGTH(vColumns) + 1;
END IF;
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
-- Check if this is the key column (e.g., A_ETL_LOAD_SET_KEY_FK) and add alias
IF UPPER(vCurrentCol) = UPPER(pKeyColumnName) THEN
vCurrentCol := 'T.' || pKeyColumnName || ' AS A_WORKFLOW_HISTORY_KEY';
ELSIF UPPER(vCurrentCol) = 'A_ETL_LOAD_SET_KEY' THEN
vCurrentCol := 'T.A_ETL_LOAD_SET_KEY AS A_WORKFLOW_HISTORY_KEY';
ELSE
-- Add T. prefix if not already present
IF INSTR(vCurrentCol, '.') = 0 THEN
vCurrentCol := 'T.' || vCurrentCol;
END IF;
END IF;
-- Add to result with comma separator
IF vResult IS NOT NULL THEN
vResult := vResult || ', ';
END IF;
vResult := vResult || vCurrentCol;
vPos := vNextPos + 1;
END LOOP;
RETURN vResult;
END processColumnList;
----------------------------------------------------------------------------------------------------
PROCEDURE EXPORT_TABLE_DATA (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
)
IS
-- Type definition for key values
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
vKeyValues key_value_tab;
vCount INTEGER;
vSql VARCHAR2(4000);
vKeyValue VARCHAR2(4000);
vQuery VARCHAR2(32767);
vUri VARCHAR2(4000);
vDataType VARCHAR2(30);
vTableName VARCHAR2(128);
vSchemaName VARCHAR2(128);
vKeyColumnName VARCHAR2(128);
vParameters VARCHAR2(4000);
vBucketUri VARCHAR2(4000);
vProcessedColumnList VARCHAR2(32767);
vCurrentCol VARCHAR2(128);
vAllColumnsList VARCHAR2(32767);
-- Function to sanitize file names
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
vFilename VARCHAR2(1000);
BEGIN
-- Replace any disallowed characters with underscores
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
RETURN vFilename;
END sanitizeFilename;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Get bucket URI based on bucket area using FILE_MANAGER function
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
-- Convert table and column names to uppercase to match data dictionary
vTableName := UPPER(pTableName);
vSchemaName := UPPER(pSchemaName);
vKeyColumnName := UPPER(pKeyColumnName);
-- Check if table exists
SELECT COUNT(*) INTO vCount
FROM all_tables
WHERE table_name = vTableName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
END IF;
-- Check if key column exists
SELECT COUNT(*) INTO vCount
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vKeyColumnName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
END IF;
-- Get the data type of the key column
SELECT data_type INTO vDataType
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vKeyColumnName
AND owner = vSchemaName;
-- Build list of all columns for the table (excluding key column to avoid duplication)
SELECT LISTAGG(column_name, ', ') WITHIN GROUP (ORDER BY column_id)
INTO vAllColumnsList
FROM all_tab_columns
WHERE table_name = vTableName
AND owner = vSchemaName
AND column_name != vKeyColumnName;
-- Process column list to add T. prefix to each column
vProcessedColumnList := processColumnList(vAllColumnsList, vTableName, vSchemaName, vKeyColumnName);
ENV_MANAGER.LOG_PROCESS_EVENT('Dynamic column list built (excluding key): ' || vAllColumnsList, 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list with T. prefix: ' || vProcessedColumnList, 'DEBUG', vParameters);
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
-- Fetch unique key values from A_LOAD_HISTORY
vSql := 'SELECT DISTINCT L.A_ETL_LOAD_SET_KEY' ||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY';
ENV_MANAGER.LOG_PROCESS_EVENT('Executing key values query: ' || vSql, 'DEBUG', vParameters);
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValues;
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValues.COUNT || ' unique key values to process', 'DEBUG', vParameters);
-- Loop over each unique key value
FOR i IN 1 .. vKeyValues.COUNT LOOP
vKeyValue := vKeyValues(i);
-- Construct the query to extract data for the current key value with A_WORKFLOW_HISTORY_KEY mapping
IF vDataType IN ('VARCHAR2', 'CHAR', 'NCHAR', 'NVARCHAR2') THEN
vQuery := 'SELECT ' || vProcessedColumnList ||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
' AND L.A_ETL_LOAD_SET_KEY = ' || CHR(39) || vKeyValue || CHR(39);
ELSIF vDataType IN ('NUMBER', 'FLOAT', 'BINARY_FLOAT', 'BINARY_DOUBLE') THEN
vQuery := 'SELECT ' || vProcessedColumnList ||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
' AND L.A_ETL_LOAD_SET_KEY = ' || vKeyValue;
ELSIF vDataType LIKE 'TIMESTAMP%' OR vDataType = 'DATE' THEN
vQuery := 'SELECT ' || vProcessedColumnList ||
' FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L' ||
' WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY' ||
' AND L.A_ETL_LOAD_SET_KEY = TO_TIMESTAMP(' || CHR(39) || vKeyValue || CHR(39) ||', ''YYYY-MM-DD HH24:MI:SS.FF'')';
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE);
END IF;
-- Construct the URI for the file in OCI Object Storage
vUri := vBucketUri ||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
sanitizeFilename(vKeyValue) || '.csv';
ENV_MANAGER.LOG_PROCESS_EVENT('Processing key value: ' || vKeyValue || ' (' || (i) || '/' || vKeyValues.COUNT || ')', 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export URI: ' || vUri, 'DEBUG', vParameters);
-- Use DBMS_CLOUD package to export data to the URI
DBMS_CLOUD.EXPORT_DATA(
credential_name => pCredentialName,
file_uri_list => vUri,
query => vQuery,
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
);
END LOOP;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in column list' ELSE '' END;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN
vgMsgTmp := ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE || ' vDataType: '||vDataType;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, vgMsgTmp);
WHEN OTHERS THEN
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END EXPORT_TABLE_DATA;
----------------------------------------------------------------------------------------------------
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
)
IS
-- Type definition for key values
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
vKeyValuesYear key_value_tab;
vKeyValuesMonth key_value_tab;
vCount INTEGER;
vSql VARCHAR2(32000);
vKeyValueYear VARCHAR2(4000);
vKeyValueMonth VARCHAR2(4000);
vQuery VARCHAR2(32767);
vUri VARCHAR2(4000);
vDataType VARCHAR2(30);
vTableName VARCHAR2(128);
vSchemaName VARCHAR2(128);
vKeyColumnName VARCHAR2(128);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vProcessedColumnList VARCHAR2(32767);
vBucketUri VARCHAR2(4000);
vCurrentCol VARCHAR2(128);
-- Function to sanitize file names
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
vFilename VARCHAR2(1000);
BEGIN
-- Replace any disallowed characters with underscores
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
RETURN vFilename;
END sanitizeFilename;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Get bucket URI based on bucket area using FILE_MANAGER function
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
-- Convert table and column names to uppercase to match data dictionary
vTableName := UPPER(pTableName);
vSchemaName := UPPER(pSchemaName);
vKeyColumnName := UPPER(pKeyColumnName);
-- Check if table exists
SELECT COUNT(*) INTO vCount
FROM all_tables
WHERE table_name = vTableName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
END IF;
-- Check if key column exists
SELECT COUNT(*) INTO vCount
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vKeyColumnName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
END IF;
-- Validate pColumnList - check if all column names exist in the table
IF pColumnList IS NOT NULL THEN
DECLARE
vColumnName VARCHAR2(128);
vColumns VARCHAR2(32767);
vPos PLS_INTEGER;
vNextPos PLS_INTEGER;
vCurrentCol VARCHAR2(128);
BEGIN
-- Remove spaces and convert to uppercase for processing
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
vPos := 1;
-- Parse comma-separated column list
WHILE vPos <= LENGTH(vColumns) LOOP
vNextPos := INSTR(vColumns, ',', vPos);
IF vNextPos = 0 THEN
vNextPos := LENGTH(vColumns) + 1;
END IF;
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
-- Remove table alias prefix if present (e.g., 'T.COLUMN_NAME' -> 'COLUMN_NAME')
IF INSTR(vCurrentCol, '.') > 0 THEN
vCurrentCol := SUBSTR(vCurrentCol, INSTR(vCurrentCol, '.') + 1);
END IF;
-- Check if column exists in the table
SELECT COUNT(*) INTO vCount
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vCurrentCol
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
END IF;
vPos := vNextPos + 1;
END LOOP;
END;
END IF;
-- Process column list to add T. prefix to each column
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (building dynamic list from table metadata)'), 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
-- Fetch unique key values
vSql := 'SELECT DISTINCT TO_CHAR(L.LOAD_START,''YYYY'') AS YR, TO_CHAR(L.LOAD_START,''MM'') AS MN
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
AND L.LOAD_START >= :pMinDate
AND L.LOAD_START < :pMaxDate
' ;
ENV_MANAGER.LOG_PROCESS_EVENT('Executing date range query: ' || vSql, 'DEBUG', vParameters);
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValuesYear, vKeyValuesMonth USING pMinDate, pMaxDate;
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValuesYear.COUNT || ' year/month combinations to export', 'DEBUG', vParameters);
-- Loop over each unique key value
FOR i IN 1 .. vKeyValuesYear.COUNT LOOP
vKeyValueYear := vKeyValuesYear(i);
vKeyValueMonth := vKeyValuesMonth(i);
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || vKeyValueYear || '/' || vKeyValueMonth || ' (' || i || '/' || vKeyValuesYear.COUNT || ')', 'DEBUG', vParameters);
-- Construct the query to extract data for the current key value
-- Note: processColumnList already handles A_WORKFLOW_HISTORY_KEY aliasing
vQuery := 'SELECT ' || vProcessedColumnList || '
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
AND TO_CHAR(L.LOAD_START,''YYYY'') = ' || CHR(39) || vKeyValueYear || CHR(39) || '
AND TO_CHAR(L.LOAD_START,''MM'') = ' || CHR(39) || vKeyValueMonth || CHR(39) || '
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
-- Construct the URI for the file in OCI Object Storage
vUri := vBucketUri ||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
'PARTITION_YEAR=' || sanitizeFilename(vKeyValueYear) || '/' ||
'PARTITION_MONTH=' || sanitizeFilename(vKeyValueMonth) || '/' ||
sanitizeFilename(vKeyValueYear) || sanitizeFilename(vKeyValueMonth) || '.parquet';
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Parquet export URI: ' || vUri, 'DEBUG', vParameters);
-- Use DBMS_CLOUD package to export data to the URI
DBMS_CLOUD.EXPORT_DATA(
credential_name => pCredentialName,
file_uri_list => vUri,
query => vQuery,
format => json_object('type' VALUE 'parquet')
);
END LOOP;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
WHEN OTHERS THEN
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END EXPORT_TABLE_DATA_BY_DATE;
----------------------------------------------------------------------------------------------------
/**
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
* @desc Exports data to a single CSV file with date filtering.
* Unlike EXPORT_TABLE_DATA_BY_DATE, this procedure creates one CSV file
* instead of multiple Parquet files partitioned by year/month.
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY.
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
* Validates that all columns in pColumnList exist in the target table.
* Automatically adds 'T.' prefix to column names in pColumnList.
* @example
* begin
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'exports',
* pFileName => 'my_export.csv',
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
* pMinDate => DATE '2024-01-01',
* pMaxDate => SYSDATE
* );
* end;
**/
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pFileName IN VARCHAR2 DEFAULT NULL,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
)
IS
-- Type definition for key values
TYPE key_value_tab IS TABLE OF VARCHAR2(4000);
vKeyValuesYear key_value_tab;
vKeyValuesMonth key_value_tab;
vCount INTEGER;
vSql VARCHAR2(4000);
vKeyValueYear VARCHAR2(4000);
vKeyValueMonth VARCHAR2(4000);
vQuery VARCHAR2(32767);
vUri VARCHAR2(4000);
vDataType VARCHAR2(30);
vTableName VARCHAR2(128);
vSchemaName VARCHAR2(128);
vKeyColumnName VARCHAR2(128);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vFileBaseName VARCHAR2(4000);
vFileExtension VARCHAR2(10);
vProcessedColumnList VARCHAR2(32767);
vBucketUri VARCHAR2(4000);
vCurrentCol VARCHAR2(128);
-- Function to sanitize file names
FUNCTION sanitizeFilename(pFilename IN VARCHAR2) RETURN VARCHAR2 IS
vFilename VARCHAR2(1000);
BEGIN
-- Replace any disallowed characters with underscores
vFilename := REGEXP_REPLACE(pFilename, '[^a-zA-Z0-9._-]', '_');
RETURN vFilename;
END sanitizeFilename;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSchemaName => '''||nvl(pSchemaName, 'NULL')||''''
,'pTableName => '''||nvl(pTableName, 'NULL')||''''
,'pKeyColumnName => '''||nvl(pKeyColumnName, 'NULL')||''''
,'pBucketArea => '''||nvl(pBucketArea, 'NULL')||''''
,'pFolderName => '''||nvl(pFolderName, 'NULL')||''''
,'pFileName => '''||nvl(pFileName, 'NULL')||''''
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
,'pMinDate => '''||nvl(TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
,'pMaxDate => '''||nvl(TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'NULL')||''''
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Get bucket URI based on bucket area using FILE_MANAGER function
vBucketUri := FILE_MANAGER.GET_BUCKET_URI(pBucketArea);
-- Convert table and column names to uppercase to match data dictionary
vTableName := UPPER(pTableName);
vSchemaName := UPPER(pSchemaName);
vKeyColumnName := UPPER(pKeyColumnName);
-- Extract base filename and extension or construct default filename
IF pFileName IS NOT NULL THEN
-- Use provided filename
IF INSTR(pFileName, '.') > 0 THEN
vFileBaseName := SUBSTR(pFileName, 1, INSTR(pFileName, '.', -1) - 1);
vFileExtension := SUBSTR(pFileName, INSTR(pFileName, '.', -1));
ELSE
vFileBaseName := pFileName;
vFileExtension := '.csv';
END IF;
ELSE
-- Construct default filename: TABLENAME.csv (without date range)
vFileBaseName := UPPER(pTableName);
vFileExtension := '.csv';
END IF;
-- Check if table exists
SELECT COUNT(*) INTO vCount
FROM all_tables
WHERE table_name = vTableName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
END IF;
-- Check if key column exists
SELECT COUNT(*) INTO vCount
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vKeyColumnName
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
END IF;
-- Validate pColumnList - check if all column names exist in the table
IF pColumnList IS NOT NULL THEN
DECLARE
vColumnName VARCHAR2(128);
vColumns VARCHAR2(32767);
vPos PLS_INTEGER;
vNextPos PLS_INTEGER;
vCurrentCol VARCHAR2(128);
BEGIN
-- Remove spaces and convert to uppercase for processing
vColumns := UPPER(REPLACE(pColumnList, ' ', ''));
vPos := 1;
-- Parse comma-separated column list
WHILE vPos <= LENGTH(vColumns) LOOP
vNextPos := INSTR(vColumns, ',', vPos);
IF vNextPos = 0 THEN
vNextPos := LENGTH(vColumns) + 1;
END IF;
vCurrentCol := SUBSTR(vColumns, vPos, vNextPos - vPos);
-- Remove table alias prefix if present (e.g., 'T.COLUMN_NAME' -> 'COLUMN_NAME')
IF INSTR(vCurrentCol, '.') > 0 THEN
vCurrentCol := SUBSTR(vCurrentCol, INSTR(vCurrentCol, '.') + 1);
END IF;
-- Check if column exists in the table
SELECT COUNT(*) INTO vCount
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vCurrentCol
AND owner = vSchemaName;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
END IF;
vPos := vNextPos + 1;
END LOOP;
END;
END IF;
-- Process column list to add T. prefix to each column
vProcessedColumnList := processColumnList(pColumnList, vTableName, vSchemaName, vKeyColumnName);
ENV_MANAGER.LOG_PROCESS_EVENT('Input column list: ' || NVL(pColumnList, 'NULL (using dynamic column list)'), 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Processed column list: ' || vProcessedColumnList, 'DEBUG', vParameters);
-- Get the data type of the key column
SELECT data_type INTO vDataType
FROM all_tab_columns
WHERE table_name = vTableName
AND column_name = vKeyColumnName
AND owner = vSchemaName;
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSchemaName) || '.' || DBMS_ASSERT.simple_sql_name(vTableName);
-- Fetch unique year/month combinations
vSql := 'SELECT DISTINCT TO_CHAR(L.LOAD_START,''YYYY'') AS YR, TO_CHAR(L.LOAD_START,''MM'') AS MN
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
AND L.LOAD_START >= :pMinDate
AND L.LOAD_START < :pMaxDate
' ;
ENV_MANAGER.LOG_PROCESS_EVENT('Executing date range query: ' || vSql, 'DEBUG', vParameters);
EXECUTE IMMEDIATE vSql BULK COLLECT INTO vKeyValuesYear, vKeyValuesMonth USING pMinDate, pMaxDate;
ENV_MANAGER.LOG_PROCESS_EVENT('Found ' || vKeyValuesYear.COUNT || ' year/month combinations to export', 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Date range: ' || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || ' to ' || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS'), 'DEBUG', vParameters);
-- Loop over each unique year/month combination
FOR i IN 1 .. vKeyValuesYear.COUNT LOOP
vKeyValueYear := vKeyValuesYear(i);
vKeyValueMonth := vKeyValuesMonth(i);
-- Construct the query to extract data for the current year/month
vQuery := 'SELECT ' || vProcessedColumnList || '
FROM ' || vTableName || ' T, CT_ODS.A_LOAD_HISTORY L
WHERE T.' || DBMS_ASSERT.simple_sql_name(vKeyColumnName) || ' = L.A_ETL_LOAD_SET_KEY
AND TO_CHAR(L.LOAD_START,''YYYY'') = ' || CHR(39) || vKeyValueYear || CHR(39) || '
AND TO_CHAR(L.LOAD_START,''MM'') = ' || CHR(39) || vKeyValueMonth || CHR(39) || '
AND L.LOAD_START >= TO_DATE(' || CHR(39) || TO_CHAR(pMinDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')
AND L.LOAD_START < TO_DATE(' || CHR(39) || TO_CHAR(pMaxDate, 'YYYY-MM-DD HH24:MI:SS') || CHR(39) || ', ''YYYY-MM-DD HH24:MI:SS'')';
-- Construct the URI for the CSV file in OCI Object Storage
vUri := vBucketUri ||
CASE WHEN pFolderName IS NOT NULL THEN pFolderName || '/' ELSE '' END ||
sanitizeFilename(vFileBaseName) || '_' ||
sanitizeFilename(vKeyValueYear) || sanitizeFilename(vKeyValueMonth) ||
vFileExtension;
ENV_MANAGER.LOG_PROCESS_EVENT('Exporting to CSV file: ' || vUri, 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Processing Year/Month: ' || vKeyValueYear || '/' || vKeyValueMonth || ' (' || i || '/' || vKeyValuesYear.COUNT || ')', 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('Export query: ' || vQuery, 'DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('File name pattern: ' || vFileBaseName || '_' || vKeyValueYear || vKeyValueMonth || vFileExtension, 'DEBUG', vParameters);
-- Use DBMS_CLOUD package to export data to CSV file
DBMS_CLOUD.EXPORT_DATA(
credential_name => pCredentialName,
file_uri_list => vUri,
query => vQuery,
format => json_object('type' VALUE 'CSV', 'header' VALUE true)
);
END LOOP;
ENV_MANAGER.LOG_PROCESS_EVENT('Export completed successfully for ' || vKeyValuesYear.COUNT || ' files', 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_TABLE_NOT_EXISTS ||': '||vTableName;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, vgMsgTmp);
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
vgMsgTmp := ENV_MANAGER.MSG_COLUMN_NOT_EXISTS || ' (TableName.ColumnName): ' || vTableName||'.'||vKeyColumnName||CASE WHEN vCurrentCol IS NOT NULL THEN '.'||vCurrentCol||' in pColumnList' ELSE '' END;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, vgMsgTmp);
WHEN OTHERS THEN
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR('Export failed: ' || SQLERRM, vParameters, 'DATA_EXPORTER');
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END EXPORT_TABLE_DATA_TO_CSV_BY_DATE;
----------------------------------------------------------------------------------------------------
-- VERSION MANAGEMENT FUNCTIONS
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION RETURN VARCHAR2 IS
BEGIN
RETURN PACKAGE_VERSION;
END GET_VERSION;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUILD_INFO RETURN VARCHAR2 IS
BEGIN
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
pPackageName => 'DATA_EXPORTER',
pVersion => PACKAGE_VERSION,
pBuildDate => PACKAGE_BUILD_DATE,
pAuthor => PACKAGE_AUTHOR
);
END GET_BUILD_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2 IS
BEGIN
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
pPackageName => 'DATA_EXPORTER',
pVersionHistory => VERSION_HISTORY
);
END GET_VERSION_HISTORY;
----------------------------------------------------------------------------------------------------
END;
/

166
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/DATA_EXPORTER.pkg Normal file
View File

@@ -0,0 +1,166 @@
create or replace PACKAGE CT_MRDS.DATA_EXPORTER
AUTHID CURRENT_USER
AS
/**
* Data Export Package: Provides comprehensive data export capabilities to various formats (CSV, Parquet)
* with support for cloud storage integration via Oracle Cloud Infrastructure (OCI).
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Package Version Information
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.1.1';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(19) := '2025-12-04 13:10:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(50) := 'MRDS Development Team';
-- Version History (last 3-5 changes)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY, added consistent column mapping and dynamic column list to EXPORT_TABLE_DATA procedure, enhanced DEBUG logging for all export operations' || CHR(10) ||
'v2.1.1 (2025-12-04): Fixed JOIN column reference A_WORKFLOW_HISTORY_KEY -> A_ETL_LOAD_SET_KEY' || CHR(10) ||
'v2.1.0 (2025-10-22): Added version tracking and PARTITION_YEAR/PARTITION_MONTH support' || CHR(10) ||
'v2.0.0 (2025-10-01): Separated export functionality from FILE_MANAGER package' || CHR(10) ||
'v1.0.0 (2025-09-15): Initial implementation within FILE_MANAGER package' || CHR(10);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgMsgTmp VARCHAR2(32000);
---------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------
/**
* @name EXPORT_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data into CSV file on OCI infrustructure.
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* @example
* begin
* DATA_EXPORTER.EXPORT_TABLE_DATA(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'csv_exports'
* );
* end;
**/
PROCEDURE EXPORT_TABLE_DATA (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
/**
* @name EXPORT_TABLE_DATA_BY_DATE
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data into PARQUET files on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
* Allows specifying custom column list or uses T.* if pColumnList is NULL.
* Validates that all columns in pColumnList exist in the target table.
* Automatically adds 'T.' prefix to column names in pColumnList.
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* @example
* begin
* DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'parquet_exports',
* pColumnList => 'COLUMN1, COLUMN2, COLUMN3', -- Optional
* pMinDate => DATE '2024-01-01',
* pMaxDate => SYSDATE
* );
* end;
**/
PROCEDURE EXPORT_TABLE_DATA_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
/**
* @name EXPORT_TABLE_DATA_TO_CSV_BY_DATE
* @desc Exports data to separate CSV files partitioned by year and month.
* Creates one CSV file for each year/month combination found in the data.
* Uses the same date filtering mechanism with CT_ODS.A_LOAD_HISTORY as EXPORT_TABLE_DATA_BY_DATE,
* but exports to CSV format instead of Parquet.
* File naming pattern: {pFileName}_YYYYMM.csv or {TABLENAME}_YYYYMM.csv (if pFileName is NULL)
* @example
* begin
* -- With custom filename
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
* pSchemaName => 'CT_MRDS',
* pTableName => 'MY_TABLE',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'DATA',
* pFolderName => 'exports',
* pFileName => 'my_export.csv',
* pMinDate => DATE '2024-01-01',
* pMaxDate => SYSDATE
* );
*
* -- With auto-generated filename (based on table name only)
* DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
* pSchemaName => 'OU_TOP',
* pTableName => 'AGGREGATED_ALLOTMENT',
* pKeyColumnName => 'A_ETL_LOAD_SET_KEY_FK',
* pBucketArea => 'ARCHIVE',
* pFolderName => 'exports',
* pMinDate => DATE '2025-09-01',
* pMaxDate => DATE '2025-09-17'
* );
* -- This will create files like: AGGREGATED_ALLOTMENT_202509.csv, etc.
* pBucketArea parameter accepts: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* end;
**/
PROCEDURE EXPORT_TABLE_DATA_TO_CSV_BY_DATE (
pSchemaName IN VARCHAR2,
pTableName IN VARCHAR2,
pKeyColumnName IN VARCHAR2,
pBucketArea IN VARCHAR2,
pFolderName IN VARCHAR2,
pFileName IN VARCHAR2 DEFAULT NULL,
pColumnList IN VARCHAR2 default NULL,
pMinDate IN DATE default DATE '1900-01-01',
pMaxDate IN DATE default SYSDATE,
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
);
---------------------------------------------------------------------------------------------------------------------------
-- VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* Returns the current package version number
* return: Version string in format X.Y.Z (e.g., '2.1.0')
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* Returns comprehensive build information including version, date, and author
* return: Formatted string with complete build details
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* Returns the version history with recent changes
* return: Multi-line string with version history
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END;
/

1165
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/ENV_MANAGER.pkb Normal file
View File

File diff suppressed because it is too large Load Diff

611
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/ENV_MANAGER.pkg Normal file
View File

@@ -0,0 +1,611 @@
create or replace PACKAGE CT_MRDS.ENV_MANAGER
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select ENV_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.1.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-10-22 20:57:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.1.0 (2025-10-22): Added package hash tracking and automatic change detection system (SHA256 hashing)' || CHR(13)||CHR(10) ||
'3.0.0 (2025-10-22): Added package versioning system with centralized version management functions' || CHR(13)||CHR(10) ||
'2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function for comprehensive CSV validation analysis' || CHR(13)||CHR(10) ||
'2.0.0 (2025-10-01): Added LOG_PROCESS_ERROR procedure with enhanced error diagnostics and stack traces' || CHR(13)||CHR(10) ||
'1.5.0 (2025-09-20): Added console logging support with gvConsoleLoggingEnabled configuration' || CHR(13)||CHR(10) ||
'1.0.0 (2025-09-01): Initial release with error management and configuration system';
TYPE Error_Record IS RECORD (
code PLS_INTEGER,
message VARCHAR2(4000)
);
TYPE tErrorList IS TABLE OF Error_Record INDEX BY PLS_INTEGER;
Errors tErrorList;
guid VARCHAR2(32);
gvEnv VARCHAR2(200);
gvUsername VARCHAR2(128);
gvOsuser VARCHAR2(128);
gvMachine VARCHAR2(64);
gvModule VARCHAR2(64);
gvNameSpace VARCHAR2(200);
gvRegion VARCHAR2(200);
gvDataBucketName VARCHAR2(200);
gvInboxBucketName VARCHAR2(200);
gvArchiveBucketName VARCHAR2(200);
gvDataBucketUri VARCHAR2(200);
gvInboxBucketUri VARCHAR2(200);
gvArchiveBucketUri VARCHAR2(200);
gvCredentialName VARCHAR2(200);
-- Overwritten by variable "LoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF'
-- Overwritten by variable "MinLogLevel" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
-- Possible values: DEBUG ,INFO ,WARNING ,ERROR
gvMinLogLevel VARCHAR2(10) := 'DEBUG';
-- Overwritten by variable "DefaultDateFormat" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvDefaultDateFormat VARCHAR2(200) := 'DD/MM/YYYY HH24:MI:SS';
-- Overwritten by variable "ConsoleLoggingEnabled" in A_FILE_MANAGER_CONFIG.CONFIG_VARIABLE table
gvConsoleLoggingEnabled VARCHAR2(3) := 'ON'; -- 'ON' or 'OFF'
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgSourceFileConfigKey PLS_INTEGER;
vgMsgTmp VARCHAR2(32000);
--Exceptions
ERR_EMPTY_FILEURI_AND_RECKEY EXCEPTION;
CODE_EMPTY_FILEURI_AND_RECKEY CONSTANT PLS_INTEGER := -20001;
MSG_EMPTY_FILEURI_AND_RECKEY VARCHAR2(4000) := 'Either pFileUri or pSourceFileReceivedKey must be not null';
PRAGMA EXCEPTION_INIT( ERR_EMPTY_FILEURI_AND_RECKEY
,CODE_EMPTY_FILEURI_AND_RECKEY);
ERR_NO_CONFIG_MATCH_FOR_FILEURI EXCEPTION;
CODE_NO_CONFIG_MATCH_FOR_FILEURI CONSTANT PLS_INTEGER := -20002;
MSG_NO_CONFIG_MATCH_FOR_FILEURI VARCHAR2(4000) := 'No match for source file in A_SOURCE_FILE_CONFIG table'
||cgBL||' The file provided in parameter: pFileUri does not have '
||cgBL||' coresponding configuration in A_SOURCE_FILE_CONFIG table';
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH_FOR_FILEURI
,CODE_NO_CONFIG_MATCH_FOR_FILEURI);
ERR_MULTIPLE_MATCH_FOR_SRCFILE EXCEPTION;
CODE_MULTIPLE_MATCH_FOR_SRCFILE CONSTANT PLS_INTEGER := -20003;
MSG_MULTIPLE_MATCH_FOR_SRCFILE VARCHAR2(4000) := 'Multiple match for source file in A_SOURCE_FILE_CONFIG table';
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_MATCH_FOR_SRCFILE
,CODE_MULTIPLE_MATCH_FOR_SRCFILE);
ERR_MISSING_COLUMN_DATE_FORMAT EXCEPTION;
CODE_MISSING_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20004;
MSG_MISSING_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Missing entry in config table: A_COLUMN_DATE_FORMAT primary key(TEMPLATE_TABLE_NAME, COLUMN_NAME)'
||cgBL||' Remember: each column which data_type IN (''DATE'', ''TIMESTAMP'')'
||cgBL||' should have DateFormat specified in A_COLUMN_DATE_FORMAT table '
||cgBL||' for example: ''YYYY-MM-DD''';
PRAGMA EXCEPTION_INIT( ERR_MISSING_COLUMN_DATE_FORMAT
,CODE_MISSING_COLUMN_DATE_FORMAT);
ERR_MULTIPLE_COLUMN_DATE_FORMAT EXCEPTION;
CODE_MULTIPLE_COLUMN_DATE_FORMAT CONSTANT PLS_INTEGER := -20005;
MSG_MULTIPLE_COLUMN_DATE_FORMAT VARCHAR2(4000) := 'Multiple records for date format in A_COLUMN_DATE_FORMAT table'
||cgBL||' There should be only one format specified for each DAT/TIMESTAMP column';
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_COLUMN_DATE_FORMAT
,CODE_MULTIPLE_COLUMN_DATE_FORMAT);
ERR_DIDNT_GET_LOAD_OPERATION_ID EXCEPTION;
CODE_DIDNT_GET_LOAD_OPERATION_ID CONSTANT PLS_INTEGER := -20006;
MSG_DIDNT_GET_LOAD_OPERATION_ID VARCHAR2(4000) := 'Didnt get load operation id from external table validation';
PRAGMA EXCEPTION_INIT( ERR_DIDNT_GET_LOAD_OPERATION_ID
,CODE_DIDNT_GET_LOAD_OPERATION_ID);
ERR_NO_CONFIG_FOR_RECEIVED_FILE EXCEPTION;
CODE_NO_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20007;
MSG_NO_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'No match for received source file in A_SOURCE_FILE_CONFIG '
||cgBL||' or missing data in A_SOURCE_FILE_RECEIVED table for provided pSourceFileReceivedKey parameter';
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_FOR_RECEIVED_FILE
,CODE_NO_CONFIG_FOR_RECEIVED_FILE);
ERR_MULTI_CONFIG_FOR_RECEIVED_FILE EXCEPTION;
CODE_MULTI_CONFIG_FOR_RECEIVED_FILE CONSTANT PLS_INTEGER := -20008;
MSG_MULTI_CONFIG_FOR_RECEIVED_FILE VARCHAR2(4000) := 'Multiple matchs for received source file in A_SOURCE_FILE_CONFIG';
PRAGMA EXCEPTION_INIT( ERR_MULTI_CONFIG_FOR_RECEIVED_FILE
,CODE_MULTI_CONFIG_FOR_RECEIVED_FILE);
ERR_FILE_NOT_FOUND_ON_CLOUD EXCEPTION;
CODE_FILE_NOT_FOUND_ON_CLOUD CONSTANT PLS_INTEGER := -20009;
MSG_FILE_NOT_FOUND_ON_CLOUD VARCHAR2(4000) := 'File not found on the cloud';
PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_FOUND_ON_CLOUD
,CODE_FILE_NOT_FOUND_ON_CLOUD);
ERR_FILE_VALIDATION_FAILED EXCEPTION;
CODE_FILE_VALIDATION_FAILED CONSTANT PLS_INTEGER := -20010;
MSG_FILE_VALIDATION_FAILED VARCHAR2(4000) := 'File validation failed';
PRAGMA EXCEPTION_INIT( ERR_FILE_VALIDATION_FAILED
,CODE_FILE_VALIDATION_FAILED);
ERR_EXCESS_COLUMNS_DETECTED EXCEPTION;
CODE_EXCESS_COLUMNS_DETECTED CONSTANT PLS_INTEGER := -20011;
MSG_EXCESS_COLUMNS_DETECTED VARCHAR2(4000) := 'CSV file contains more columns than template allows';
PRAGMA EXCEPTION_INIT( ERR_EXCESS_COLUMNS_DETECTED
,CODE_EXCESS_COLUMNS_DETECTED);
ERR_NO_CONFIG_MATCH EXCEPTION;
CODE_NO_CONFIG_MATCH CONSTANT PLS_INTEGER := -20012;
MSG_NO_CONFIG_MATCH VARCHAR2(4000) := 'No match for specified parameters in A_SOURCE_FILE_CONFIG table';
PRAGMA EXCEPTION_INIT( ERR_NO_CONFIG_MATCH
,CODE_NO_CONFIG_MATCH);
ERR_UNKNOWN_PREFIX EXCEPTION;
CODE_UNKNOWN_PREFIX CONSTANT PLS_INTEGER := -20013;
MSG_UNKNOWN_PREFIX VARCHAR2(4000) := 'Unknown prefix';
PRAGMA EXCEPTION_INIT( ERR_UNKNOWN_PREFIX
,CODE_UNKNOWN_PREFIX);
ERR_TABLE_NOT_EXISTS EXCEPTION;
CODE_TABLE_NOT_EXISTS CONSTANT PLS_INTEGER := -20014;
MSG_TABLE_NOT_EXISTS VARCHAR2(4000) := 'Table does not exist';
PRAGMA EXCEPTION_INIT( ERR_TABLE_NOT_EXISTS
,CODE_TABLE_NOT_EXISTS);
ERR_COLUMN_NOT_EXISTS EXCEPTION;
CODE_COLUMN_NOT_EXISTS CONSTANT PLS_INTEGER := -20015;
MSG_COLUMN_NOT_EXISTS VARCHAR2(4000) := 'Column does not exist in table';
PRAGMA EXCEPTION_INIT( ERR_COLUMN_NOT_EXISTS
,CODE_COLUMN_NOT_EXISTS);
ERR_UNSUPPORTED_DATA_TYPE EXCEPTION;
CODE_UNSUPPORTED_DATA_TYPE CONSTANT PLS_INTEGER := -20016;
MSG_UNSUPPORTED_DATA_TYPE VARCHAR2(4000) := 'Unsupported data type';
PRAGMA EXCEPTION_INIT( ERR_UNSUPPORTED_DATA_TYPE
,CODE_UNSUPPORTED_DATA_TYPE);
ERR_MISSING_SOURCE_KEY EXCEPTION;
CODE_MISSING_SOURCE_KEY CONSTANT PLS_INTEGER := -20017;
MSG_MISSING_SOURCE_KEY VARCHAR2(4000) := 'The Source was not found in parent table A_SOURCE';
PRAGMA EXCEPTION_INIT( ERR_MISSING_SOURCE_KEY
,CODE_MISSING_SOURCE_KEY);
ERR_NULL_SOURCE_FILE_CONFIG_KEY EXCEPTION;
CODE_NULL_SOURCE_FILE_CONFIG_KEY CONSTANT PLS_INTEGER := -20018;
MSG_NULL_SOURCE_FILE_CONFIG_KEY VARCHAR2(4000) := 'No entry in A_SOURCE_FILE_CONFIG table for specified A_SOURCE_FILE_CONFIG_KEY';
PRAGMA EXCEPTION_INIT( ERR_NULL_SOURCE_FILE_CONFIG_KEY
,CODE_NULL_SOURCE_FILE_CONFIG_KEY);
ERR_DUPLICATED_SOURCE_KEY EXCEPTION;
CODE_DUPLICATED_SOURCE_KEY CONSTANT PLS_INTEGER := -20019;
MSG_DUPLICATED_SOURCE_KEY VARCHAR2(4000) := 'The Source already exists in the A_SOURCE table';
PRAGMA EXCEPTION_INIT( ERR_DUPLICATED_SOURCE_KEY
,CODE_DUPLICATED_SOURCE_KEY);
ERR_MISSING_CONTAINER_CONFIG EXCEPTION;
CODE_MISSING_CONTAINER_CONFIG CONSTANT PLS_INTEGER := -20020;
MSG_MISSING_CONTAINER_CONFIG VARCHAR2(4000) := 'No match in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID';
PRAGMA EXCEPTION_INIT( ERR_MISSING_CONTAINER_CONFIG
,CODE_MISSING_CONTAINER_CONFIG);
ERR_MULTIPLE_CONTAINER_ENTRIES EXCEPTION;
CODE_MULTIPLE_CONTAINER_ENTRIES CONSTANT PLS_INTEGER := -20021;
MSG_MULTIPLE_CONTAINER_ENTRIES VARCHAR2(4000) := 'Multiple matches in A_SOURCE_FILE_CONFIG table where SOURCE_FILE_TYPE=''CONTAINER'' and specified SOURCE_FILE_ID';
PRAGMA EXCEPTION_INIT( ERR_MULTIPLE_CONTAINER_ENTRIES
,CODE_MULTIPLE_CONTAINER_ENTRIES);
ERR_WRONG_DESTINATION_PARAM EXCEPTION;
CODE_WRONG_DESTINATION_PARAM CONSTANT PLS_INTEGER := -20022;
MSG_WRONG_DESTINATION_PARAM VARCHAR2(4000) := 'Wrong destination parameter provided.';
PRAGMA EXCEPTION_INIT( ERR_WRONG_DESTINATION_PARAM
,CODE_WRONG_DESTINATION_PARAM);
ERR_FILE_NOT_EXISTS_ON_CLOUD EXCEPTION;
CODE_FILE_NOT_EXISTS_ON_CLOUD CONSTANT PLS_INTEGER := -20023;
MSG_FILE_NOT_EXISTS_ON_CLOUD VARCHAR2(4000) := 'File not exists on cloud.';
PRAGMA EXCEPTION_INIT( ERR_FILE_NOT_EXISTS_ON_CLOUD
,CODE_FILE_NOT_EXISTS_ON_CLOUD);
ERR_FILE_ALREADY_REGISTERED EXCEPTION;
CODE_FILE_ALREADY_REGISTERED CONSTANT PLS_INTEGER := -20024;
MSG_FILE_ALREADY_REGISTERED VARCHAR2(4000) := 'File already registered in A_SOURCE_FILE_RECEIVED table.';
PRAGMA EXCEPTION_INIT( ERR_FILE_ALREADY_REGISTERED
,CODE_FILE_ALREADY_REGISTERED);
ERR_WRONG_DATE_TIMESTAMP_FORMAT EXCEPTION;
CODE_WRONG_DATE_TIMESTAMP_FORMAT CONSTANT PLS_INTEGER := -20025;
MSG_WRONG_DATE_TIMESTAMP_FORMAT VARCHAR2(4000) := 'Provided DATE or TIMESTAMP format has errors (possible duplicated codes, ex: ''DD'').';
PRAGMA EXCEPTION_INIT( ERR_WRONG_DATE_TIMESTAMP_FORMAT
,CODE_WRONG_DATE_TIMESTAMP_FORMAT);
ERR_ENVIRONMENT_NOT_SET EXCEPTION;
CODE_ENVIRONMENT_NOT_SET CONSTANT PLS_INTEGER := -20026;
MSG_ENVIRONMENT_NOT_SET VARCHAR2(4000) := 'EnvironmentID not set'
||cgBL||' Information about environment is needed to get proper configuration values.'
||cgBL||' It can be set up in two different ways:'
||cgBL||' 1. Set it on session level: execute DBMS_SESSION.SET_IDENTIFIER (client_id => ''dev'')'
||cgBL||' 2. Set it on configuration level: Insert into CT_MRDS.A_FILE_MANAGER_CONFIG (ENVIRONMENT_ID,CONFIG_VARIABLE,CONFIG_VARIABLE_VALUE) values (''default'',''environment_id'',''dev'')'
||cgBL||' Session level setup (1.) takes precedence over configuration level one (2.)'
;
PRAGMA EXCEPTION_INIT( ERR_ENVIRONMENT_NOT_SET
,CODE_ENVIRONMENT_NOT_SET);
ERR_CONFIG_VARIABLE_NOT_SET EXCEPTION;
CODE_CONFIG_VARIABLE_NOT_SET CONSTANT PLS_INTEGER := -20027;
MSG_CONFIG_VARIABLE_NOT_SET VARCHAR2(4000) := 'Missing configuration value in A_FILE_MANAGER_CONFIG';
PRAGMA EXCEPTION_INIT( ERR_CONFIG_VARIABLE_NOT_SET
,CODE_CONFIG_VARIABLE_NOT_SET);
ERR_NOT_INPUT_SOURCE_FILE_TYPE EXCEPTION;
CODE_NOT_INPUT_SOURCE_FILE_TYPE CONSTANT PLS_INTEGER := -20028;
MSG_NOT_INPUT_SOURCE_FILE_TYPE VARCHAR2(4000) := 'Archival can be executed only for A_SOURCE_FILE_CONFIG_KEY where SOURCE_FILE_TYPE=''INPUT''';
PRAGMA EXCEPTION_INIT( ERR_NOT_INPUT_SOURCE_FILE_TYPE
,CODE_NOT_INPUT_SOURCE_FILE_TYPE);
ERR_EXP_DATA_FOR_ARCH_FAILED EXCEPTION;
CODE_EXP_DATA_FOR_ARCH_FAILED CONSTANT PLS_INTEGER := -20029;
MSG_EXP_DATA_FOR_ARCH_FAILED VARCHAR2(4000) := 'Export data for archival failed.';
PRAGMA EXCEPTION_INIT( ERR_EXP_DATA_FOR_ARCH_FAILED
,CODE_EXP_DATA_FOR_ARCH_FAILED);
ERR_RESTORE_FILE_FROM_TRASH EXCEPTION;
CODE_RESTORE_FILE_FROM_TRASH CONSTANT PLS_INTEGER := -20030;
MSG_RESTORE_FILE_FROM_TRASH VARCHAR2(4000) := 'Unexpected issues occured while archival process. Restoration of exported files failed.';
PRAGMA EXCEPTION_INIT( ERR_RESTORE_FILE_FROM_TRASH
,CODE_RESTORE_FILE_FROM_TRASH);
ERR_CHANGE_STAT_TO_ARCHIVED_FAILED EXCEPTION;
CODE_CHANGE_STAT_TO_ARCHIVED_FAILED CONSTANT PLS_INTEGER := -20031;
MSG_CHANGE_STAT_TO_ARCHIVED_FAILED VARCHAR2(4000) := 'Failed to change file status to: ARCHIVED in A_SOURCE_FILE_RECEIVED table.';
PRAGMA EXCEPTION_INIT( ERR_CHANGE_STAT_TO_ARCHIVED_FAILED
,CODE_CHANGE_STAT_TO_ARCHIVED_FAILED);
ERR_MOVE_FILE_TO_TRASH_FAILED EXCEPTION;
CODE_MOVE_FILE_TO_TRASH_FAILED CONSTANT PLS_INTEGER := -20032;
MSG_MOVE_FILE_TO_TRASH_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.';
PRAGMA EXCEPTION_INIT( ERR_MOVE_FILE_TO_TRASH_FAILED
,CODE_MOVE_FILE_TO_TRASH_FAILED);
ERR_DROP_EXPORTED_FILES_FAILED EXCEPTION;
CODE_DROP_EXPORTED_FILES_FAILED CONSTANT PLS_INTEGER := -20033;
MSG_DROP_EXPORTED_FILES_FAILED VARCHAR2(4000) := 'FAILED to move file to TRASH before DROPPING it.';
PRAGMA EXCEPTION_INIT( ERR_DROP_EXPORTED_FILES_FAILED
,CODE_DROP_EXPORTED_FILES_FAILED);
ERR_INVALID_BUCKET_AREA EXCEPTION;
CODE_INVALID_BUCKET_AREA CONSTANT PLS_INTEGER := -20034;
MSG_INVALID_BUCKET_AREA VARCHAR2(4000) := 'Invalid bucket area specified. Valid values: INBOX, ODS, DATA, ARCHIVE';
PRAGMA EXCEPTION_INIT( ERR_INVALID_BUCKET_AREA
,CODE_INVALID_BUCKET_AREA);
ERR_UNKNOWN EXCEPTION;
CODE_UNKNOWN CONSTANT PLS_INTEGER := -20999;
MSG_UNKNOWN VARCHAR2(4000) := 'Unknown Error Occured';
PRAGMA EXCEPTION_INIT( ERR_UNKNOWN
,CODE_UNKNOWN);
---------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------
/**
* @name LOG_PROCESS_EVENT
* @desc Insert a new log record into A_PROCESS_LOG table.
* Also outputs to console if gvConsoleLoggingEnabled = 'ON'.
* Respects logging level configuration (gvMinLogLevel).
* @example ENV_MANAGER.LOG_PROCESS_EVENT('Process completed successfully', 'INFO', 'pParam1=value1');
* @ex_rslt Record inserted into A_PROCESS_LOG table and optionally displayed in console output
**/
PROCEDURE LOG_PROCESS_EVENT (
pLogMessage VARCHAR2
,pLogLevel VARCHAR2 DEFAULT 'ERROR'
,pParameters VARCHAR2 DEFAULT NULL
,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER'
);
/**
* @name LOG_PROCESS_ERROR
* @desc Insert a detailed error record into A_PROCESS_LOG table with full stack trace, backtrace, and call stack.
* This procedure captures comprehensive error information for debugging purposes while
* allowing clean user-facing error messages to be raised separately.
* @param pLogMessage - Base error message description
* @param pParameters - Procedure parameters for context
* @param pProcessName - Name of the calling process/package
* @ex_rslt Record inserted into A_PROCESS_LOG table with complete error stack information
*/
PROCEDURE LOG_PROCESS_ERROR (
pLogMessage VARCHAR2
,pParameters VARCHAR2 DEFAULT NULL
,pProcessName VARCHAR2 DEFAULT 'FILE_MANAGER'
);
/**
* @name INIT_ERRORS
* @desc Loads data into Errors array.
* Errors array is a list of Record(Error_Code, Error_Message) index by Error_Code.
* Called automatically during package initialization.
* @example Called automatically when package is first referenced
* @ex_rslt Errors array populated with all error codes and messages
**/
PROCEDURE INIT_ERRORS;
/**
* @name GET_DEFAULT_ENV
* @desc It returns string with name of default environment.
* Return string is A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID value.
* @example select ENV_MANAGER.GET_DEFAULT_ENV() from dual;
* @ex_rslt dev
**/
FUNCTION GET_DEFAULT_ENV
RETURN VARCHAR2;
/**
* @name INIT_VARIABLES
* @desc For specified pEnv parameter (A_FILE_MANAGER_CONFIG.ENVIRONMENT_ID)
* Assign values to following global package variables:
* - gvNameSpace
* - gvRegion
* - gvCredentialName
* - gvInboxBucketName
* - gvDataBucketName
* - gvArchiveBucketName
* - gvInboxBucketUri
* - gvDataBucketUri
* - gvArchiveBucketUri
* - gvLoggingEnabled
* - gvMinLogLevel
* - gvDefaultDateFormat
* - gvConsoleLoggingEnabled
**/
PROCEDURE INIT_VARIABLES(
pEnv VARCHAR2
);
/**
* @name GET_ERROR_MESSAGE
* @desc It returns string with error message for specified pCode (Error_Code).
* Error message is take from Errors Array loaded by INIT_ERRORS procedure
* @example select ENV_MANAGER.GET_ERROR_MESSAGE(pCode => -20009) from dual;
* @ex_rslt File not found on the cloud
**/
FUNCTION GET_ERROR_MESSAGE(
pCode PLS_INTEGER
) RETURN VARCHAR2;
/**
* @name GET_ERROR_STACK
* @desc It returns string with all possible error stack info.
* Error message is take from Errors Array loaded by INIT_ERRORS procedure
* @example
* select ENV_MANAGER.GET_ERROR_STACK(
* pFormat => 'OUTPUT'
* ,pCode => -20009
* ,pSourceFileReceivedKey => NULL)
* from dual
* @ex_rslt
* ------------------------------------------------------+
* Error Message:
* ORA-0000: normal, successful completion
* -------------------------------------------------------
* Error Stack:
* -------------------------------------------------------
* Error Backtrace:
* ------------------------------------------------------+
**/
FUNCTION GET_ERROR_STACK(
pFormat VARCHAR2
,pCode PLS_INTEGER
,pSourceFileReceivedKey CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL
) RETURN VARCHAR2;
/**
* @name FORMAT_PARAMETERS
* @desc Formats parameter list for logging purposes.
* Converts SYS.ODCIVARCHAR2LIST to formatted string with proper NULL handling.
* @example select ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('param1=value1', 'param2=NULL')) from dual;
* @ex_rslt param1=value1 ,
* param2=NULL
**/
FUNCTION FORMAT_PARAMETERS(
pParameterList SYS.ODCIVARCHAR2LIST
) RETURN VARCHAR2;
/**
* @name ANALYZE_VALIDATION_ERRORS
* @desc Analyzes CSV validation errors and generates detailed diagnostic report.
* Compares CSV structure with template table and provides specific error analysis.
* Includes suggested solutions for common validation issues.
* @param pValidationLogTable - Name of validation log table (e.g., VALIDATE$242_LOG)
* @param pTemplateSchema - Schema of template table (e.g., CT_ET_TEMPLATES)
* @param pTemplateTable - Name of template table (e.g., MOCK_PROC_TABLE)
* @param pCsvFileUri - URI of CSV file being validated
* @example SELECT ENV_MANAGER.ANALYZE_VALIDATION_ERRORS('VALIDATE$242_LOG', 'CT_ET_TEMPLATES', 'MOCK_PROC_TABLE', 'https://...') FROM DUAL;
* @ex_rslt Detailed validation analysis report with column mismatches and solutions
**/
FUNCTION ANALYZE_VALIDATION_ERRORS(
pValidationLogTable VARCHAR2,
pTemplateSchema VARCHAR2,
pTemplateTable VARCHAR2,
pCsvFileUri VARCHAR2
) RETURN VARCHAR2;
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_VERSION
* @desc Returns the current version number of the ENV_MANAGER package.
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
* @example SELECT ENV_MANAGER.GET_VERSION() FROM DUAL;
* @ex_rslt 3.0.0
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* @name GET_BUILD_INFO
* @desc Returns comprehensive build information including version, build date, and author.
* Formatted for display in logs or monitoring systems.
* @example SELECT ENV_MANAGER.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: ENV_MANAGER
* Version: 3.0.0
* Build Date: 2025-10-22 16:00:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* @name GET_VERSION_HISTORY
* @desc Returns complete version history with all releases and changes.
* Shows evolution of package features over time.
* @example SELECT ENV_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
* @ex_rslt ENV_MANAGER Version History:
* 3.0.0 (2025-10-22): Added package versioning system...
* 2.1.0 (2025-10-15): Added ANALYZE_VALIDATION_ERRORS function...
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
/**
* @name GET_PACKAGE_VERSION_INFO
* @desc Universal function to get formatted version information for any package.
* This centralized function is used by all packages in the system.
* @param pPackageName - Name of the package
* @param pVersion - Version string (MAJOR.MINOR.PATCH format)
* @param pBuildDate - Build date timestamp
* @param pAuthor - Package author name
* @example SELECT ENV_MANAGER.GET_PACKAGE_VERSION_INFO('FILE_MANAGER', '2.1.0', '2025-10-22 15:00:00', 'Grzegorz Michalski') FROM DUAL;
* @ex_rslt Package: FILE_MANAGER
* Version: 2.1.0
* Build Date: 2025-10-22 15:00:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_PACKAGE_VERSION_INFO(
pPackageName VARCHAR2,
pVersion VARCHAR2,
pBuildDate VARCHAR2,
pAuthor VARCHAR2
) RETURN VARCHAR2;
/**
* @name FORMAT_VERSION_HISTORY
* @desc Universal function to format version history for any package.
* Adds package name header and proper formatting.
* @param pPackageName - Name of the package
* @param pVersionHistory - Complete version history text
* @example SELECT ENV_MANAGER.FORMAT_VERSION_HISTORY('FILE_MANAGER', '2.1.0 (2025-10-22): Export procedures...') FROM DUAL;
* @ex_rslt FILE_MANAGER Version History:
* 2.1.0 (2025-10-22): Export procedures...
**/
FUNCTION FORMAT_VERSION_HISTORY(
pPackageName VARCHAR2,
pVersionHistory VARCHAR2
) RETURN VARCHAR2;
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE HASH + CHANGE DETECTION FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name CALCULATE_PACKAGE_HASH
* @desc Calculates SHA256 hash of package source code from ALL_SOURCE.
* Returns hash for both SPEC and BODY (if exists).
* Used for automatic change detection.
* @param pPackageOwner - Schema owner of the package
* @param pPackageName - Name of the package
* @param pPackageType - Type of package code ('PACKAGE' for SPEC, 'PACKAGE BODY' for BODY)
* @example SELECT ENV_MANAGER.CALCULATE_PACKAGE_HASH('CT_MRDS', 'FILE_MANAGER', 'PACKAGE') FROM DUAL;
* @ex_rslt A7B3C5D9E8F1234567890ABCDEF... (64-character SHA256 hash)
**/
FUNCTION CALCULATE_PACKAGE_HASH(
pPackageOwner VARCHAR2,
pPackageName VARCHAR2,
pPackageType VARCHAR2 -- 'PACKAGE' or 'PACKAGE BODY'
) RETURN VARCHAR2;
/**
* @name TRACK_PACKAGE_VERSION
* @desc Records package version and source code hash in A_PACKAGE_VERSION_TRACKING table.
* Automatically detects if source code changed without version update.
* Should be called after every package deployment.
* @param pPackageOwner - Schema owner of the package
* @param pPackageName - Name of the package
* @param pPackageVersion - Current version from PACKAGE_VERSION constant
* @param pPackageBuildDate - Build date from PACKAGE_BUILD_DATE constant
* @param pPackageAuthor - Author from PACKAGE_AUTHOR constant
* @example EXEC ENV_MANAGER.TRACK_PACKAGE_VERSION('CT_MRDS', 'FILE_MANAGER', '3.2.0', '2025-10-22 16:30:00', 'Grzegorz Michalski');
* @ex_rslt Record inserted into A_PACKAGE_VERSION_TRACKING with change detection status
**/
PROCEDURE TRACK_PACKAGE_VERSION(
pPackageOwner VARCHAR2,
pPackageName VARCHAR2,
pPackageVersion VARCHAR2,
pPackageBuildDate VARCHAR2,
pPackageAuthor VARCHAR2
);
/**
* @name CHECK_PACKAGE_CHANGES
* @desc Checks if package source code has changed since last tracking.
* Compares current hash with last recorded hash in A_PACKAGE_VERSION_TRACKING.
* Returns detailed change detection report.
* @param pPackageOwner - Schema owner of the package
* @param pPackageName - Name of the package
* @example SELECT ENV_MANAGER.CHECK_PACKAGE_CHANGES('CT_MRDS', 'FILE_MANAGER') FROM DUAL;
* @ex_rslt WARNING: Package changed without version update!
* Last Version: 3.2.0
* Current Hash (SPEC): A7B3C5D9...
* Last Hash (SPEC): B8C4D6E0...
* RECOMMENDATION: Update PACKAGE_VERSION and PACKAGE_BUILD_DATE
**/
FUNCTION CHECK_PACKAGE_CHANGES(
pPackageOwner VARCHAR2,
pPackageName VARCHAR2
) RETURN VARCHAR2;
/**
* @name GET_PACKAGE_HASH_INFO
* @desc Returns formatted information about package hash and tracking history.
* Includes current hash, last tracked hash, and change detection status.
* @param pPackageOwner - Schema owner of the package
* @param pPackageName - Name of the package
* @example SELECT ENV_MANAGER.GET_PACKAGE_HASH_INFO('CT_MRDS', 'FILE_MANAGER') FROM DUAL;
* @ex_rslt Package: CT_MRDS.FILE_MANAGER
* Current Version: 3.2.0
* Current Hash (SPEC): A7B3C5D9...
* Last Tracked: 2025-10-22 16:30:00
* Status: OK - No changes detected
**/
FUNCTION GET_PACKAGE_HASH_INFO(
pPackageOwner VARCHAR2,
pPackageName VARCHAR2
) RETURN VARCHAR2;
END ENV_MANAGER;

443
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_ARCHIVER.pkb Normal file
View File

@@ -0,0 +1,443 @@
create or replace PACKAGE BODY CT_MRDS.FILE_ARCHIVER
AS
----------------------------------------------------------------------------------------------------
FUNCTION GET_TABLE_STAT(pSourceFileConfigKey IN NUMBER)
RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
IS
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vCount PLS_INTEGER;
vSourceFileType CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT count(*) , min(SOURCE_FILE_TYPE)
INTO vCount, vSourceFileType
FROM CT_MRDS.A_TABLE_STAT s
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
IF vCount=0 and vSourceFileType='INPUT' THEN
GATHER_TABLE_STAT(pSourceFileConfigKey);
END IF;
BEGIN
SELECT *
INTO vTableStat
FROM CT_MRDS.A_TABLE_STAT
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
-- EXCEPTION
-- WHEN NO_DATA_FOUND THEN
--
END;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vTableStat;
END GET_TABLE_STAT;
----------------------------------------------------------------------------------------------------
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
)
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableStat CT_MRDS.A_TABLE_STAT%ROWTYPE;
vQuery VARCHAR2(4000);
vTableName VARCHAR2(200);
vUri VARCHAR2(1000);
vfiles T_FILENAMES;
vFilename VARCHAR2(300);
vOperationId NUMBER := -1;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vArchivalTriggeredBy VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
vProcessControlStatus VARCHAR2(60) := 'OK';
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
end if;
if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
vTableStat := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
end if;
if vTableStat.OVER_ARCH_THRESOLD_FILE_COUNT >= vSourceFileConfig.FILES_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := 'FILES_COUNT';
elsif vTableStat.OVER_ARCH_THRESOLD_ROW_COUNT >= vSourceFileConfig.ROWS_COUNT_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', ROWS_COUNT';
elsif vTableStat.OVER_ARCH_THRESOLD_SIZE >= vSourceFileConfig.BYTES_SUM_OVER_ARCHIVE_THRESHOLD then vArchivalTriggeredBy := vArchivalTriggeredBy||', BYTES_SUM';
else ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival triggers reached','INFO');
end if;
if LENGTH(vArchivalTriggeredBy)>0 THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Archival Triggered By: '||vArchivalTriggeredBy,'INFO');
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
vQuery := '
select t_filename(
file$name
,file$path
, to_char(h.workflow_start,''yyyy'')
, to_char(h.workflow_start,''mm'')
)
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
where extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD
;
-- Get all files that will be archived into "vfiles" collection ("regular data files")
execute immediate vQuery bulk collect into vfiles;
-- Start EXPORT "regular data files" to parquet and DROP "csv"
FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
vQuery:=
'select
s.*
-- ,r.partition_year
-- ,r.partition_month
from '|| vTableName ||' s
join CT_MRDS.A_SOURCE_FILE_RECEIVED r
on s.file$name = r.source_file_name
and r.a_source_file_config_key = '||pSourceFileConfigKey||'
and r.partition_year='''||ym_loop.year||'''
and r.partition_month='''||ym_loop.month||'''
and r.PROCESSING_STATUS = ''INGESTED''
'
;
vUri := FILE_MANAGER.GET_BUCKET_URI('ARCHIVE')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/PARTITION_YEAR='||ym_loop.year||'/PARTITION_MONTH='||ym_loop.month||'/';
ENV_MANAGER.LOG_PROCESS_EVENT('Start Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list' ,'DEBUG',vUri);
ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query' ,'DEBUG',vQuery);
BEGIN
DBMS_CLOUD.EXPORT_DATA(
credential_name => ENV_MANAGER.gvCredentialName,
file_uri_list => vUri||'d' ,
format => json_object('type' value 'parquet'),
query => vQuery,
operation_id => vOperationId
);
EXCEPTION
WHEN OTHERS THEN
vProcessControlStatus :='EXPORT_FAILURE';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED);
END;
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');
-- Get USER_LOAD_OPERATIONS info
select *
into vUserLoadOperations
from USER_LOAD_OPERATIONS
where id = vOperationId;
IF vUserLoadOperations.STATUS <>'COMPLETED' THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Export ended with status '||vUserLoadOperations.STATUS);
ELSIF vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED,
ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED ||cgBL|| ' Zero rows were exported.');
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to archival file for YEAR_MONTH: 2025_01','INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
END IF;
SELECT
object_name
into vFilename
from DBMS_CLOUD.LIST_OBJECTS(
credential_name => 'OCI$RESOURCE_PRINCIPAL',
location_uri => vUri)
where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name, '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7'))
between vUserLoadOperations.START_TIME
and vUserLoadOperations.UPDATE_TIME
;
-- Try to drop EXPORTED FILES ("regular data files")
BEGIN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop
-- first change of status
BEGIN
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'ARCHIVED'
,ARCH_FILE_NAME = vUri||vFilename
WHERE r.a_source_file_config_key= pSourceFileConfigKey
AND r.source_file_name = f.filename
AND r.processing_status = 'INGESTED'
;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE';
-- move file to trash before dropping
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => f.pathname||'/'||f.filename,
target_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
rollback;
vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
END;
EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
commit;
END LOOP;
--------------------------------------------------------------------
-- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
IF vProcessControlStatus = 'OK' THEN
FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
--Drop file from TRASH
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
END LOOP;
--ROLLBACK PART
--ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
FOR f in ( SELECT vf.filename, vf.pathname
FROM TABLE(vfiles) vf
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
ON r.source_file_name = vf.filename
AND r.a_source_file_config_key = pSourceFileConfigKey
AND r.PROCESSING_STATUS = 'ARCHIVED'
AND vf.year = ym_loop.year
AND vf.month = ym_loop.month
) LOOP
BEGIN
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
target_object_uri => f.pathname||'/'||f.filename,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
SET PROCESSING_STATUS = 'INGESTED'
,ARCH_FILE_NAME = NULL
WHERE r.a_source_file_config_key = pSourceFileConfigKey
AND r.source_file_name = f.filename
;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
END;
END LOOP;
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => vUri||vFilename);
ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_ARCHIVED_FAILURE' THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED, 'ERROR', vParameters);
DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
object_uri => vUri||vFilename);
ENV_MANAGER.LOG_PROCESS_EVENT('Archival PARQUET file dropped.','DEBUG', vUri||vFilename);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
END IF;
EXCEPTION
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_ARCHIVED_FAILED);
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);
WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error during archiving process','ERROR');
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
END;
-- END of "Try to drop EXPORTED FILES"
ENV_MANAGER.LOG_PROCESS_EVENT('All archived files had been dropped.','INFO');
ENV_MANAGER.LOG_PROCESS_EVENT('End Archiving for YEAR_MONTH: '||ym_loop.year||'_'||ym_loop.month ,'INFO');
END LOOP; --ym_loop end (YEAR_MONTH)
COMMIT;
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Non of archival thresholds reached. Skip archiving.'||vArchivalTriggeredBy,'INFO');
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_ARCH_FAILED THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_ARCH_FAILED , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_ARCH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_ARCHIVED_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_ARCHIVED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END ARCHIVE_TABLE_DATA;
----------------------------------------------------------------------------------------------------
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vStats CT_MRDS.A_TABLE_STAT%ROWTYPE;
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vTableName VARCHAR2(200);
vQuery VARCHAR2(32000);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
vQuery :=
'with tmp as (
select
s.*
,file$name as filename
,h.workflow_start
, to_char(h.workflow_start,''yyyy'') as year
, to_char(h.workflow_start,''mm'') as month
from '||vTableName||' s
join CT_MRDS.a_workflow_history h
on s.a_workflow_history_key = h.a_workflow_history_key
)
, tmp_gr as (
select
filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
from tmp
group by filename
)
select
NULL as A_TABLE_STAT_KEY
,'||pSourceFileConfigKey||' as A_SOURCE_FILE_CONFIG_KEY
,'''||vTableName||''' as TABLE_NAME
,count(*) as FILE_COUNT
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then 1 else 0 end) as OLD_FILE_COUNT
,sum (row_count_per_file) as ROW_COUNT
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then row_count_per_file else 0 end) as OLD_ROW_COUNT
,sum(r.bytes) as BYTES
,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' then r.bytes else 0 end) as OLD_BYTES
,'||vSourceFileConfig.DAYS_FOR_ARCHIVE_THRESHOLD||' as DAYS_FOR_ARCHIVE_THRESHOLD
,systimestamp as CREATED
from tmp_gr t
join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
credential_name => '''||ENV_MANAGER.gvCredentialName||''',
location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/''
)
) r
on t.filename = r.object_name'
;
ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
execute immediate vQuery into vStats;
vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
insert into A_TABLE_STAT_HIST values vStats;
delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
insert into A_TABLE_STAT values vStats;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GATHER_TABLE_STAT;
----------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION
RETURN VARCHAR2
IS
BEGIN
RETURN PACKAGE_VERSION;
END GET_VERSION;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUILD_INFO
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
pPackageName => 'FILE_ARCHIVER',
pVersion => PACKAGE_VERSION,
pBuildDate => PACKAGE_BUILD_DATE,
pAuthor => PACKAGE_AUTHOR
);
END GET_BUILD_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION_HISTORY
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
pPackageName => 'FILE_ARCHIVER',
pVersionHistory => VERSION_HISTORY
);
END GET_VERSION_HISTORY;
----------------------------------------------------------------------------------------------------
END;
/

90
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_ARCHIVER.pkg Normal file
View File

@@ -0,0 +1,90 @@
create or replace PACKAGE CT_MRDS.FILE_ARCHIVER
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '2.0.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-10-22 16:45:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'2.0.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
'1.5.0 (2025-10-18): Enhanced ARCHIVE_TABLE_DATA with Hive-style partitioning support' || CHR(13)||CHR(10) ||
'1.0.0 (2025-09-15): Initial release with table archival and statistics gathering';
cgBL CONSTANT VARCHAR2(2) := ENV_MANAGER.cgBL;
/**
* @name ARCHIVE_TABLE_DATA
* @desc Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
* Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
* Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
**/
PROCEDURE ARCHIVE_TABLE_DATA (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
);
/**
* @name GATHER_TABLE_STAT
* @desc Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
* Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
**/
PROCEDURE GATHER_TABLE_STAT (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
);
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_VERSION
* @desc Returns the current version number of the FILE_ARCHIVER package.
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
* @example SELECT FILE_ARCHIVER.GET_VERSION() FROM DUAL;
* @ex_rslt 2.0.0
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* @name GET_BUILD_INFO
* @desc Returns comprehensive build information including version, build date, and author.
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
* @example SELECT FILE_ARCHIVER.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: FILE_ARCHIVER
* Version: 2.0.0
* Build Date: 2025-10-22 16:45:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* @name GET_VERSION_HISTORY
* @desc Returns complete version history with all releases and changes.
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
* @example SELECT FILE_ARCHIVER.GET_VERSION_HISTORY() FROM DUAL;
* @ex_rslt FILE_ARCHIVER Version History:
* 2.0.0 (2025-10-22): Added package versioning system...
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END;
/

2220
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkb Normal file
View File

File diff suppressed because it is too large Load Diff

697
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/FILE_MANAGER.pkg Normal file
View File

@@ -0,0 +1,697 @@
create or replace PACKAGE CT_MRDS.FILE_MANAGER
AUTHID CURRENT_USER
AS
/**
* General comment for package: Please put comments for functions and procedures as shown in below example.
* It is a standard.
* The structure of comment is used by GET_PACKAGE_DOCUMENTATION function
* which returns documentation text for confluence page (to Copy-Paste it).
**/
-- Example comment:
/**
* @name EX_PROCEDURE_NAME
* @desc Procedure description
* @example select FILE_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
* @ex_rslt Example Result
**/
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '3.4.0';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-27 14:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) ||
'3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
'3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
'3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
'3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
'3.1.0 (2025-10-20): Enhanced PROCESS_SOURCE_FILE with 6-step validation workflow' || CHR(13)||CHR(10) ||
'3.0.0 (2025-10-15): Separated export procedures into dedicated DATA_EXPORTER package' || CHR(13)||CHR(10) ||
'2.5.0 (2025-10-10): Added DELETE_SOURCE_CASCADE for safe configuration removal' || CHR(13)||CHR(10) ||
'2.0.0 (2025-09-25): Added official path patterns support (INBOX 3-level, ODS 2-level, ARCHIVE 2-level)' || CHR(13)||CHR(10) ||
'1.0.0 (2025-09-01): Initial release with file processing and validation capabilities';
TYPE tSourceFileReceived IS RECORD
(
A_SOURCE_FILE_RECEIVED_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE,
A_SOURCE_FILE_CONFIG_KEY CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY%TYPE,
SOURCE_FILE_PREFIX_INBOX VARCHAR2(430),
SOURCE_FILE_PREFIX_ODS VARCHAR2(430),
SOURCE_FILE_PREFIX_QUARANTINE VARCHAR2(430),
SOURCE_FILE_PREFIX_ARCHIVE VARCHAR2(430),
SOURCE_FILE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME%TYPE,
RECEPTION_DATE CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE%TYPE,
PROCESSING_STATUS CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS%TYPE,
EXTERNAL_TABLE_NAME CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME%TYPE
);
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
vgSourceFileConfigKey PLS_INTEGER;
vgMsgTmp VARCHAR2(32000);
---------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_SOURCE_FILE_CONFIG
* @desc Get source file type by matching the source file name against source file type naming patterns
* or by specifying the id of a received source file.
* @example ...
* @ex_rslt "CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE"
**/
FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL
, pSourceFileReceivedKey IN NUMBER DEFAULT NULL
, pSourceFileConfigKey IN NUMBER DEFAULT NULL)
RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
/**
* @name REGISTER_SOURCE_FILE_RECEIVED
* @desc Register a newly received source file in A_SOURCE_FILE_RECEIVED table.
* This overload automatically determines source file type from the file name.
* It returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
* @ex_rslt 3245
**/
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
pSourceFileReceivedName IN VARCHAR2
)
RETURN PLS_INTEGER;
/**
* @name REGISTER_SOURCE_FILE_RECEIVED
* @desc Register a new new source file in A_SOURCE_FILE_RECEIVED table based on pSourceFileReceivedName and pSourceFileConfig.
* Then it returns the value of A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY column for newly added record.
* @example vSourceFileReceivedKey := FILE_MANAGER.REGISTER_SOURCE_FILE_RECEIVED(
* pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv'
* ,pSourceFileConfig => ...A_SOURCE_FILE_CONFIG%ROWTYPE... );
* @ex_rslt 3245
**/
FUNCTION REGISTER_SOURCE_FILE_RECEIVED (
pSourceFileReceivedName IN VARCHAR2,
pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
)
RETURN PLS_INTEGER;
/**
* @name SET_SOURCE_FILE_RECEIVED_STATUS
* @desc Set status of file in A_SOURCE_FILE_RECEIVED table - PROCESSING_STATUS column
* based on A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY
* and provided value of pStatus parameter
* @example exec FILE_MANAGER.SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => 377, pStatus => 'READY_FOR_INGESTION');
**/
PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(
pSourceFileReceivedKey IN PLS_INTEGER,
pStatus IN VARCHAR2
);
/**
* @name GET_EXTERNAL_TABLE_COLUMNS
* @desc Function used to get string with all table columns definitions based on pTargetTableTemplate "TEMPLATE TABLE" name.
* It used for creating "EXTERNAL TABLE" using CREATE_EXTERNAL_TABLE procedure.
* @example select FILE_MANAGER.GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER') from dual;
* @ex_rslt "A_KEY" NUMBER(38,0) NOT NULL ENABLE,
* "A_WORKFLOW_HISTORY_KEY" NUMBER(38,0) NOT NULL ENABLE,
* "REV_NUMBER" NUMBER(28,0),
* "REF_DATE" DATE,
* "FREE_TEXT" VARCHAR2(1000 CHAR),
* "MLF_BS_TOTAL" NUMBER(28,10),
* "DF_BS_TOTAL" NUMBER(28,10),
* "MLF_SF_TOTAL" NUMBER(28,10),
* "DF_SF_TOTAL" NUMBER(28,10)
**/
FUNCTION GET_EXTERNAL_TABLE_COLUMNS (
pTargetTableTemplate IN VARCHAR2
)
RETURN CLOB;
/**
* @name CREATE_EXTERNAL_TABLE
* @desc A wrapper procedure for DBMS_CLOUD.CREATE_EXTERNAL_TABLE which creates External Table
* MARS-1049: Added pEncoding parameter for CSV character set specification
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252')
* If provided, adds CHARACTERSET clause to external table definition
* @example
* begin
* FILE_MANAGER.CREATE_EXTERNAL_TABLE(
* pTableName => 'STANDING_FACILITIES_HEADER',
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER',
* pPrefix => 'ODS/LM/STANDING_FACILITIES_HEADER/',
* pBucketUri => 'https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/',
* pFileName => NULL,
* pDelimiter => ',',
* pEncoding => 'UTF8'
* );
* end;
**/
PROCEDURE CREATE_EXTERNAL_TABLE (
pTableName IN VARCHAR2,
pTemplateTableName IN VARCHAR2,
pPrefix IN VARCHAR2,
pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri,
pFileName IN VARCHAR2 DEFAULT NULL,
pDelimiter IN VARCHAR2 DEFAULT ',',
pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NOWY PARAMETR
);
/**
* @name CREATE_EXTERNAL_TABLE
* @desc Creates External Table for single file provided by
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
* @example exec FILE_MANAGER.CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);;
**/
PROCEDURE CREATE_EXTERNAL_TABLE (
pSourceFileReceivedKey IN NUMBER
);
/**
* @name VALIDATE_SOURCE_FILE_RECEIVED
* @desc A wrapper procedure for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
* It validate External table build upon single file
* provided by pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
* @example exec FILE_MANAGER.VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey => 377);
**/
PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED
(
pSourceFileReceivedKey IN NUMBER
);
/**
* @name VALIDATE_EXTERNAL_TABLE
* @desc A wrapper function for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE.
* It validates External Table provided by parameter pTableName.
* It returns: PASSED or FAILED.
* @example
* declare
* vStatus VARCHAR2(100);
* begin
* vStatus := FILE_MANAGER.VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
* end;
*
* @ex_rslt FAILED
**/
FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
RETURN VARCHAR2;
/**
* @name S_VALIDATE_EXTERNAL_TABLE
* @desc A function which checks if SELECT query reterns any rows.
* It trys to selects External Table provided by parameter pTableName.
* It returns: PASSED or FAILED.
* @example
* declare
* vStatus VARCHAR2(100);
* begin
* vStatus := FILE_MANAGER.S_VALIDATE_EXTERNAL_TABLE(pTableName => 'STANDING_FACILITIES_HEADER');
* DBMS_OUTPUT.PUT_LINE('vStatus = '||vStatus);
* end;
*
* @ex_rslt PASSED
**/
FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
RETURN VARCHAR2;
/**
* @name DROP_EXTERNAL_TABLE
* @desc It drops External Table for single file provided by
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
* @example exec FILE_MANAGER.DROP_EXTERNAL_TABLE(pSourceFileReceivedKey => 377);
**/
PROCEDURE DROP_EXTERNAL_TABLE (
pSourceFileReceivedKey IN NUMBER
);
/**
* @name COPY_FILE
* @desc It copies file provided by
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
* into destination provided by pDestination parameter.
* pDestination parameter allowed values are: 'ODS'
* @example exec FILE_MANAGER.COPY_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
**/
PROCEDURE COPY_FILE(
pSourceFileReceivedKey IN NUMBER,
pDestination IN VARCHAR2
);
/**
* @name MOVE_FILE
* @desc It moves file provided by
* pSourceFileReceivedKey parameter (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY)
* into destination provided by pDestination parameter.
* pDestination parameter allowed values are: 'ODS', 'QUARANTINE'
* @example exec FILE_MANAGER.MOVE_FILE(pSourceFileReceivedKey => 377, pDestination => 'ODS');
**/
PROCEDURE MOVE_FILE(
pSourceFileReceivedKey IN NUMBER,
pDestination IN VARCHAR2
);
/**
* @name DELETE_FOLDER_CONTENTS
* @desc It deletes all files from specified folder in the cloud storage.
* The procedure lists all objects in the specified folder prefix and deletes them one by one.
* pBucketArea parameter specifies which bucket to use: 'INBOX', 'DATA', 'ARCHIVE'
* pFolderPrefix parameter specifies the folder path within the bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/')
* @example exec FILE_MANAGER.DELETE_FOLDER_CONTENTS(pBucketArea => 'INBOX', pFolderPrefix => 'C2D/UC_DISSEM/UC_NMA_DISSEM/');
**/
PROCEDURE DELETE_FOLDER_CONTENTS(
pBucketArea IN VARCHAR2,
pFolderPrefix IN VARCHAR2
);
/**
* @name PROCESS_SOURCE_FILE
* @desc It process file provided by pSourceFileReceivedName parameter.
* Ubmrella procedure that calls:
* - REGISTER_SOURCE_FILE_RECEIVED;
* - CREATE_EXTERNAL_TABLE;
* - VALIDATE_SOURCE_FILE_RECEIVED;
* - DROP_EXTERNAL_TABLE;
* - MOVE_FILE;
* @example exec FILE_MANAGER.PROCESS_SOURCE_FILE(pSourceFileReceivedName => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
**/
PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
;
/**
* @name PROCESS_SOURCE_FILE
* @desc It process file provided by pSourceFileReceivedName parameter and return processing result value.
* It returns (success/failure) => 0 / -(value).
* Ubmrella function that calls PROCESS_SOURCE_FILE procedure.
* @example
* declare
* vResult PLS_INTEGER;
* begin
* vResult := CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE(PSOURCEFILERECEIVEDNAME => 'INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM/UC_NMA_DISSEM-277740.csv');
* DBMS_OUTPUT.PUT_LINE('vResult = ' || vResult);
* end;
* @ex_rslt 0
* -20021
**/
FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
RETURN PLS_INTEGER;
/**
* @name GET_DATE_FORMAT
* @desc Returns date format for specified template table name and column name.
* Date is taken from configuration A_COLUMN_DATE_FORMAT table.
* @example select FILE_MANAGER.GET_DATE_FORMAT(
* pTemplateTableName => 'STANDING_FACILITIES_HEADER',
* pColumnName => 'SNAPSHOT_DATE')
* from dual;
* @ex_rslt DD/MM/YYYY HH24:MI:SS
**/
FUNCTION GET_DATE_FORMAT(
pTemplateTableName IN VARCHAR2,
pColumnName IN VARCHAR2
) RETURN VARCHAR2;
/**
* @name GENERATE_EXTERNAL_TABLE_PARAMS
* @desc It builds two strings: pColumnList and pFieldList for specified Template Table name, by parameter: pTemplateTableName.
* @example
* declare
* vColumnList CLOB;
* vFieldList CLOB;
* begin
* FILE_MANAGER.GENERATE_EXTERNAL_TABLE_PARAMS (
* pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER'
* ,pColumnList => vColumnList
* ,pFieldList => vFieldList
* );
* DBMS_OUTPUT.PUT_LINE('vColumnList = '||vColumnList);
* DBMS_OUTPUT.PUT_LINE('vFieldList = '||vFieldList);
* end;
* /
**/
PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS (
pTemplateTableName IN VARCHAR2,
pColumnList OUT CLOB,
pFieldList OUT CLOB
);
/**
* @name ADD_SOURCE
* @desc Insert a new record to A_SOURCE table.
* pSourceKey is a PRIMARY KEY value.
**/
PROCEDURE ADD_SOURCE (
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE,
pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE
);
/**
* @name DELETE_SOURCE_CASCADE
* @desc Safely deletes a SOURCE specified by pSourceKey parameter from A_SOURCE table and all dependent tables:
* - A_SOURCE_FILE_CONFIG
* - A_SOURCE_FILE_RECEIVED
* - A_COLUMN_DATE_FORMAT (only if template table is not shared with other source systems)
* The procedure checks if template tables are shared before deleting date format configurations.
* If a template table is used by multiple source systems, date formats are preserved.
* @example CALL CT_MRDS.FILE_MANAGER.DELETE_SOURCE_CASCADE(pSourceKey => 'TEST_SYS');
**/
PROCEDURE DELETE_SOURCE_CASCADE (
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE
);
/**
* @name GET_CONTAINER_SOURCE_FILE_CONFIG_KEY
* @desc For specified parameter pSourceFileId (A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID)
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY for related CONTAINER record.
* @example select FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(
* pSourceFileId => 'UC_DISSEM')
* from dual;
* @ex_rslt 126
**/
FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY (
pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
) RETURN PLS_INTEGER;
/**
* @name GET_SOURCE_FILE_CONFIG_KEY
* @desc For specified input parameters,
* it returns A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY.
* @example select FILE_MANAGER.GET_SOURCE_FILE_CONFIG_KEY (
* pSourceFileType => 'INPUT'
* ,pSourceFileId => 'UC_DISSEM'
* ,pTableId => 'UC_NMA_DISSEM')
* from dual;
* @ex_rslt 126
**/
FUNCTION GET_SOURCE_FILE_CONFIG_KEY (
pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT'
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE
) RETURN PLS_INTEGER;
/**
* @name ADD_SOURCE_FILE_CONFIG
* @desc Insert a new record to A_SOURCE_FILE_CONFIG table.
* MARS-1049: Added pEncoding parameter for CSV character set specification.
* @param pEncoding - Character set encoding for CSV files (e.g., 'UTF8', 'WE8MSWIN1252', 'EE8ISO8859P2')
* If NULL, no CHARACTERSET clause is added to external table definitions
* @example CALL CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
* pSourceKey => 'C2D', pSourceFileType => 'INPUT',
* pSourceFileId => 'UC_DISSEM', pTableId => 'METADATA_LOADS',
* pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS',
* pEncoding => 'UTF8'
* );
**/
PROCEDURE ADD_SOURCE_FILE_CONFIG (
pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE
,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE
,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR
);
/**
* @name ADD_COLUMN_DATE_FORMAT
* @desc Insert a new record to A_COLUMN_DATE_FORMAT table.
**/
PROCEDURE ADD_COLUMN_DATE_FORMAT (
pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE
,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE
,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE
);
/**
* @name GET_BUCKET_URI
* @desc Function used to get string with bucket http url.
* Possible input values for pBucketArea are: 'INBOX', 'ODS', 'DATA', 'ARCHIVE'
* @example select FILE_MANAGER.GET_BUCKET_URI(pBucketArea => 'ODS') from dual;
* @ex_rslt https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/frcnomajoc7v/b/mrds_data_tst/o/
**/
FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2)
RETURN VARCHAR2;
/**
* @name GET_DET_SOURCE_FILE_CONFIG_INFO
* @desc Function returns details about A_SOURCE_FILE_CONFIG record
* for specified pSourceFileConfigKey (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO (
* pSourceFileConfigKey => 128
* ,pIncludeContainerInfo => 1
* ,pIncludeColumnFormatInfo => 1
* ) from dual;
* @ex_rslt
* Details about File Configuration:
* --------------------------------
* A_SOURCE_FILE_CONFIG_KEY = 128
* A_SOURCE_KEY = C2D
* ...
* --------------------------------
*
* Details about related Container Config:
* --------------------------------
* A_SOURCE_FILE_CONFIG_KEY = 126
* A_SOURCE_KEY = C2D
* ...
* --------------------------------
*
* Column Date Format config entries:
* --------------------------------
* TEMPLATE_TABLE_NAME = CT_ET_TEMPLATES.C2D_UC_MA_DISSEM
* ...
* --------------------------------
**/
FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
) RETURN VARCHAR2;
/**
* @name GET_DET_SOURCE_FILE_RECEIVED_INFO
* @desc Function returns details about A_SOURCE_FILE_RECEIVED record
* for specified pSourceFileReceivedKey (A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY).
* If pIncludeConfigInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
* If pIncludeContainerInfo is <> 0 it returns additional info about related Container config record (A_SOURCE_FILE_CONFIG)
* If pIncludeColumnFormatInfo is <> 0 it returns additional info about related ColumnFormat config record (A_COLUMN_DATE_FORMAT)
* @example select FILE_MANAGER.GET_DET_SOURCE_FILE_RECEIVED_INFO (
* pSourceFileReceivedKey => 377
* ,pIncludeConfigInfo => 1
* ,pIncludeContainerInfo => 1
* ,pIncludeColumnFormatInfo => 1
* ) from dual;
*
**/
FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE
,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
) RETURN VARCHAR2;
/**
* @name GET_DET_USER_LOAD_OPERATIONS
* @desc Function returns details from USER_LOAD_OPERATIONS table
* for specified pOperationId.
* @example select FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => 3608) from dual;
* @ex_rslt
* Details about USER_LOAD_OPERATIONS where ID = 3608
* --------------------------------
* ID = 3608
* TYPE = VALIDATE
* SID = 31260
* SERIAL# = 52915
* START_TIME = 2025-05-20 10.08.24.436983 EUROPE/BELGRADE
* UPDATE_TIME = 2025-05-20 10.08.24.458643 EUROPE/BELGRADE
* STATUS = FAILED
* OWNER_NAME = CT_MRDS
* TABLE_NAME = STANDING_FACILITIES_HEADER
* PARTITION_NAME =
* SUBPARTITION_NAME =
* FILE_URI_LIST =
* ROWS_LOADED =
* LOGFILE_TABLE = VALIDATE$3608_LOG
* BADFILE_TABLE = VALIDATE$3608_BAD
* STATUS_TABLE =
* TEMPEXT_TABLE =
* CREDENTIAL_NAME =
* EXPIRATION_TIME = 2025-05-22 10.08.24.436983000 EUROPE/BELGRADE
* --------------------------------
**/
FUNCTION GET_DET_USER_LOAD_OPERATIONS (
pOperationId PLS_INTEGER
) RETURN VARCHAR2;
/**
* @name ANALYZE_VALIDATION_ERRORS
* @desc Wrapper function that analyzes validation errors for a source file using its received key.
* Automatically derives template schema, table name, CSV URI and validation log table
* from file metadata and calls ENV_MANAGER.ANALYZE_VALIDATION_ERRORS.
* @example SELECT FILE_MANAGER.ANALYZE_VALIDATION_ERRORS(63) FROM DUAL;
* @ex_rslt Detailed validation analysis report with column mismatches and solutions
**/
FUNCTION ANALYZE_VALIDATION_ERRORS(
pSourceFileReceivedKey IN NUMBER
) RETURN VARCHAR2;
---------------------------------------------------------------------------------------------------------------------------
-- EXTERNAL TABLE BATCH OPERATIONS (MARS-1057)
---------------------------------------------------------------------------------------------------------------------------
/**
* @name CREATE_EXTERNAL_TABLES_SET
* @desc Creates a complete set of 3 external tables (INBOX, ODS, ARCHIVE) for a single configuration
* from A_SOURCE_FILE_CONFIG table. Automatically generates table names and paths following
* official path patterns. Optionally drops and recreates existing tables.
* @param pSourceFileConfigKey - Primary key from A_SOURCE_FILE_CONFIG table
* @param pRecreate - If TRUE, drops existing tables before creating new ones; if FALSE, fails if tables exist
* @example BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_SET(
* pSourceFileConfigKey => 123,
* pRecreate => FALSE
* );
* END;
* @ex_rslt Creates three external tables in ODS schema:
* - C2D_A_UC_DISSEM_METADATA_LOADS_INBOX
* - C2D_A_UC_DISSEM_METADATA_LOADS_ODS
* - C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE
**/
PROCEDURE CREATE_EXTERNAL_TABLES_SET (
pSourceFileConfigKey IN NUMBER,
pRecreate IN BOOLEAN DEFAULT FALSE
);
/**
* @name CREATE_EXTERNAL_TABLES_BATCH
* @desc Creates external table sets for multiple configurations based on filter criteria.
* Processes only INPUT type files from A_SOURCE_FILE_CONFIG. Creates 3 tables per configuration
* (INBOX, ODS, ARCHIVE). Continues processing even if individual sets fail.
* @param pSourceKey - Filter by A_SOURCE_KEY (NULL = all sources)
* @param pSourceFileId - Filter by SOURCE_FILE_ID (NULL = all file types)
* @param pTableId - Filter by TABLE_ID (NULL = all tables)
* @param pRecreate - If TRUE, drops and recreates existing tables; if FALSE, skips if tables exist
* @example -- Create all external tables for C2D source
* BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
* pSourceKey => 'C2D',
* pRecreate => FALSE
* );
* END;
*
* -- Recreate all external tables for all sources
* BEGIN
* FILE_MANAGER.CREATE_EXTERNAL_TABLES_BATCH(
* pRecreate => TRUE
* );
* END;
* @ex_rslt Returns summary: Total: 10, Processed: 9, Failed: 1
**/
PROCEDURE CREATE_EXTERNAL_TABLES_BATCH (
pSourceKey IN VARCHAR2 DEFAULT NULL,
pSourceFileId IN VARCHAR2 DEFAULT NULL,
pTableId IN VARCHAR2 DEFAULT NULL,
pRecreate IN BOOLEAN DEFAULT FALSE
);
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_VERSION
* @desc Returns the current version number of the FILE_MANAGER package.
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
* @example SELECT FILE_MANAGER.GET_VERSION() FROM DUAL;
* @ex_rslt 3.2.0
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* @name GET_BUILD_INFO
* @desc Returns comprehensive build information including version, build date, and author.
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
* @example SELECT FILE_MANAGER.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: FILE_MANAGER
* Version: 3.2.0
* Build Date: 2025-10-22 16:30:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* @name GET_VERSION_HISTORY
* @desc Returns complete version history with all releases and changes.
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
* @example SELECT FILE_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
* @ex_rslt FILE_MANAGER Version History:
* 3.2.0 (2025-10-22): Added package versioning system...
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END;
/
/

215
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/WORKFLOW_MANAGER.pkb Normal file
View File

@@ -0,0 +1,215 @@
CREATE OR REPLACE PACKAGE BODY CT_MRDS.WORKFLOW_MANAGER
IS
FUNCTION INIT_WORKFLOW(pServiceName IN VARCHAR2, pWorkflowRunId IN VARCHAR2, pWorkflowName in VARCHAR2)
RETURN NUMBER
IS
vWorkflowHistoryKey NUMBER;
BEGIN
vWorkflowHistoryKey := A_WORKFLOW_HISTORY_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_WORKFLOW_HISTORY (SERVICE_NAME, A_WORKFLOW_HISTORY_KEY, ORCHESTRATION_RUN_ID,
WORKFLOW_NAME, WORKFLOW_START, WORKFLOW_SUCCESSFUL)
VALUES (pServiceName, vWorkflowHistoryKey, pWorkflowRunId,
pWorkflowName, SYSTIMESTAMP, vpRunningStatus);
return vWorkflowHistoryKey;
END INIT_WORKFLOW;
--
-- Overload without service name for backward compatability, to be cleaned up later
--
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pWorkflowStatus IN VARCHAR2)
IS
BEGIN
FINALISE_WORKFLOW(pWorkflowHistoryKey, NULL, pWorkflowStatus);
END;
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pServiceName IN VARCHAR2, pWorkflowStatus IN VARCHAR2)
IS
BEGIN
UPDATE CT_MRDS.A_WORKFLOW_HISTORY SET WORKFLOW_SUCCESSFUL = pWorkflowStatus,
WORKFLOW_END = SYSTIMESTAMP
WHERE A_WORKFLOW_HISTORY_KEY = pWorkflowHistoryKey
AND SERVICE_NAME = NVL(pServiceName,SERVICE_NAME);
END FINALISE_WORKFLOW;
FUNCTION INIT_TASK(pTaskRunId IN VARCHAR2, pTaskName in VARCHAR2, pWorkflowHistoryKey IN NUMBER)
RETURN NUMBER
IS
vTaskHistoryKey NUMBER;
BEGIN
vTaskHistoryKey := A_TASK_HISTORY_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_TASK_HISTORY (A_TASK_HISTORY_KEY, TASK_RUN_ID, A_WORKFLOW_HISTORY_KEY,
TASK_NAME, TASK_START, TASK_SUCCESSFUL, SERVICE_NAME)
VALUES (vTaskHistoryKey, pTaskRunId, pWorkflowHistoryKey,
pTaskName, SYSTIMESTAMP, vpRunningStatus, 'ODS');
return vTaskHistoryKey;
END INIT_TASK;
PROCEDURE FINALISE_TASK(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2)
IS
BEGIN
UPDATE CT_MRDS.A_TASK_HISTORY SET TASK_SUCCESSFUL = pTaskStatus,
TASK_END = SYSTIMESTAMP
WHERE A_TASK_HISTORY_KEY = pTaskHistoryKey;
END FINALISE_TASK;
-- Internal helper function without COMMIT for use within other functions
FUNCTION STORE_TASK_SOURCE_INFO_INTERNAL(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2, pNumRows IN NUMBER)
RETURN NUMBER
IS
vTaskHistorySourceKey NUMBER;
BEGIN
vTaskHistorySourceKey := A_TASK_HISTORY_SOURCE_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_TASK_HISTORY_SOURCE (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY,
SOURCE_NAME, ROW_COUNT)
VALUES (vTaskHistorySourceKey, pTaskHistoryKey,
pSourceName, pNumRows);
RETURN vTaskHistorySourceKey;
END;
FUNCTION STORE_TASK_SOURCE_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2, pNumRows IN NUMBER)
RETURN NUMBER
IS
vTaskHistorySourceKey NUMBER;
BEGIN
vTaskHistorySourceKey := STORE_TASK_SOURCE_INFO_INTERNAL(pTaskHistoryKey, pSourceName, pNumRows);
COMMIT;
RETURN vTaskHistorySourceKey;
END;
-- Internal helper function without COMMIT for use within other functions
FUNCTION STORE_TASK_TARGET_INFO_INTERNAL(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2,
pNumRowsApplied IN NUMBER, pNumRowsRejected IN NUMBER,
pLoadSuccessful IN VARCHAR2, pServiceName IN VARCHAR2)
RETURN NUMBER
IS
vTaskHistoryTargetKey NUMBER;
BEGIN
vTaskHistoryTargetKey := A_TASK_HISTORY_TARGET_KEY_SEQ.NEXTVAL;
INSERT INTO CT_MRDS.A_TASK_HISTORY_TARGET (A_TASK_HISTORY_TARGET_KEY, A_TASK_HISTORY_KEY,
TARGET_NAME, ROW_COUNT_APPLIED, ROW_COUNT_REJECTED,
LOAD_SUCCESSFUL, SERVICE_NAME)
VALUES (vTaskHistoryTargetKey, pTaskHistoryKey,
pSourceName, pNumRowsApplied, pNumRowsRejected,
pLoadSuccessful, pServiceName);
RETURN vTaskHistoryTargetKey;
END;
FUNCTION STORE_TASK_TARGET_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2,
pNumRowsApplied IN NUMBER, pNumRowsRejected IN NUMBER,
pLoadSuccessful IN VARCHAR2, pServiceName IN VARCHAR2)
RETURN NUMBER
IS
vTaskHistoryTargetKey NUMBER;
BEGIN
vTaskHistoryTargetKey := STORE_TASK_TARGET_INFO_INTERNAL(
pTaskHistoryKey, pSourceName, pNumRowsApplied,
pNumRowsRejected, pLoadSuccessful, pServiceName
);
COMMIT;
RETURN vTaskHistoryTargetKey;
END;
FUNCTION FINALISE_TASK_WITH_TARGET_INFO(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2,
pTargetName IN VARCHAR2, pNumRowsApplied IN NUMBER,
pNumRowsRejected IN NUMBER, pLoadSuccessful IN VARCHAR2,
pServiceName IN VARCHAR2)
RETURN NUMBER
IS
vTaskHistoryTargetKey NUMBER;
BEGIN
-- Store target information using internal function (without COMMIT)
vTaskHistoryTargetKey := STORE_TASK_TARGET_INFO_INTERNAL(
pTaskHistoryKey => pTaskHistoryKey,
pSourceName => pTargetName,
pNumRowsApplied => pNumRowsApplied,
pNumRowsRejected => pNumRowsRejected,
pLoadSuccessful => pLoadSuccessful,
pServiceName => pServiceName
);
-- Finalize the task using existing procedure (no COMMIT in procedure)
FINALISE_TASK(pTaskHistoryKey, pTaskStatus);
-- Single COMMIT for the entire atomic operation
COMMIT;
RETURN vTaskHistoryTargetKey;
END;
PROCEDURE SET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
,pValue IN VARCHAR2
) IS
BEGIN
INSERT INTO CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME, PROPERTY, VALUE)
VALUES (pWorkflowHistoryKey, pServiceName, pProperty, pValue);
END;
FUNCTION GET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
) RETURN VARCHAR2
IS
vValue CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY.VALUE%TYPE;
BEGIN
SELECT VALUE
INTO vValue
FROM CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY
WHERE A_WORKFLOW_HISTORY_KEY = pWorkflowHistoryKey
AND SERVICE_NAME = pServiceName
AND PROPERTY = pProperty;
RETURN vValue;
END;
----------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION
RETURN VARCHAR2
IS
BEGIN
RETURN PACKAGE_VERSION;
END GET_VERSION;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUILD_INFO
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
pPackageName => 'WORKFLOW_MANAGER',
pVersion => PACKAGE_VERSION,
pBuildDate => PACKAGE_BUILD_DATE,
pAuthor => PACKAGE_AUTHOR
);
END GET_BUILD_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION_HISTORY
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
pPackageName => 'WORKFLOW_MANAGER',
pVersionHistory => VERSION_HISTORY
);
END GET_VERSION_HISTORY;
----------------------------------------------------------------------------------------------------
END WORKFLOW_MANAGER;

99
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/packages/WORKFLOW_MANAGER.pkg Normal file
View File

@@ -0,0 +1,99 @@
create or replace PACKAGE CT_MRDS.WORKFLOW_MANAGER
IS
-- Package Version Information (Semantic Versioning: MAJOR.MINOR.PATCH)
PACKAGE_VERSION CONSTANT VARCHAR2(10) := '1.7.1';
PACKAGE_BUILD_DATE CONSTANT VARCHAR2(20) := '2025-11-18 12:00:00';
PACKAGE_AUTHOR CONSTANT VARCHAR2(100) := 'Grzegorz Michalski';
-- Version History (Latest changes first)
VERSION_HISTORY CONSTANT VARCHAR2(4000) :=
'1.7.1 (2025-11-18): Added SERVICE_NAME column with hardcoded ''ODS'' value to INIT_TASK function' || CHR(13)||CHR(10) ||
'1.7.0 (2025-10-31): Added FINALISE_TASK_WITH_TARGET_INFO function for atomic task completion with target info' || CHR(13)||CHR(10) ||
'1.6.0 (2025-10-31): Converted STORE_TASK_SOURCE_INFO and STORE_TASK_TARGET_INFO from procedures to functions returning sequence keys' || CHR(13)||CHR(10) ||
'1.5.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||
'1.2.0 (2025-10-05): Added workflow property management (SET/GET_WORKFLOW_PROPERTY)' || CHR(13)||CHR(10) ||
'1.0.0 (2025-09-10): Initial release with workflow and task lifecycle management';
-- Line break constant for consistent CRLF formatting across Windows environments
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
FUNCTION INIT_WORKFLOW(pServiceName IN VARCHAR2, pWorkflowRunId IN VARCHAR2, pWorkflowName in VARCHAR2)
RETURN NUMBER;
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pWorkflowStatus IN VARCHAR2);
PROCEDURE FINALISE_WORKFLOW(pWorkflowHistoryKey IN NUMBER, pServiceName IN VARCHAR2, pWorkflowStatus IN VARCHAR2);
FUNCTION INIT_TASK(pTaskRunId IN VARCHAR2, pTaskName in VARCHAR2, pWorkflowHistoryKey IN NUMBER)
RETURN NUMBER;
PROCEDURE FINALISE_TASK(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2);
FUNCTION STORE_TASK_SOURCE_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2, pNumRows IN NUMBER)
RETURN NUMBER;
FUNCTION STORE_TASK_TARGET_INFO(pTaskHistoryKey IN NUMBER, pSourceName IN VARCHAR2,
pNumRowsApplied IN NUMBER, pNumRowsRejected IN NUMBER,
pLoadSuccessful IN VARCHAR2, pServiceName IN VARCHAR2)
RETURN NUMBER;
FUNCTION FINALISE_TASK_WITH_TARGET_INFO(pTaskHistoryKey IN NUMBER, pTaskStatus IN VARCHAR2,
pTargetName IN VARCHAR2, pNumRowsApplied IN NUMBER,
pNumRowsRejected IN NUMBER, pLoadSuccessful IN VARCHAR2,
pServiceName IN VARCHAR2)
RETURN NUMBER;
vpRunningStatus CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL%TYPE := 'R';
--
-- Set and get information on workflow level
--
PROCEDURE SET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2
,pValue IN VARCHAR2
);
FUNCTION GET_WORKFLOW_PROPERTY(
pWorkflowHistoryKey IN NUMBER
,pServiceName IN VARCHAR2
,pProperty IN VARCHAR2)
RETURN VARCHAR2;
---------------------------------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
---------------------------------------------------------------------------------------------------------------------------
/**
* @name GET_VERSION
* @desc Returns the current version number of the WORKFLOW_MANAGER package.
* Uses semantic versioning format (MAJOR.MINOR.PATCH).
* @example SELECT WORKFLOW_MANAGER.GET_VERSION() FROM DUAL;
* @ex_rslt 1.5.0
**/
FUNCTION GET_VERSION RETURN VARCHAR2;
/**
* @name GET_BUILD_INFO
* @desc Returns comprehensive build information including version, build date, and author.
* Uses centralized ENV_MANAGER.GET_PACKAGE_VERSION_INFO function.
* @example SELECT WORKFLOW_MANAGER.GET_BUILD_INFO() FROM DUAL;
* @ex_rslt Package: WORKFLOW_MANAGER
* Version: 1.5.0
* Build Date: 2025-10-22 17:00:00
* Author: Grzegorz Michalski
**/
FUNCTION GET_BUILD_INFO RETURN VARCHAR2;
/**
* @name GET_VERSION_HISTORY
* @desc Returns complete version history with all releases and changes.
* Uses centralized ENV_MANAGER.FORMAT_VERSION_HISTORY function.
* @example SELECT WORKFLOW_MANAGER.GET_VERSION_HISTORY() FROM DUAL;
* @ex_rslt WORKFLOW_MANAGER Version History:
* 1.5.0 (2025-10-22): Added package versioning system...
**/
FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2;
END WORKFLOW_MANAGER;

216
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/seed/seed.sql Normal file
View File

@@ -0,0 +1,216 @@
DELETE FROM CT_MRDS.A_FILE_MANAGER_CONFIG;
DELETE FROM CT_MRDS.A_SOURCE_FILE_RECEIVED;
DELETE FROM CT_MRDS.A_COLUMN_DATE_FORMAT;
DELETE FROM CT_MRDS.A_SOURCE_FILE_CONFIG;
DELETE FROM CT_MRDS.A_SOURCE;
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('default', 'EnvironmentID','dev');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'Region', 'eu-frankfurt-1');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'NameSpace', 'frcnomajoc7v');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'InboxBucketName', 'mrds_inbox_tst');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'DataBucketName', 'mrds_data_tst');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'HistoryBucketName', 'mrds_history_tst');
--INSERT INTO A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'UriName', 'https://objectstorage.'||vpRegion||'.oraclecloud.com/n/'||vpNameSpace||'/b/'||vpDataBucketName||'/o/');
--INSERT INTO A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'DataBucketUri', 'https://objectstorage.'||vpRegion||'.oraclecloud.com/n/'||vpNameSpace||'/b/'||vpDataBucketName||'/o/');
--INSERT INTO A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'InboxBucketUri', 'http://dev.example.com/inboxBucket');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'CredentialName', 'OCI$RESOURCE_PRINCIPAL');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'LoggingEnabled', 'ON');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'MinLogLevel', 'DEBUG');
INSERT INTO CT_MRDS.A_FILE_MANAGER_CONFIG (environment_id, config_variable, config_variable_value) VALUES ('dev', 'DefaultDateFormat', 'DD/MM/YYYY HH24:MI:SS');
commit;
-- ISD
call CT_MRDS.FILE_MANAGER.ADD_SOURCE(pSourceKey => 'ISD', pSourceNAme => 'Integrated Surface Data');
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'ISD'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'IDS_DATA'
,pSourceFileDesc => 'Yearly figures by station'
,pSourceFileNamePattern => '*.csv'
,pTableId => 'IDS_DATA'
,pTemplateTableName => 'CT_ET_TEMPLATES.IDS_DATA'
);
-- LM
call CT_MRDS.FILE_MANAGER.ADD_SOURCE(pSourceKey => 'LM', pSourceNAme => 'LM');
-- STANDING_FACILITIES
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'LM'
,pSourceFileType => 'CONTAINER'
,pSourceFileId => 'STANDING_FACILITIES'
,pSourceFileDesc => 'LM Standing facilities'
,pSourceFileNamePattern => '*.xml'
,pTableId => null
,pTemplateTableName => null
,pContainerFileKey => null
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'LM'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'STANDING_FACILITIES'
,pSourceFileDesc => 'LM Standing facilities header'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'STANDING_FACILITIES_HEADER'
,pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER'
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'LM'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'STANDING_FACILITIES'
,pSourceFileDesc => 'LM Standing facilities'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'STANDING_FACILITIES'
,pTemplateTableName => 'CT_ET_TEMPLATES.LM_STANDING_FACILITIES'
);
--
-- C2D
--
call CT_MRDS.FILE_MANAGER.ADD_SOURCE(pSourceKey => 'C2D', pSourceNAme => 'C2D');
-- UC_ACC
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'UC_NMA_ACC'
,pSourceFileDesc => 'UC NonMarketableAssets ACC'
,pSourceFileNamePattern => '*.csv'
,pTableId => 'UC_NMA_ACC'
,pTemplateTableName => 'CT_ET_TEMPLATES.C2D_UC_NMA_ACC'
);
-- UC_DISSEM
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'CONTAINER'
,pSourceFileId => 'UC_DISSEM'
,pSourceFileDesc => 'UC Dissemination'
,pSourceFileNamePattern => '*.xml'
,pTableId => null
,pTemplateTableName => null
,pContainerFileKey => null
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'UC_DISSEM'
,pSourceFileDesc => 'UC Dissemination Metadata'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'A_UC_DISSEM_METADATA_LOADS'
,pTemplateTableName => 'CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'A_UC_DISSEM_METADATA_LOADS'
,pColumnName => 'SNAPSHOT_DATE'
,pDateFormat => 'YYYY-MM-DD'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'A_UC_DISSEM_METADATA_LOADS'
,pColumnName => 'FILE_CREATION_DATE'
,pDateFormat => 'YYYY-MM-DD"T"HH24:MI:SS'
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'UC_DISSEM'
,pSourceFileDesc => 'UC MarketableAssets Dissemination'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'UC_MA_DISSEM'
,pTemplateTableName => 'CT_ET_TEMPLATES.C2D_UC_MA_DISSEM'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'UC_MA_DISSEM'
,pColumnName => 'SNAPSHOT_DATE'
,pDateFormat => 'YYYY-MM-DD'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'UC_MA_DISSEM'
,pColumnName => 'FILE_CREATION_DATE'
,pDateFormat => 'YYYY-MM-DD"T"HH24:MI:SS'
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'UC_DISSEM'
,pSourceFileDesc => 'UC MarketableAssets Dissemination'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'UC_NMA_DISSEM'
,pTemplateTableName => 'CT_ET_TEMPLATES.C2D_UC_NMA_DISSEM'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'UC_NMA_DISSEM'
,pColumnName => 'SNAPSHOT_DATE'
,pDateFormat => 'YYYY-MM-DD'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'UC_NMA_DISSEM'
,pColumnName => 'FILE_CREATION_DATE'
,pDateFormat => 'YYYY-MM-DD"T"HH24:MI:SS'
);
call CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT(
pSourceFileId => 'UC_DISSEM'
,pTableId => 'UC_NMA_DISSEM'
,pColumnName => 'MATURITY_DATE'
,pDateFormat => 'YYYY-MM-DD'
);
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'C2D'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'UC_DISSEM'
,pSourceFileDesc => 'UC Additional Credit Claims'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'UC_ACC'
,pTemplateTableName => 'CT_ET_TEMPLATES.C2D_UC_NMA_ACC'
);
--
-- MDP
--
call CT_MRDS.FILE_MANAGER.ADD_SOURCE(pSourceKey => 'MDP', pSourceNAme => 'MDP');
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'MDP'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'BBG_EQUITY_OT'
,pSourceFileDesc => 'Bloomberg Debt'
,pSourceFileNamePattern => '.*.csv'
,pTableId => 'BBG_EQUITY_OT'
,pTemplateTableName => 'CT_ET_TEMPLATES.ODS_MDP_BBG_EQUITY_OT'
,pContainerFileKey => null
);
-- LOAD_CONFIG MDP
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'MDP'
,pSourceFileType => 'LOAD_CONFIG'
,pSourceFileId => null
,pSourceFileDesc => 'Reuters list ODS Cloud'
,pSourceFileNamePattern => '.*.txt'
,pTableId => null
,pTemplateTableName => null
,pContainerFileKey => null
);

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/seed/seed_csdb.sql Normal file
View File

@@ -0,0 +1,22 @@
-- CSDB
--call CT_MRDS.FILE_MANAGER.ADD_SOURCE(pSourceKey => 'CSDB', pSourceNAme => 'Centralised Securities Database');
--call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
-- pSourceKey => 'CSDB'
-- ,pSourceFileType => 'INPUT'
-- ,pSourceFileId => 'RT_INSTRUMENT_RATING'
-- ,pSourceFileDesc => 'Instrument Ratings from DEVO'
-- ,pSourceFileNamePattern => '*.csv'
-- ,pTableId => 'RT_INSTRUMENT_RATING'
-- ,pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_RT_INSTRUMENT_RATING'
-- );
call CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG(
pSourceKey => 'CSDB'
,pSourceFileType => 'INPUT'
,pSourceFileId => 'DEBT_DAILY'
,pSourceFileDesc => 'CSDB Debt Daily'
,pSourceFileNamePattern => '*.csv'
,pTableId => 'DEBT_DAILY'
,pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY'
);

57
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/seed/seed_replica_mgmt.sql Normal file
View File

@@ -0,0 +1,57 @@
-- ====================================================================
-- Seed Data for DEVO Replica Management Tables
-- ====================================================================
-- Purpose: Initial configuration data for MOPDB and RAR replication management
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_tables.sql
-- ====================================================================
-- Initial MOPDB replica management configuration
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_CEPH';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'MPEC' AND TABLE_NAME = 'T_MPEC';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CAPITAL_KEY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'CT_MOPDB' AND TABLE_NAME = 'MOPDB_METADATA_INVENTORY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CSDB_RATING_ACTION';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_CURRENCY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'CLSLNK_SNPSHT_DT' WHERE OWNER = 'RIAD' AND TABLE_NAME = 'T_CLSLNK_PSTNZ_HIST';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_AGGREGATED_ALLOTMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'ELA' AND TABLE_NAME = 'T_ELA_INFO';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_ORIGINATOR';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_ADD_GUARANTOR';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_IAB';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_SERVICER';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_LSP';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_IRSP';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS_FXSC';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_COLLATERAL';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'REFERENCE' AND TABLE_NAME = 'T_COUNTRY';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_SNAPSHOT_DATE';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT_MODIFICATIONS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ANNOUNCEMENT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_FBL_ITEM';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_FULLBID_ARRAY_COMPILED';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REF_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_STANDING_FACILITIES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REFERENCE_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_CURRENT_ACCOUNTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REF_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_BALANCE_SHEET';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'REFERENCE_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FORECAST';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'ADJUSTMENT_DATE' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_ADH_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_CSM_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'LM' AND TABLE_NAME = 'T_FC_QRE_ADJUSTMENTS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_COLLATERAL_ALL_SOURCES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_ALL_ASSETS';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'TIME_PERIOD' WHERE OWNER = 'SDW' AND TABLE_NAME = 'T_EXCHANGE_RATES';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'OVERWRITE', PARTITION_COLUMN = 'ROWID' WHERE OWNER = 'TOP' AND TABLE_NAME = 'T_ALLOTMENT_MODIFICATIONS_AGGREGATED';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'T_INTRADAY_CREDIT';
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB SET SAVE_MODE = 'APPEND', PARTITION_COLUMN = 'SNAPSHOT_DATE' WHERE OWNER = 'C2D' AND TABLE_NAME = 'MV_PUBLIC_ELIGIBLE_MARKETABLE_ASSETS';
COMMIT;
-- Initial RAR replica management configuration
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR
SET SAVE_MODE = 'OVERWRITE' , PARTITION_COLUMN = 'ROWID';
COMMIT;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_SNAPSHOT_DATE' WHERE TABLE_NAME = 'NH_HF_PRICE' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_PRICE' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_POSITION' AND OWNER = 'CORR_RAR' ;
UPDATE CT_MRDS.A_DEVO_REPLICA_MGMT_RAR SET SAVE_MODE = 'APPEND' , PARTITION_COLUMN = 'A_INSERTION_DATE' WHERE TABLE_NAME = 'NH_PERFORMANCE' AND OWNER = 'CORR_RAR' ;
COMMIT;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_SOURCE_FILE_CONFIG_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_SOURCE_FILE_CONFIG_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_SOURCE_FILE_CONFIG table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ TO MRDS_LOADER_ROLE;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_SOURCE_FILE_RECEIVED_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_SOURCE_FILE_RECEIVED_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_SOURCE_FILE_RECEIVED table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ TO MRDS_LOADER_ROLE;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_TABLE_STAT_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_TABLE_STAT_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_TABLE_STAT table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_TABLE_STAT_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_TABLE_STAT_KEY_SEQ TO MRDS_LOADER_ROLE;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_TASK_HISTORY_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_TASK_HISTORY_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_TASK_HISTORY table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_TASK_HISTORY_KEY_SEQ TO MRDS_LOADER;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_TASK_HISTORY_SOURCE_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_TASK_HISTORY_SOURCE_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_TASK_HISTORY_SOURCE table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_SOURCE_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_TASK_HISTORY_SOURCE_KEY_SEQ TO MRDS_LOADER;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_TASK_HISTORY_TARGET_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_TASK_HISTORY_TARGET_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_TASK_HISTORY_TARGET table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_TASK_HISTORY_TARGET_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_TASK_HISTORY_TARGET_KEY_SEQ TO MRDS_LOADER;

9
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/sequences/A_WORKFLOW_HISTORY_KEY_SEQ.sql Normal file
View File

@@ -0,0 +1,9 @@
-- ====================================================================
-- A_WORKFLOW_HISTORY_KEY_SEQ Sequence
-- ====================================================================
-- Purpose: Generate primary keys for A_WORKFLOW_HISTORY table
-- ====================================================================
CREATE SEQUENCE CT_MRDS.A_WORKFLOW_HISTORY_KEY_SEQ START WITH 1 INCREMENT BY 1;
GRANT SELECT ON CT_MRDS.A_WORKFLOW_HISTORY_KEY_SEQ TO MRDS_LOADER;

18
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_CASPER_FILEVAULT.sql Normal file
View File

@@ -0,0 +1,18 @@
-- ====================================================================
-- A_CASPER_FILEVAULT Table Definition
-- ====================================================================
-- Purpose: File vault tracking for CASPER workflow operations
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_tables.sql
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_CASPER_FILEVAULT" (
"A_WORKFLOW_HISTORY_KEY" NUMBER(28,0),
"WLA_RUN_ID" NUMBER(28,0),
"WORKFLOW_NAME" CHAR(16 BYTE),
"FILENAME" VARCHAR2(250 CHAR),
"STATUS" VARCHAR2(30 CHAR),
"FILESIZE" NUMBER(28,0),
"FILE_ID" NUMBER(28,0),
"IS_TEST" VARCHAR2(5 CHAR),
"REFERENCE_TIME" TIMESTAMP (6)
);

20
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_COLUMN_DATE_FORMAT.sql Normal file
View File

@@ -0,0 +1,20 @@
-- ====================================================================
-- A_COLUMN_DATE_FORMAT Table
-- ====================================================================
-- Purpose: Store column-specific date format configurations
-- ====================================================================
CREATE TABLE CT_MRDS.A_COLUMN_DATE_FORMAT (
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
COLUMN_NAME VARCHAR2(200) NOT NULL,
DATE_FORMAT VARCHAR2(200) NOT NULL,
CONSTRAINT A_COLUMN_DATE_FORMAT_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY, COLUMN_NAME),
CONSTRAINT ACDF_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY)
) TABLESPACE "DATA";
-- Primary key index (matching constraint definition)
CREATE UNIQUE INDEX "CT_MRDS"."A_COLUMN_DATE_FORMAT_PK"
ON "CT_MRDS"."A_COLUMN_DATE_FORMAT" ("A_SOURCE_FILE_CONFIG_KEY", "COLUMN_NAME")
TABLESPACE "DATA";
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_COLUMN_DATE_FORMAT TO MRDS_LOADER_ROLE;

23
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_METADATA_INVENTORY.sql Normal file
View File

@@ -0,0 +1,23 @@
-- ====================================================================
-- A_DEVO_METADATA_INVENTORY Table Definition
-- ====================================================================
-- Purpose: Metadata inventory for DEVO data sources and column information
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_METADATA_INVENTORY" (
"A_VALID_FROM" DATE,
"A_VALID_TO" DATE,
"OWNER" VARCHAR2(128 BYTE),
"TABLE_NAME" VARCHAR2(128 BYTE),
"COLUMN_NAME" VARCHAR2(128 BYTE),
"DATA_TYPE" VARCHAR2(128 BYTE),
"CHAR_LENGTH" NUMBER,
"DATA_PRECISION" NUMBER,
"DATA_SCALE" NUMBER,
"COLUMN_ID" NUMBER,
"DATA_DESCRIPTION" VARCHAR2(4000 BYTE),
"LIST_OF_SOURCES" VARCHAR2(20 BYTE),
"RAR3_TYPE_OF_ACCESS" CHAR(2 BYTE),
"SERVICE_NAME" CHAR(5 BYTE)
) TABLESPACE "DATA";

23
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_REPLICA_MGMT_LOG.sql Normal file
View File

@@ -0,0 +1,23 @@
-- ====================================================================
-- A_DEVO_REPLICA_MGMT_LOG Table Definition
-- ====================================================================
-- Purpose: Audit log table for replica management operations
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_LOG" (
"OWNER" VARCHAR2(50 BYTE),
"TABLE_NAME" VARCHAR2(50 BYTE),
"TABLE_ALIAS" VARCHAR2(50 BYTE),
"LAST_RUN_ID" VARCHAR2(200 BYTE),
"LAST_START_TIME" DATE,
"LAST_END_TIME" DATE,
"LAST_STATUS" VARCHAR2(200 BYTE),
"SAVE_MODE" VARCHAR2(50 BYTE),
"PARTITION_COLUMN" VARCHAR2(50 BYTE),
"MAX_THREADS" NUMBER(3,0),
"LAST_PROCESSED_KEY" NUMBER(28,10),
"ACTION" VARCHAR2(30 BYTE),
"LOG_DATE" DATE,
"LOG_SOURCE" VARCHAR2(30 BYTE)
) TABLESPACE "DATA";

20
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_REPLICA_MGMT_MOPDB.sql Normal file
View File

@@ -0,0 +1,20 @@
-- ====================================================================
-- A_DEVO_REPLICA_MGMT_MOPDB Table Definition
-- ====================================================================
-- Purpose: MOPDB replication management configuration table
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_tables.sql
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_MOPDB" AS
SELECT OWNER,
TABLE_NAME,
TABLE_ALIAS,
CAST(NULL AS VARCHAR2(200)) AS LAST_RUN_ID,
CAST(NULL AS DATE) AS LAST_START_TIME,
CAST(NULL AS DATE) AS LAST_END_TIME,
CAST(NULL AS VARCHAR2(200)) AS LAST_STATUS,
SAVE_MODE,
BUCKET_COLUMN AS PARTITION_COLUMN,
NUMBER_OF_BUCKETS AS MAX_THREADS,
CAST(NULL AS NUMBER(28,10)) AS LAST_PROCESSED_KEY
FROM CT_MOPDB.MOPDB_ON_DISC_REPLICA_MGMT;

20
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_REPLICA_MGMT_RAR.sql Normal file
View File

@@ -0,0 +1,20 @@
-- ====================================================================
-- A_DEVO_REPLICA_MGMT_RAR Table Definition
-- ====================================================================
-- Purpose: RAR replication management configuration table
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_tables.sql
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_RAR" AS
SELECT OWNER,
TABLE_NAME,
TABLE_ALIAS,
CAST(NULL AS VARCHAR2(200)) AS LAST_RUN_ID,
CAST(NULL AS DATE) AS LAST_START_TIME,
CAST(NULL AS DATE) AS LAST_END_TIME,
CAST(NULL AS VARCHAR2(200)) AS LAST_STATUS,
SAVE_MODE,
BUCKET_COLUMN AS PARTITION_COLUMN,
NUMBER_OF_BUCKETS AS MAX_THREADS,
CAST(NULL AS NUMBER(28,10)) AS LAST_PROCESSED_KEY
FROM CT_REF.RAR_ON_DISC_REPLICA_MGMT;

20
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_REPLICA_MGMT_RQSD.sql Normal file
View File

@@ -0,0 +1,20 @@
-- ====================================================================
-- A_DEVO_REPLICA_MGMT_RQSD Table Definition
-- ====================================================================
-- Purpose: RQSD replica management configuration table
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_REPLICA_MGMT_RQSD" (
"OWNER" VARCHAR2(50 BYTE),
"TABLE_NAME" VARCHAR2(50 BYTE),
"TABLE_ALIAS" VARCHAR2(50 BYTE),
"LAST_RUN_ID" VARCHAR2(200 BYTE),
"LAST_START_TIME" DATE,
"LAST_END_TIME" DATE,
"LAST_STATUS" VARCHAR2(200 BYTE),
"SAVE_MODE" VARCHAR2(50 BYTE),
"PARTITION_COLUMN" VARCHAR2(50 BYTE),
"MAX_THREADS" NUMBER(3,0),
"LAST_PROCESSED_KEY" NUMBER(28,10)
) TABLESPACE "DATA";

14
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_DEVO_SOURCES_IGAM.sql Normal file
View File

@@ -0,0 +1,14 @@
-- ====================================================================
-- A_DEVO_SOURCES_IGAM Table Definition
-- ====================================================================
-- Purpose: IGAM sources configuration for DEVO environments
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE TABLE "CT_MRDS"."A_DEVO_SOURCES_IGAM" (
"ENVIRONMENT" VARCHAR2(100 BYTE),
"MRDS_SOURCE" VARCHAR2(30 BYTE),
"MRDS_SUBSOURCE_ID" VARCHAR2(30 BYTE),
"MRDS_ENTITLEMENT" VARCHAR2(100 BYTE),
"SERVICE_NAME" VARCHAR2(5 BYTE)
) TABLESPACE "DATA";

13
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_FILE_MANAGER_CONFIG.sql Normal file
View File

@@ -0,0 +1,13 @@
-- ====================================================================
-- A_FILE_MANAGER_CONFIG Table
-- ====================================================================
-- Purpose: Store environment configuration variables for file manager
-- ====================================================================
CREATE TABLE CT_MRDS.A_FILE_MANAGER_CONFIG (
ENVIRONMENT_ID VARCHAR2(100),
CONFIG_VARIABLE VARCHAR2(200),
CONFIG_VARIABLE_VALUE VARCHAR2(200),
CONFIG_VARIABLE_COMMENT VARCHAR2(600),
PRIMARY KEY (ENVIRONMENT_ID, CONFIG_VARIABLE)
) TABLESPACE "DATA";

44
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_PACKAGE_VERSION_TRACKING.sql Normal file
View File

@@ -0,0 +1,44 @@
-- ====================================================================
-- A_PACKAGE_VERSION_TRACKING Table
-- ====================================================================
-- Purpose: Track package versions, source code hashes, and deployment history
-- This table enables automatic detection of package changes even when
-- version numbers are not updated by developers
-- ====================================================================
CREATE TABLE CT_MRDS.A_PACKAGE_VERSION_TRACKING (
A_PACKAGE_VERSION_TRACKING_KEY NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
PACKAGE_OWNER VARCHAR2(128) NOT NULL,
PACKAGE_NAME VARCHAR2(128) NOT NULL,
PACKAGE_TYPE VARCHAR2(20) NOT NULL CHECK (PACKAGE_TYPE IN ('SPEC', 'BODY', 'BOTH')),
PACKAGE_VERSION VARCHAR2(10),
PACKAGE_BUILD_DATE VARCHAR2(20),
PACKAGE_AUTHOR VARCHAR2(100),
SOURCE_CODE_HASH_SPEC VARCHAR2(64), -- SHA256 hash of specification source
SOURCE_CODE_HASH_BODY VARCHAR2(64), -- SHA256 hash of body source
LINE_COUNT_SPEC NUMBER,
LINE_COUNT_BODY NUMBER,
DETECTED_CHANGE_WITHOUT_VERSION CHAR(1) DEFAULT 'N' CHECK (DETECTED_CHANGE_WITHOUT_VERSION IN ('Y', 'N')),
CHANGE_DETECTION_MESSAGE VARCHAR2(4000),
TRACKING_DATE TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL,
TRACKED_BY_USER VARCHAR2(128) DEFAULT USER NOT NULL,
TRACKED_BY_MODULE VARCHAR2(64) DEFAULT SYS_CONTEXT('USERENV','MODULE'),
CONSTRAINT UK_PKG_VERSION_TRACK UNIQUE (PACKAGE_OWNER, PACKAGE_NAME, TRACKING_DATE)
) TABLESPACE "DATA";
-- Comments
COMMENT ON TABLE CT_MRDS.A_PACKAGE_VERSION_TRACKING IS 'Tracks package version history and source code hashes for change detection';
COMMENT ON COLUMN CT_MRDS.A_PACKAGE_VERSION_TRACKING.SOURCE_CODE_HASH_SPEC IS 'SHA256 hash of package specification source code';
COMMENT ON COLUMN CT_MRDS.A_PACKAGE_VERSION_TRACKING.SOURCE_CODE_HASH_BODY IS 'SHA256 hash of package body source code';
COMMENT ON COLUMN CT_MRDS.A_PACKAGE_VERSION_TRACKING.DETECTED_CHANGE_WITHOUT_VERSION IS 'Y if source hash changed but version did not';
COMMENT ON COLUMN CT_MRDS.A_PACKAGE_VERSION_TRACKING.CHANGE_DETECTION_MESSAGE IS 'Details about detected changes without version update';
-- Grant access to other schemas
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_PACKAGE_VERSION_TRACKING TO MRDS_LOADER;
-- Performance indexes for package version tracking
-- Index for quick lookups by package name and owner
CREATE INDEX IDX_PKG_TRACK_NAME ON CT_MRDS.A_PACKAGE_VERSION_TRACKING(PACKAGE_OWNER, PACKAGE_NAME);
-- Index for date-based queries
CREATE INDEX IDX_PKG_TRACK_DATE ON CT_MRDS.A_PACKAGE_VERSION_TRACKING(TRACKING_DATE);

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_PROCESS_LOG.sql Normal file
View File

@@ -0,0 +1,22 @@
DROP TABLE CT_MRDS.A_PROCESS_LOG;
CREATE TABLE CT_MRDS.A_PROCESS_LOG (
a_process_log_key NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
guid VARCHAR2(32),
Username VARCHAR2(128),
Osuser VARCHAR2(128),
Machine VARCHAR2(64),
Module VARCHAR2(64),
process_name VARCHAR2(200),
procedure_name VARCHAR2(200),
procedure_parameters VARCHAR2(20000),
log_level VARCHAR2(10), -- DEBUG, INFO, WARNING, ERROR
log_message VARCHAR2(20000),
log_timestamp TIMESTAMP DEFAULT SYSTIMESTAMP
)
PARTITION BY RANGE (log_timestamp)
INTERVAL (NUMTODSINTERVAL(1,'DAY'))
(
PARTITION p_init VALUES LESS THAN (TIMESTAMP '2025-01-01 00:00:00')
);
/

18
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE.sql Normal file
View File

@@ -0,0 +1,18 @@
-- ====================================================================
-- A_SOURCE Table
-- ====================================================================
-- Purpose: Store source system definitions
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE (
A_SOURCE_KEY VARCHAR2(30),
SOURCE_NAME VARCHAR2(200),
CONSTRAINT A_SOURCE_PK PRIMARY KEY (A_SOURCE_KEY)
) TABLESPACE "DATA";
-- Primary key index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_PK"
ON "CT_MRDS"."A_SOURCE" ("A_SOURCE_KEY")
TABLESPACE "DATA";
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE TO MRDS_LOADER_ROLE;

45
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_CONFIG.sql Normal file
View File

@@ -0,0 +1,45 @@
-- ====================================================================
-- A_SOURCE_FILE_CONFIG Table
-- ====================================================================
-- Purpose: Store source file configuration and processing rules
-- MARS-1049: Added ENCODING column for CSV character set support
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_CONFIG (
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
A_SOURCE_KEY VARCHAR2(30) NOT NULL ENABLE,
SOURCE_FILE_TYPE VARCHAR2(200), -- Can be 'INPUT' or 'CONTAINER' or 'LOAD_CONFIG'
SOURCE_FILE_ID VARCHAR2(200),
SOURCE_FILE_DESC VARCHAR2(2000),
SOURCE_FILE_NAME_PATTERN VARCHAR2(200),
TABLE_ID VARCHAR2(200),
TEMPLATE_TABLE_NAME VARCHAR2(200),
CONTAINER_FILE_KEY NUMBER(38,0),
DAYS_FOR_ARCHIVE_THRESHOLD NUMBER(4,0),
FILES_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
BYTES_SUM_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
ODS_SCHEMA_NAME VARCHAR2(100),
ROWS_COUNT_OVER_ARCHIVE_THRESHOLD NUMBER(38,0),
HOURS_TO_EXPIRE_STATISTICS NUMBER(38,3),
ENCODING VARCHAR2(50) DEFAULT 'UTF8',
CONSTRAINT A_SOURCE_FILE_CONFIG_PK PRIMARY KEY (A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT SOURCE_FILE_TYPE_CHK CHECK (SOURCE_FILE_TYPE IN ('INPUT', 'CONTAINER', 'LOAD_CONFIG')),
CONSTRAINT ASFC_A_SOURCE_KEY_FK FOREIGN KEY(A_SOURCE_KEY) REFERENCES CT_MRDS.A_SOURCE(A_SOURCE_KEY),
CONSTRAINT ASFC_CONTAINER_FILE_KEY_FK FOREIGN KEY(CONTAINER_FILE_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_CONFIG_UQ1 UNIQUE(SOURCE_FILE_TYPE, SOURCE_FILE_ID, TABLE_ID)
) TABLESPACE "DATA";
-- Primary key index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_PK"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("A_SOURCE_FILE_CONFIG_KEY")
TABLESPACE "DATA";
-- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_SOURCE_FILE_CONFIG_UQ1"
ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABLE_ID")
TABLESPACE "DATA";
-- Column comments
COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049';
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE;

29
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_SOURCE_FILE_RECEIVED.sql Normal file
View File

@@ -0,0 +1,29 @@
-- ====================================================================
-- A_SOURCE_FILE_RECEIVED Table
-- ====================================================================
-- Purpose: Track received files and their processing status
-- ====================================================================
CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE,
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL,
CHECKSUM VARCHAR2(128),
CREATED TIMESTAMP(6) WITH TIME ZONE,
BYTES NUMBER,
RECEPTION_DATE DATE NOT NULL,
PROCESSING_STATUS VARCHAR2(200),
EXTERNAL_TABLE_NAME VARCHAR2(200),
PARTITION_YEAR VARCHAR2(4),
PARTITION_MONTH VARCHAR2(2),
ARCH_FILE_NAME VARCHAR2(1000),
CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY),
CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED'))
) TABLESPACE "DATA";
-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)
CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1
ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES);
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE;

25
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT.sql Normal file
View File

@@ -0,0 +1,25 @@
-- ====================================================================
-- A_TABLE_STAT Table
-- ====================================================================
-- Purpose: Store current table statistics and archival thresholds
-- ====================================================================
CREATE TABLE CT_MRDS.A_TABLE_STAT (
A_TABLE_STAT_KEY NUMBER(38,0) PRIMARY KEY,
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL,
TABLE_NAME VARCHAR2(200) NOT NULL,
FILE_COUNT NUMBER(38,0),
OVER_ARCH_THRESOLD_FILE_COUNT NUMBER(38,0),
ROW_COUNT NUMBER(38,0),
OVER_ARCH_THRESOLD_ROW_COUNT NUMBER(38,0),
SIZE NUMBER(38,0),
OVER_ARCH_THRESOLD_SIZE NUMBER(38,0),
ARCH_THRESHOLD_DAYS NUMBER(4,0),
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP,
CONSTRAINT A_TABLE_STAT_UK1 UNIQUE(A_SOURCE_FILE_CONFIG_KEY)
) TABLESPACE "DATA";
-- Unique constraint index (from production export)
CREATE UNIQUE INDEX "CT_MRDS"."A_TABLE_STAT_UK1"
ON "CT_MRDS"."A_TABLE_STAT" ("A_SOURCE_FILE_CONFIG_KEY")
TABLESPACE "DATA";

19
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TABLE_STAT_HIST.sql Normal file
View File

@@ -0,0 +1,19 @@
-- ====================================================================
-- A_TABLE_STAT_HIST Table
-- ====================================================================
-- Purpose: Store historical table statistics for trend analysis
-- ====================================================================
CREATE TABLE CT_MRDS.A_TABLE_STAT_HIST (
A_TABLE_STAT_HIST_KEY NUMBER(38,0) PRIMARY KEY,
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL,
TABLE_NAME VARCHAR2(200) NOT NULL,
FILE_COUNT NUMBER(38,0),
OVER_ARCH_THRESOLD_FILE_COUNT NUMBER(38,0),
ROW_COUNT NUMBER(38,0),
OVER_ARCH_THRESOLD_ROW_COUNT NUMBER(38,0),
SIZE NUMBER(38,0),
OVER_ARCH_THRESOLD_SIZE NUMBER(38,0),
ARCH_THRESHOLD_DAYS NUMBER(4,0),
CREATED TIMESTAMP(6) DEFAULT SYSTIMESTAMP
) TABLESPACE "DATA";

32
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TASK_HISTORY.sql Normal file
View File

@@ -0,0 +1,32 @@
-- ====================================================================
-- A_TASK_HISTORY Table
-- ====================================================================
-- Purpose: Log of subprocesses being part of the DWH flows
-- ====================================================================
CREATE TABLE CT_MRDS.A_TASK_HISTORY (
A_TASK_HISTORY_KEY NUMBER(38,0) NOT NULL ENABLE,
A_WORKFLOW_HISTORY_KEY NUMBER(38,0),
TASK_RUN_ID VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
TASK_NAME VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
TASK_START TIMESTAMP(6),
TASK_END TIMESTAMP(6),
TASK_STATUS VARCHAR2(200) COLLATE "USING_NLS_COMP"
) TABLESPACE "DATA";
-- Comments
COMMENT ON TABLE CT_MRDS.A_TASK_HISTORY IS 'Log of subprocesses being part of the DWH flows';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.A_TASK_HISTORY_KEY IS 'Surrogate key for logged subprocesses';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.A_WORKFLOW_HISTORY_KEY IS 'Technical key to identify a load operation. Generated with Oracle sequence "CT_MRDS.SA_TASK_HISTORY_KEY_SEQ"';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_NAME IS 'Mapping name of the logged subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_START IS 'Start time of the subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_END IS 'End time of the subprocess';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY.TASK_STATUS IS 'Indicates that the subprocess completed successfully. Allowed values are ''RUNNING'',''FAILED'' and ''SUCCESS''.';
-- Grants
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_TASK_HISTORY TO MRDS_LOADER;
-- Bitmap index for task history queries
CREATE BITMAP INDEX CT_MRDS.A_TASK_HISTORY_IDX1
ON CT_MRDS.A_TASK_HISTORY (A_WORKFLOW_HISTORY_KEY)
TABLESPACE "DATA";

19
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TASK_HISTORY_SOURCE.sql Normal file
View File

@@ -0,0 +1,19 @@
-- ====================================================================
-- A_TASK_HISTORY_SOURCE Table
-- ====================================================================
-- Purpose: Track source data statistics for task history
-- ====================================================================
CREATE TABLE CT_MRDS.A_TASK_HISTORY_SOURCE (
A_TASK_HISTORY_SOURCE_KEY NUMBER(38,0) NOT NULL ENABLE,
A_TASK_HISTORY_KEY NUMBER(38,0),
A_WORKFLOW_HISTORY_SOURCE_KEY NUMBER(38,0),
SOURCE_NAME VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
ROW_COUNT NUMBER(38,0)
) TABLESPACE "DATA";
-- Comments
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.A_TASK_HISTORY_SOURCE_KEY IS 'Technical primary key';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.A_TASK_HISTORY_KEY IS 'Foreign key referencing the subprocess delivering to the target';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.SOURCE_NAME IS 'Name of the DWH target table written in the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_SOURCE.ROW_COUNT IS 'Number of rows read from the source during the load process';

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_TASK_HISTORY_TARGET.sql Normal file
View File

@@ -0,0 +1,22 @@
-- ====================================================================
-- A_TASK_HISTORY_TARGET Table
-- ====================================================================
-- Purpose: Track target data statistics for task history
-- ====================================================================
CREATE TABLE CT_MRDS.A_TASK_HISTORY_TARGET (
A_TASK_HISTORY_TARGET_KEY NUMBER(38,0) NOT NULL ENABLE,
A_TASK_HISTORY_KEY NUMBER(38,0),
TARGET_NAME VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
ROW_COUNT_APPLIED NUMBER(38,0),
ROW_COUNT_REJECTED NUMBER(38,0),
LOAD_STATUS CHAR(1 CHAR) COLLATE "USING_NLS_COMP"
) TABLESPACE "DATA";
-- Comments
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.A_TASK_HISTORY_TARGET_KEY IS 'Technical primary key';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.A_TASK_HISTORY_KEY IS 'Foreign key referencing the subprocess delivering to the target';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.TARGET_NAME IS 'Name of the DWH target table written in the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.ROW_COUNT_APPLIED IS 'Number of rows succesfully applied (written to) the DWH target table during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.ROW_COUNT_REJECTED IS 'Number of rows rejected (NOT written to) the DWH target table during the load process';
COMMENT ON COLUMN CT_MRDS.A_TASK_HISTORY_TARGET.LOAD_STATUS IS 'Allowed values are RUNNING, SUCCESS and FAILED.';

32
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,32 @@
-- ====================================================================
-- A_WORKFLOW_HISTORY Table
-- ====================================================================
-- Purpose: Load history table for the Data Warehouse (DWH)
-- ====================================================================
CREATE TABLE CT_MRDS.A_WORKFLOW_HISTORY (
A_WORKFLOW_HISTORY_KEY NUMBER(38,0) NOT NULL ENABLE,
ORCHESTRATION_RUN_ID VARCHAR2(300),
WORKFLOW_NAME VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
WORKFLOW_START TIMESTAMP(6),
WORKFLOW_END TIMESTAMP(6),
WORKFLOW_STATUS VARCHAR2(200),
DBT_INVOCATION_ID VARCHAR2(200)
) TABLESPACE "DATA";
-- Comments
COMMENT ON TABLE CT_MRDS.A_WORKFLOW_HISTORY IS 'Load history table for the Data Warehouse (DWH)';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY IS 'Technical key to identify a load operation. Generated with Oracle sequence "CT_MRDS.SEQ_WORKFLOW_HISTORY"';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.ORCHESTRATION_RUN_ID IS 'Run_id of the workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_NAME IS 'The name of the workflow, this is always w_DWH_RAR_LOAD';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_START IS 'Start time of the Informatica workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_END IS 'End time of the Informatica workflow';
COMMENT ON COLUMN CT_MRDS.A_WORKFLOW_HISTORY.WORKFLOW_STATUS IS 'Overall status of the workflow, either RUNNING|SUCCESS|ERROR';
-- Grants
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_WORKFLOW_HISTORY TO MRDS_LOADER;
-- Bitmap index for workflow history queries
CREATE BITMAP INDEX CT_MRDS.A_WORKFLOW_HISTORY_IDX1
ON CT_MRDS.A_WORKFLOW_HISTORY (A_WORKFLOW_HISTORY_KEY, WORKFLOW_NAME, WORKFLOW_START)
TABLESPACE "DATA";

18
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/A_WORKFLOW_HISTORY_PROPERTY.sql Normal file
View File

@@ -0,0 +1,18 @@
-- ====================================================================
-- A_WORKFLOW_HISTORY_PROPERTY Table
-- ====================================================================
-- Purpose: Store workflow history properties
-- ====================================================================
CREATE TABLE CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY (
A_WORKFLOW_HISTORY_KEY NUMBER,
SERVICE_NAME VARCHAR2(200),
PROPERTY VARCHAR2(200),
VALUE VARCHAR2(200),
CONSTRAINT A_WORKFLOW_HISTORY_FK FOREIGN KEY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME)
REFERENCES CT_MRDS.A_WORKFLOW_HISTORY (A_WORKFLOW_HISTORY_KEY, SERVICE_NAME)
) TABLESPACE "DATA";
-- Index for performance on workflow history property queries
CREATE INDEX CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY_IDX1
ON CT_MRDS.A_WORKFLOW_HISTORY_PROPERTY(A_WORKFLOW_HISTORY_KEY, SERVICE_NAME);

25
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/tables/ERR_LOG.sql Normal file
View File

@@ -0,0 +1,25 @@
-- ====================================================================
-- ERR_LOG Table Definition
-- ====================================================================
-- Purpose: Error logging table for workflow and mapping errors
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_tables.sql
-- ====================================================================
CREATE TABLE "CT_MRDS"."ERR_LOG"
("ERR_LOG_KEY" NUMBER(*,0),
"A_WORKFLOW_HISTORY_KEY" NUMBER(38,0),
"ERR_MESSAGE_FK" NUMBER,
"ERR_SEVERITY_FK" NUMBER,
"ERR_WORKFLOW_RUN_ID" CHAR(17 BYTE) COLLATE "USING_NLS_COMP",
"ERR_WORKFLOW_NAME" CHAR(18 BYTE) COLLATE "USING_NLS_COMP",
"ERR_MAPPING_NAME" CHAR(74 BYTE) COLLATE "USING_NLS_COMP",
"ERR_LOCATION" CHAR(74 BYTE) COLLATE "USING_NLS_COMP",
"ERR_SOURCE" CHAR(22 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_BUSINESS" CHAR(6 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_BUSINESS_VALUE" CHAR(14 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_TECHNICAL" CHAR(22 BYTE) COLLATE "USING_NLS_COMP",
"ERR_KEY_TECHNICAL_VALUE" VARCHAR2(40 BYTE) COLLATE "USING_NLS_COMP",
"ERR_DESCRIPTION" CHAR(94 BYTE) COLLATE "USING_NLS_COMP",
"ERR_MESSAGE" VARCHAR2(255 BYTE) COLLATE "USING_NLS_COMP",
"INDEP_SUBPROCESS_FK" NUMBER(38,0)
);

35
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/triggers/TRG_A_DEVO_REPLICA_MGMT_MOPDB.sql Normal file
View File

@@ -0,0 +1,35 @@
-- ====================================================================
-- TRG_A_DEVO_REPLICA_MGMT_MOPDB Trigger Definition
-- ====================================================================
-- Purpose: Audit trigger for A_DEVO_REPLICA_MGMT_MOPDB operations
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."TRG_A_DEVO_REPLICA_MGMT_MOPDB"
AFTER INSERT OR UPDATE OR DELETE
ON CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB
FOR EACH ROW
DECLARE
vAction VARCHAR2(100);
BEGIN
IF INSERTING THEN
vAction := 'INSERT';
ELSIF UPDATING THEN
vAction := 'UPDATE';
ELSIF DELETING THEN
vAction := 'DELETE';
END IF;
INSERT INTO
CT_MRDS.A_DEVO_REPLICA_MGMT_LOG (OWNER, TABLE_NAME, TABLE_ALIAS,
LAST_RUN_ID, LAST_START_TIME, LAST_END_TIME,
LAST_STATUS, SAVE_MODE, PARTITION_COLUMN,
MAX_THREADS, LAST_PROCESSED_KEY, ACTION,
LOG_DATE, LOG_SOURCE)
VALUES (:NEW.OWNER, :NEW.TABLE_NAME, :NEW.TABLE_ALIAS,
:NEW.LAST_RUN_ID, :NEW.LAST_START_TIME, :NEW.LAST_END_TIME,
:NEW.LAST_STATUS, :NEW.SAVE_MODE, :NEW.PARTITION_COLUMN,
:NEW.MAX_THREADS, :NEW.LAST_PROCESSED_KEY, vAction,
SYSDATE, 'MOPDB');
END;
/

35
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/triggers/TRG_A_DEVO_REPLICA_MGMT_RAR.sql Normal file
View File

@@ -0,0 +1,35 @@
-- ====================================================================
-- A_DEVO_REPLICA_MGMT_RAR_TRG Trigger Definition
-- ====================================================================
-- Purpose: Audit trigger for A_DEVO_REPLICA_MGMT_RAR operations
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."A_DEVO_REPLICA_MGMT_RAR_TRG"
AFTER INSERT OR UPDATE OR DELETE
ON CT_MRDS.A_DEVO_REPLICA_MGMT_RAR
FOR EACH ROW
DECLARE
vAction VARCHAR2(100);
BEGIN
IF INSERTING THEN
vAction := 'INSERT';
ELSIF UPDATING THEN
vAction := 'UPDATE';
ELSIF DELETING THEN
vAction := 'DELETE';
END IF;
INSERT INTO
CT_MRDS.A_DEVO_REPLICA_MGMT_LOG (OWNER, TABLE_NAME, TABLE_ALIAS,
LAST_RUN_ID, LAST_START_TIME, LAST_END_TIME,
LAST_STATUS, SAVE_MODE, PARTITION_COLUMN,
MAX_THREADS, LAST_PROCESSED_KEY, ACTION,
LOG_DATE, LOG_SOURCE)
VALUES (:NEW.OWNER, :NEW.TABLE_NAME, :NEW.TABLE_ALIAS,
:NEW.LAST_RUN_ID, :NEW.LAST_START_TIME, :NEW.LAST_END_TIME,
:NEW.LAST_STATUS, :NEW.SAVE_MODE, :NEW.PARTITION_COLUMN,
:NEW.MAX_THREADS, :NEW.LAST_PROCESSED_KEY, vAction,
SYSDATE, 'RAR');
END;
/

40
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/triggers/TRG_A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,40 @@
-- ====================================================================
-- TRG_A_WORKFLOW_HISTORY Trigger Definition
-- ====================================================================
-- Purpose: Trigger to insert workflow completion data to CT_ODS.A_LOAD_HISTORY
-- Source: new_EXP.sql export from production database
-- ====================================================================
CREATE OR REPLACE EDITIONABLE TRIGGER "CT_MRDS"."TRG_A_WORKFLOW_HISTORY"
AFTER INSERT OR UPDATE OF workflow_successful ON CT_MRDS.A_WORKFLOW_HISTORY
REFERENCING NEW AS new OLD AS old
FOR EACH ROW
DECLARE
v_workflow_name VARCHAR2(128);
v_wla_id NUMBER;
BEGIN
IF :new.workflow_name IN ('w_ODS_LM_STANDING_FACILITIES', 'w_ODS_CSDB_DEBT', 'w_ODS_CSDB_DEBT_DAILY', 'w_ODS_CSDB_RATINGS_FULL') AND :new.service_name = 'ODS' THEN
IF :new.workflow_successful <> :old.workflow_successful AND :new.workflow_successful = 'Y' THEN
IF :new.workflow_name = 'w_ODS_LM_STANDING_FACILITIES' THEN
v_workflow_name := 'w_ODS_LM_STANDING_FACILITY';
ELSE
v_workflow_name := :new.workflow_name;
END IF;
BEGIN
v_wla_id := TO_NUMBER(:new.orchestration_run_id);
EXCEPTION
WHEN OTHERS THEN
NULL;
END;
INSERT INTO CT_ODS.A_LOAD_HISTORY (
a_etl_load_set_key, workflow_name, infa_run_id, load_start, load_end,
exdi_appl_req_id, exdi_correlation_id, load_successful, wla_run_id, dq_flag
) VALUES (
:new.a_workflow_history_key, v_workflow_name, NULL, :new.workflow_start, :new.workflow_end,
NULL, NULL, :new.workflow_successful, v_wla_id, 'F'
);
END IF;
END IF;
END;
/

16
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/triggers/TRG_BIU_CHCK_TEMPLATE_TABLE_NAME.sql Normal file
View File

@@ -0,0 +1,16 @@
CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BIU_CHCK_TEMPLATE_TABLE_NAME
BEFORE INSERT OR UPDATE ON "CT_MRDS"."A_COLUMN_DATE_FORMAT"
FOR EACH ROW
DECLARE
vCount NUMBER;
BEGIN
SELECT COUNT(*)
INTO vCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE TEMPLATE_TABLE_NAME = :NEW.TEMPLATE_TABLE_NAME;
IF vCount = 0 THEN
RAISE_APPLICATION_ERROR(-20001, 'TEMPLATE_TABLE_NAME = '''||:NEW.TEMPLATE_TABLE_NAME||''' not exists in A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME ');
END IF;
END;
/

53
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/triggers/TRG_BI_A_SOURCE_FILE_CONFIG_CHECK.sql Normal file
View File

@@ -0,0 +1,53 @@
CREATE OR REPLACE TRIGGER CT_MRDS.TRG_BI_A_SOURCE_FILE_CONFIG_CHECK
BEFORE INSERT ON CT_MRDS.A_SOURCE_FILE_CONFIG
FOR EACH ROW
DECLARE
-- NO_CONTAINER_FOR_INPUT EXCEPTION;
WRONG_CONTAINER_FOR_INPUT EXCEPTION;
CONTAINER_ALREADY_EXISTS EXCEPTION;
vCount PLS_INTEGER;
vContainerFileKey CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE;
BEGIN
--Check if there is CONTAINER entity in A_SOURCE_FILE_CONFIG table
IF (:NEW.SOURCE_FILE_TYPE = 'INPUT' AND :NEW.CONTAINER_FILE_KEY IS NULL) THEN
:NEW.CONTAINER_FILE_KEY := CT_MRDS.FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(:NEW.SOURCE_FILE_ID);
-- IF :NEW.CONTAINER_FILE_KEY is null THEN
-- RAISE NO_CONTAINER_FOR_INPUT;
-- END IF;
-- Check if provided CONTAINER_FILE_KEY is the one which exists in the A_SOURCE_FILE_CONFIG table
ELSIF (:NEW.SOURCE_FILE_TYPE = 'INPUT' AND :NEW.CONTAINER_FILE_KEY IS NOT NULL) THEN
vContainerFileKey := CT_MRDS.FILE_MANAGER.GET_CONTAINER_SOURCE_FILE_CONFIG_KEY(:NEW.SOURCE_FILE_ID);
IF :NEW.CONTAINER_FILE_KEY <> vContainerFileKey THEN
RAISE WRONG_CONTAINER_FOR_INPUT;
END IF;
-- Check if there is already a CONTAINER for provided SOURCE_FILE_ID
ELSIF (:NEW.SOURCE_FILE_TYPE = 'CONTAINER') THEN
SELECT count(*)
INTO vCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE :NEW.SOURCE_FILE_TYPE = 'CONTAINER'
AND SOURCE_FILE_ID = :NEW.SOURCE_FILE_ID;
IF vCount > 0 THEN
RAISE CONTAINER_ALREADY_EXISTS;
END IF;
END IF;
SELECT CT_MRDS.A_SOURCE_FILE_CONFIG_KEY_SEQ.NEXTVAL INTO :NEW.A_SOURCE_FILE_CONFIG_KEY FROM DUAL;
EXCEPTION
-- WHEN NO_CONTAINER_FOR_INPUT THEN
-- RAISE_APPLICATION_ERROR(-20011, 'Error while adding entity where SOURCE_FILE_TYPE = ''INPUT'''
-- ||CHR(13)||CHR(10)||'There is no A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER'''
-- ||CHR(13)||CHR(10)||'Add ''CONTAINER'' config entity before ''INPUT'' one');
WHEN WRONG_CONTAINER_FOR_INPUT THEN
RAISE_APPLICATION_ERROR(-20012, 'Provided CONTAINER_FILE_KEY='||:NEW.CONTAINER_FILE_KEY||' is wrong'
||CHR(13)||CHR(10)||'There is A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER'''
||CHR(13)||CHR(10)||'Existing CONTAINER_FILE_KEY='||vContainerFileKey);
WHEN CONTAINER_ALREADY_EXISTS THEN
RAISE_APPLICATION_ERROR(-20013, 'Such CONTAINER already exists!'
||CHR(13)||CHR(10)||'There is A_SOURCE_FILE_CONFIG entity where SOURCE_FILE_ID = '''||:NEW.SOURCE_FILE_ID||''' AND SOURCE_FILE_TYPE = ''CONTAINER''');
END;
/

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/types/T_FILENAME_TYPES.sql Normal file
View File

@@ -0,0 +1,22 @@
-- ====================================================================
-- Oracle Object Types for File Management
-- ====================================================================
-- Purpose: Define object types for filename handling in file management system
-- ====================================================================
-- Drop existing types if they exist
DROP TYPE CT_MRDS.T_FILENAMES;
DROP TYPE CT_MRDS.T_FILENAME;
-- Create filename object type
CREATE OR REPLACE TYPE CT_MRDS.T_FILENAME AS OBJECT (
FILENAME VARCHAR2(200),
PATHNAME VARCHAR2(800),
YEAR VARCHAR2(4),
MONTH VARCHAR2(2)
);
/
-- Create collection of filename objects
CREATE TYPE CT_MRDS.T_FILENAMES IS TABLE OF T_FILENAME;
/

29
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/views/A_DWH_LOAD_HISTORY.sql Normal file
View File

@@ -0,0 +1,29 @@
-- ====================================================================
-- A_DWH_LOAD_HISTORY View Definition
-- ====================================================================
-- Purpose: View for DWH (RAR) load history with workflow and task tracking
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_views.sql
-- ====================================================================
CREATE OR REPLACE FORCE EDITIONABLE VIEW "CT_MRDS"."A_DWH_LOAD_HISTORY"
("A_WORKFLOW_HISTORY_KEY", "A_WORKFLOW_HISTORY_SOURCE_KEY", "A_TASK_HISTORY_KEY",
"TASK_NAME", "TASK_START", "TASK_END", "WORKFLOW_SUCCESSFUL")
DEFAULT COLLATION "USING_NLS_COMP" AS
SELECT DISTINCT wh.a_workflow_history_key,
ths.a_workflow_history_source_key,
th.a_task_history_key,
th.task_name,
th.task_start,
th.task_end,
wh.workflow_successful
FROM CT_MRDS.A_WORKFLOW_HISTORY wh
JOIN CT_MRDS.A_TASK_HISTORY th
ON wh.a_workflow_history_key = th.a_workflow_history_key
AND wh.service_name = th.service_name
JOIN CT_MRDS.A_TASK_HISTORY_SOURCE ths
ON th.a_task_history_key = ths.a_task_history_key
AND th.service_name = ths.service_name
WHERE wh.service_name = 'RAR'
AND wh.workflow_successful = 'Y'
AND LOWER(th.task_name) NOT LIKE '%_sq'
AND ths.a_workflow_history_source_key IS NOT NULL;

29
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/views/A_MOPDB_LOAD_HISTORY.sql Normal file
View File

@@ -0,0 +1,29 @@
-- ====================================================================
-- A_MOPDB_LOAD_HISTORY View Definition
-- ====================================================================
-- Purpose: View for MOPDB load history with workflow and task tracking
-- Source: MARS-770 install_package_MarsGeneralRelease_MARS770_views.sql
-- ====================================================================
CREATE OR REPLACE FORCE EDITIONABLE VIEW "CT_MRDS"."A_MOPDB_LOAD_HISTORY"
("A_WORKFLOW_HISTORY_KEY", "A_WORKFLOW_HISTORY_SOURCE_KEY", "A_TASK_HISTORY_KEY",
"TASK_NAME", "TASK_START", "TASK_END", "WORKFLOW_SUCCESSFUL")
DEFAULT COLLATION "USING_NLS_COMP" AS
SELECT DISTINCT wh.a_workflow_history_key,
ths.a_workflow_history_source_key,
th.a_task_history_key,
th.task_name,
th.task_start,
th.task_end,
wh.workflow_successful
FROM CT_MRDS.A_WORKFLOW_HISTORY wh
JOIN CT_MRDS.A_TASK_HISTORY th
ON wh.a_workflow_history_key = th.a_workflow_history_key
AND wh.service_name = th.service_name
JOIN CT_MRDS.A_TASK_HISTORY_SOURCE ths
ON th.a_task_history_key = ths.a_task_history_key
AND th.service_name = ths.service_name
WHERE wh.service_name = 'MOPDB'
AND wh.workflow_successful = 'Y'
AND LOWER(th.task_name) NOT LIKE '%_sq'
AND ths.a_workflow_history_source_key IS NOT NULL;

17
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/views/VW_A_PROCESS_LOG.sql Normal file
View File

@@ -0,0 +1,17 @@
create or replace view CT_MRDS.VW_A_PROCESS_LOG as
select * from (
select
l.a_process_log_key
,l.guid
,l.process_name
,l.procedure_name
,l.log_level
,l.log_message
,l.log_timestamp
,lead(log_message) over (partition by l.process_name, l.procedure_name, guid order by l.a_process_log_key) as next_log_message
,lead(log_timestamp) over (partition by l.process_name, l.procedure_name, guid order by l.a_process_log_key) as next_log_timestamp
from CT_MRDS.a_process_log l
) where 1=1
and log_message='Start'
order by log_timestamp
;

16
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/views/VW_FILE_RECONCILIATION.sql Normal file
View File

@@ -0,0 +1,16 @@
create or replace view CT_MRDS.VW_FILE_RECONCILIATION as
select
case
when c.bytes is null then 'RECEIVED_MISSING_ON_CLOUD'
when r.bytes is null then 'NOT_RECEIVED'
when r.bytes is not null and c.bytes is not null then 'RECEIVED'
else 'UNKNOWN'
end as status
,c.object_name as cloud_file
,r.*
from DBMS_CLOUD.LIST_OBJECTS('OCI$RESOURCE_PRINCIPAL', CT_MRDS.FILE_MANAGER.GET_INBOX_BUCKET_URI()) c
full join a_source_file_received r
on c.bytes = r.bytes
and c.checksum = r.checksum
and c.created = r.created
;

15
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SCHEMA/views/VW_FILE_RECONCILIATION_SUMMARY.sql Normal file
View File

@@ -0,0 +1,15 @@
create or replace view CT_MRDS.VW_FILE_RECONCILIATION_SUMMARY as
select
all_statuses.status
,nvl(recon_stats.cnt, 0) as number_of_files
from (
select 'RECEIVED_MISSING_ON_CLOUD' as status from dual
union all select 'NOT_RECEIVED' from dual
union all select 'RECEIVED' from dual
union all select 'UNKNOWN' from dual
) all_statuses
left join (select status, count(*) as cnt from CT_MRDS.VW_FILE_RECONCILIATION group by status
) recon_stats
on all_statuses.status = recon_stats.status
order by 1
;

22
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/CT_ODS.A_LOAD_HISTORY.sql Normal file
View File

@@ -0,0 +1,22 @@
ALTER TABLE CT_ODS.A_LOAD_HISTORY RENAME COLUMN A_ETL_LOAD_SET_KEY TO A_WORKFLOW_HISTORY_KEY;
create or replace TRIGGER "CT_ODS"."AUTOKEY_A_LOAD_HISTORY"
BEFORE INSERT
ON "CT_ODS"."A_LOAD_HISTORY"
FOR EACH ROW
DECLARE
e_more_than_28_digits EXCEPTION;
BEGIN
IF :NEW.A_WORKFLOW_HISTORY_KEY IS NULL THEN
:NEW.A_WORKFLOW_HISTORY_KEY := AUTOKEY_A_LOAD_HISTORY.NEXTVAL;
ELSE
-- Informatica PowerCenter cannot handle more than 28 significant places
-- so raise exception if more were given manually
IF :NEW.A_WORKFLOW_HISTORY_KEY > 9999999999999999999999999999 THEN
RAISE e_more_than_28_digits;
END IF;
END IF;
END;
/

1
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/ODS_TMS_ACMCURRENCYFLOW.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE OU_TMS.ACMCURRENCYFLOW RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.ACMENTRYSTATELEDGERGROUP RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_ACMENTRYSTATELEDGERGROUP;

28
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_C2D_A_UC_DISSEM_METADATA_LOADS.sql Normal file
View File

@@ -0,0 +1,28 @@
--
-- Properly set up C2D UC Dissem
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.C2D_A_UC_DISSEM_METADATA_LOADS
CREATE TABLE CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS
AS SELECT * FROM OU_C2D.A_UC_DISSEM_METADATA_LOADS WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_A_UC_DISSEM_METADATA_LOADS_INBOX','CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS','INBOX/C2D/UC_DISSEM/A_UC_DISSEM_METADATA_LOADS');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_A_UC_DISSEM_METADATA_LOADS_ODS','CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS','ODS/C2D/UC_DISSEM/A_UC_DISSEM_METADATA_LOADS');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_A_UC_DISSEM_METADATA_LOADS_ARCHIVE','CT_ET_TEMPLATES.C2D_A_UC_DISSEM_METADATA_LOADS','ARCHIVE/C2D/A_UC_DISSEM_METADATA_LOADS');
-- 3. Export table OU_C2D.A_UC_DISSEM_METADATA_LOADS
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_C2D','A_UC_DISSEM_METADATA_LOADS','A_ETL_LOAD_SET_FK','mrds_inbox_tst','ARCHIVE/C2D/A_UC_DISSEM_METADATA_LOADS');
-- 4. Rename table to legacy table
ALTER TABLE OU_C2D.A_UC_DISSEM_METADATA_LOADS RENAME TO A_UC_DISSEM_METADATA_LOADS_LEGACY;
-- 5. Create OU view on top of ODS table
GRANT SELECT ON ods.C2D_A_UC_DISSEM_METADATA_LOADS_ODS TO OU_C2D;
create or replace view OU_C2D.A_UC_DISSEM_METADATA_LOADS AS SELECT * from ods.C2D_A_UC_DISSEM_METADATA_LOADS_ODS;
show errors

28
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_C2D_UC_MA_DISSEM.sql Normal file
View File

@@ -0,0 +1,28 @@
--
-- Properly set up C2D UC Dissem
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.C2D_UC_MA_DISSEM
CREATE TABLE CT_ET_TEMPLATES.C2D_UC_MA_DISSEM
AS SELECT * FROM OU_C2D.UC_MA_DISSEM WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_MA_DISSEM_INBOX','CT_ET_TEMPLATES.C2D_UC_MA_DISSEM','INBOX/C2D/UC_DISSEM/UC_MA_DISSEM');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_MA_DISSEM_ODS','CT_ET_TEMPLATES.C2D_UC_MA_DISSEM','ODS/C2D/UC_MA_DISSEM');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_MA_DISSEM_ARCHIVE','CT_ET_TEMPLATES.C2D_UC_MA_DISSEM','ARCHIVE/C2D/UC_MA_DISSEM');
-- 3. Export table OU_C2D.UC_MA_DISSEM
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_C2D','UC_MA_DISSEM','A_ETL_LOAD_SET_FK','mrds_inbox_tst','ARCHIVE/C2D/UC_MA_DISSEM');
-- 4. Rename table to legacy table
ALTER TABLE OU_C2D.UC_MA_DISSEM RENAME TO UC_MA_DISSEM_LEGACY;
-- 5. Create OU view on top of ODS table
GRANT SELECT ON ods.C2D_UC_MA_DISSEM_ODS TO OU_C2D;
create or replace view OU_C2D.UC_MA_DISSEM AS SELECT * from ods.C2D_UC_MA_DISSEM_ODS;
show errors

28
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_C2D_UC_NMA_DISSEM.sql Normal file
View File

@@ -0,0 +1,28 @@
--
-- Properly set up C2D UC Dissem
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.C2D_UC_NMA_DISSEM
CREATE TABLE CT_ET_TEMPLATES.C2D_UC_NMA_DISSEM
AS SELECT * FROM OU_C2D.UC_NMA_DISSEM WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_NMA_DISSEM_INBOX','CT_ET_TEMPLATES.C2D_UC_NMA_DISSEM','INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_NMA_DISSEM_ODS','CT_ET_TEMPLATES.C2D_UC_NMA_DISSEM','ODS/C2D/UC_NMA_DISSEM');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('C2D_UC_NMA_DISSEM_ARCHIVE','CT_ET_TEMPLATES.C2D_UC_NMA_DISSEM','ARCHIVE/C2D/UC_NMA_DISSEM');
-- 3. Export table OU_C2D.UC_NMA_DISSEM
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_C2D','UC_NMA_DISSEM','A_ETL_LOAD_SET_FK','mrds_inbox_tst','ARCHIVE/C2D/UC_NMA_DISSEM');
-- 4. Rename table to legacy table
ALTER TABLE OU_C2D.UC_NMA_DISSEM RENAME TO UC_NMA_DISSEM_LEGACY;
-- 5. Create OU view on top of ODS table
GRANT SELECT ON ods.C2D_UC_NMA_DISSEM_ODS TO OU_C2D;
create or replace view OU_C2D.UC_NMA_DISSEM AS SELECT * from ods.C2D_UC_NMA_DISSEM_ODS;
show errors

174
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_CSDB_DEBT_DAILY.sql Normal file
View File

@@ -0,0 +1,174 @@
--
-- Properly set up CSDB DEVO Instrument Ratings
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.CSDB_DEBT_DAILY
drop table "CT_ET_TEMPLATES"."CSDB_DEBT_DAILY";
CREATE TABLE "CT_ET_TEMPLATES"."CSDB_DEBT_DAILY" (
"A_KEY" NUMBER(38, 0)
NOT NULL ENABLE,
"A_WORKFLOW_HISTORY_KEY" NUMBER(38, 0)
NOT NULL ENABLE,
"NEWUPDATED" DATE,
"IDLOADDATE_DIM" DATE,
"EXTERNALCODE_ISIN" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"EXTERNALCODETYPE_NC" VARCHAR2(124 CHAR) COLLATE "USING_NLS_COMP",
"EXTERNALCODE_NATIONAL" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRINSTRUMENT" NUMBER(28, 0),
"SHORTNAME" VARCHAR2(255 CHAR) COLLATE "USING_NLS_COMP",
"IDIRDEPOSITORY" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"IDIRDEBTTYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRASSETSECTYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_CFI" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAI" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAI_DM" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCURRENCY_NOMINAL" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"AMOUNTISSUED" NUMBER(28, 10),
"AMOUNTOUTSTANDING" NUMBER(28, 10),
"AMOUNTOUTSTANDING_EUR" NUMBER(28, 10),
"POOLFACTOR" NUMBER(28, 10),
"ISSUEPRICE" NUMBER(28, 10),
"IDISSUEDATE" DATE,
"IDIRCOUPONTYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCOUPONFREQUENCY" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCURRENCY_COUPON" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATE" NUMBER(28, 10),
"COUPONDATE" DATE,
"IDIRREDEMPTIONTYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRREDEMPTIONFREQUENCY" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCURRENCY_REDEMPTION" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"REDEMPTIONPRICE" NUMBER(28, 10),
"IDMATURITYDATE" DATE,
"IDIRORGANISATIONALIASTYPE_IS" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ISSUERSOURCECODE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ISSUEREXTERNALCODE_MFI" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ISSUEREXTERNALCODE_BIC" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ISSUEREXTERNALCODE_BEI" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRORGANISATION_ISSUER" NUMBER(28, 0),
"ISSUERNAME" VARCHAR2(255 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCOUNTRY" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCOUNTRY_DM" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAO" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAO_DM" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_NACE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"PUBLICATIONPRICEDATE" DATE,
"PUBLICATIONPRICE" NUMBER(28, 10),
"PUBLICATIONPRICETYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"PUBLICATIONPRICEQUOTATIONBASIS" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"MONTHLYAVERAGEPRICE" NUMBER(28, 10),
"ACCRUALSTARTDATE" DATE,
"DEBTACCRUALDEBTOR" NUMBER(28, 10),
"DEBTACCRUALDEBTOR_DM" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"DEBTACCRUALCREDITOR" NUMBER(28, 10),
"DEBTACCRUALCREDITOR_TYP" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ACCRUEDINTEREST" NUMBER(28, 10),
"YTMNONOPTIONADJUSTED" NUMBER(28, 10),
"ESCB_ISSUER_IDENT" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ESCB_ISSUER_IDENT_TYP" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDUDCMPPARTY" NUMBER(28, 0),
"AMOUNTOUTSTANDINGTYPE" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"MARKETCAPITALISATION" NUMBER(28, 10),
"MARKETCAPITALISATION_EUR" NUMBER(28, 10),
"VA_SECURITYSTATUS" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"VA_INSTRSUPPLEMENTARYCLASS" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"VA_RESIDUALMATURITYCLASS" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"VA_ISINSEC" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"VA_ISELIGIBLEFOREADB" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAI10" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRCLASSIFICATIONCODE_ESAO10" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"IDIRDEBTTYPE_N" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"SENIORITY" VARCHAR2(32 CHAR) COLLATE "USING_NLS_COMP",
"ISSUEREXTERNALCODE_LEI" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"INSTR_ESA2010_CLASS_VALUETYPE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"ISS_ESA2010_CLASS_VALUETYPE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"SEC_STATUS_DATE" DATE,
"GROUP_TYPE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"HAS_EMBEDDED_OPTION" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"VOLUME_TRADED" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PRIMARY_LISTING_NAME" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PRIM_LISTING_RESIDENCY_COUNTRY" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"INSTR_PORTFOLIO_FLAGS" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"BOND_DURATION" NUMBER(28, 10),
"RESIDUAL_MATURITY" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"ORIGINAL_MATURITY" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"CFIN_CLASSIFICATION" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONFIRSTPAYMENTDATE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONLASTPAYMENTDATE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATEUNDERLYINGCODE_ISIN" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATESPREAD" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATEMULTIPLIER" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATECAP" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"COUPONRATEFLOOR" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"IDISSUEDATE_TRANCHE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"ISSUEPRICE_TRANCHE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"VA_ISPRIVATEPLACEMENT" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"RIAD_CODE" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"RIAD_OUID" NUMBER(38, 0),
"ESG1" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"ESG2" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"ESG3" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"STRIP" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"DEPOSITORY_RECEIPT" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"RULE_144A" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"REG_S" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"WARRANT" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"CSEC_RELEVANCE_STOCK" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"CSEC_RELEVANCE_GROSS_ISSUANCE" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"CSEC_RELEVANCE_REDEMPTION" VARCHAR2(1 CHAR) COLLATE "USING_NLS_COMP",
"ACCRUING_COUPON" NUMBER(28, 10),
"ACCRUING_DISCOUNT" NUMBER(28, 10),
"PLACEHOLDER32" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER33" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER34" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER35" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER36" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER37" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER38" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER39" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER40" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER41" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER42" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER43" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER44" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER45" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER46" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER47" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER48" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER49" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP",
"PLACEHOLDER50" VARCHAR2(4000 CHAR) COLLATE "USING_NLS_COMP"
) DEFAULT COLLATION "USING_NLS_COMP"
SEGMENT CREATION IMMEDIATE
PCTFREE 10 PCTUSED 40 INITRANS 10 MAXTRANS 255 COLUMN STORE COMPRESS FOR QUERY HIGH ROW LEVEL LOCKING LOGGING
STORAGE ( INITIAL 65536 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645 PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1 BUFFER_POOL DEFAULT
FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT )
TABLESPACE "DATA";
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_DEBT_DAILY_INBOX','CT_ET_TEMPLATES.CSDB_DEBT_DAILY','INBOX/CSDB/DEBT_DAILY/DEBT_DAILY');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_DEBT_DAILY_ODS','CT_ET_TEMPLATES.CSDB_DEBT_DAILY','ODS/CSDB/DEBT_DAILY');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_DEBT_DAILY_ARCHIVE','CT_ET_TEMPLATES.CSDB_DEBT_DAILY','ARCHIVE/CSDB/DEBT_DAILY');
-- 3. Export table OU_CSDB.DEBT_DAILY
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_CSDB','DEBT_DAILY','A_WORKFLOW_HISTORY_KEY','mrds_inbox_tst','ARCHIVE/CSDB/DEBT_DAILY');
-- 4. Rename table to legacy table
ALTER TABLE OU_CSDB.DEBT_DAILY RENAME TO DEBT_DAILY_LEGACY;
-- 5. Create OU view on top of ODS table
GRANT SELECT ON ods.CSDB_DEBT_DAILY_ODS TO OU_CSDB;
create or replace view OU_CSDB.RATINGS_DEBT_DAILY AS SELECT * from ods.CSDB_DEBT_DAILY_ODS;
show errors
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','NEWUPDATED','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','IDLOADDATE_DIM','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','IDISSUEDATE','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','COUPONDATE','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','IDMATURITYDATE','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','PUBLICATIONPRICEDATE','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','ACCRUALSTARTDATE','DD/MM/YYYY');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('CT_ET_TEMPLATES.DEBT_DAILY','SEC_STATUS_DATE','DD/MM/YYYY');

41
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_CSDB_RATINGS_RT_INSTRUMENT_RATING.sql Normal file
View File

@@ -0,0 +1,41 @@
--
-- Properly set up CSDB DEVO Instrument Ratings
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.CSDB_RT_INSTRUMENT_RATING
CREATE TABLE CT_ET_TEMPLATES.CSDB_RT_INSTRUMENT_RATING
AS SELECT * FROM OU_CSDB.RATINGS_RT_INSTRUMENT_RATING WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_RT_INSTRUMENT_RATING_INBOX','CT_ET_TEMPLATES.CSDB_RT_INSTRUMENT_RATING','INBOX/CSDB/RT_INSTRUMENT_RATING/RT_INSTRUMENT_RATING');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_RT_INSTRUMENT_RATING_ODS','CT_ET_TEMPLATES.CSDB_RT_INSTRUMENT_RATING','ODS/CSDB/RT_INSTRUMENT_RATING');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('CSDB_RT_INSTRUMENT_RATING_ARCHIVE','CT_ET_TEMPLATES.CSDB_RT_INSTRUMENT_RATING','ARCHIVE/CSDB/RT_INSTRUMENT_RATING');
-- 3. Export table OU_CSDB.RT_INSTRUMENT_RATING
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_CSDB','RT_INSTRUMENT_RATING','A_ETL_LOAD_SET_FK','mrds_inbox_tst','ARCHIVE/CSDB/RT_INSTRUMENT_RATING');
-- 4. Rename table to legacy table
ALTER TABLE OU_CSDB.RATINGS_RT_INSTRUMENT_RATING RENAME TO RATINGS_RT_INSTR_RATING_LEGACY;
-- 5. Create OU view on top of ODS table
GRANT SELECT ON ods.CSDB_RT_INSTRUMENT_RATING_ODS TO OU_CSDB;
create or replace view OU_CSDB.RATINGS_RT_INSTRUMENT_RATING AS SELECT * from ods.CSDB_RT_INSTRUMENT_RATING_ODS;
show errors
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','CORRECTFROM','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','VALIDUNTIL','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','TEC_EXECUTION_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','TEC_INGESTION_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','VALIDFROM','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','BATCH_LAST_UPDATED_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','CORRECTUNTIL','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','OUTLOOK_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','RATING_ACTION_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','RATING_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','TEC_BUSINESS_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','BATCH_FIRST_CREATED_DATE','YYYY-MM-DD HH24:MI:SS');
EXEC CT_MRDS.FILE_MANAGER.ADD_COLUMN_DATE_FORMAT('RT_INSTRUMENT_RATING','RT_INSTRUMENT_RATING','WATCHLIST_DATE','YYYY-MM-DD HH24:MI:SS');

29
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_LM_STANDING_FACILITIES.sql Normal file
View File

@@ -0,0 +1,29 @@
--
-- Properly set up LM Standing Facilities
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.LM_STANDING_FACILITIES
CREATE TABLE CT_ET_TEMPLATES.LM_STANDING_FACILITIES
AS SELECT * FROM OU_LM.STANDING_FACILITIES WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_INBOX','CT_ET_TEMPLATES.LM_STANDING_FACILITIES','INBOX/LM/STANDING_FACILITIES/STANDING_FACILITIES');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_ODS','CT_ET_TEMPLATES.LM_STANDING_FACILITIES','ODS/LM/STANDING_FACILITIES');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_ARCHIVE','CT_ET_TEMPLATES.LM_STANDING_FACILITIES','ARCHIVE/LM/STANDING_FACILITIES');
-- 3. Export table OU_LM.STANDING_FACILITIES to ARCHIVE
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_LM','STANDING_FACILITIES','A_WORKFLOW_HISTORY_KEY','mrds_inbox_tst','ARCHIVE/LM/STANDING_FACILITIES');
-- 4. Rename OU_LM.STANDING_FACILITIES to OU_LM.STANDING_FACILITIES_LEGACY
ALTER TABLE OU_LM.STANDING_FACILITIES RENAME TO STANDING_FACILITIES_LEGACY;
-- 5. Create view OU_LM.STANDING_FACILITIES on top of ODS.LM_STANDING_FACILITIES_INBOX
REVOKE SELECT ON ods.lm_standing_facilities_inbox from OU_LM;
GRANT SELECT ON ods.lm_standing_facilities_ods TO OU_LM;
create or replace view OU_LM.STANDING_FACILITIES as select * from ods.lm_standing_facilities_ods;

28
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_LM_STANDING_FACILITIES_HEADER.sql Normal file
View File

@@ -0,0 +1,28 @@
--
-- Properly set up LM Standing Facilities Header
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.LM_STANDING_FACILITIES
CREATE TABLE CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER
AS SELECT * FROM OU_LM.STANDING_FACILITIES_HEADER WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_HEADER_INBOX','CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER','INBOX/LM/STANDING_FACILITIES/STANDING_FACILITIES_HEADER');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_HEADER_ODS','CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER','ODS/LM/STANDING_FACILITIES_HEADER');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('LM_STANDING_FACILITIES_HEADER_ARCHIVE','CT_ET_TEMPLATES.LM_STANDING_FACILITIES_HEADER','ARCHIVE/LM/STANDING_FACILITIES_HEADER');
-- 3. Export table OU_LM.STANDING_FACILITIES to ARCHIVE
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_LM','STANDING_FACILITIES_HEADER','A_WORKFLOW_HISTORY_KEY','mrds_inbox_tst','ARCHIVE/LM/STANDING_FACILITIES_HEADER');
-- 4. Rename OU_LM.STANDING_FACILITIES to OU_LM.STANDING_FACILITIES_LEGACY
ALTER TABLE OU_LM.STANDING_FACILITIES_HEADER RENAME TO STANDING_FACILITIES_HEADER_LEGACY;
-- 5. Create view OU_LM.STANDING_FACILITIES on top of ODS.LM_STANDING_FACILITIES_INBOX
GRANT SELECT ON ods.lm_standing_facilities_header_ods TO OU_LM;
create or replace view OU_LM.STANDING_FACILITIES_HEADER AS SELECT * from ods.LM_STANDING_FACILITIES_HEADER_ods;

29
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_MDP_BBG_EQUITY_OT.sql Normal file
View File

@@ -0,0 +1,29 @@
--
-- Properly set up MDP Standing Facilities
--
-- 1. Create Template table CT_TEMPLTATE_TABLES.MDP_BBG_EQUITY_OT
CREATE TABLE CT_ET_TEMPLATES.MDP_BBG_EQUITY_OT
AS SELECT * FROM OU_MDP.BBG_EQUITY_OT WHERE 1=2;
-- 2. Create external tables
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('MDP_BBG_EQUITY_OT_INBOX','CT_ET_TEMPLATES.MDP_BBG_EQUITY_OT','INBOX/MDP/BBG_EQUITY_OT/BBG_EQUITY_OT');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('MDP_BBG_EQUITY_OT_ODS','CT_ET_TEMPLATES.MDP_BBG_EQUITY_OT','ODS/MDP/BBG_EQUITY_OT');
EXEC CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE('MDP_BBG_EQUITY_OT_ARCHIVE','CT_ET_TEMPLATES.MDP_BBG_EQUITY_OT','ARCHIVE/MDP/BBG_EQUITY_OT');
-- 3. Export table OU_MDP.BBG_EQUITY_OT to ARCHIVE
EXEC CT_MRDS.FILE_MANAGER.EXPORT_TABLE_DATA_BY_DATE('OU_MDP','BBG_EQUITY_OT','A_WORKFLOW_HISTORY_KEY','mrds_inbox_tst','ARCHIVE/MDP/BBG_EQUITY_OT');
-- 4. Rename OU_MDP.BBG_EQUITY_OT to OU_MDP.BBG_EQUITY_OT_LEGACY
ALTER TABLE OU_MDP.BBG_EQUITY_OT RENAME TO BBG_EQUITY_OT_LEGACY;
-- 5. Create view OU_MDP.BBG_EQUITY_OT on top of ODS.MDP_BBG_EQUITY_OT_INBOX
REVOKE SELECT ON ods.lm_standing_facilities_inbox from OU_MDP;
GRANT SELECT ON ods.lm_standing_facilities_ods TO OU_MDP;
create or replace view OU_MDP.BBG_EQUITY_OT as select * from ods.lm_standing_facilities_ods;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_ACMENTRYSTATELEDGERGROUP.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.ACMENTRYSTATELEDGERGROUP RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_ACMENTRYSTATELEDGERGROUP;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_ACTIVITYLOGDUE.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.ACTIVITYLOGDUE RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_ACTIVITYLOGDUE;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_ACTIVITY_LOG.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.ACTIVITY_LOG RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_ACTIVITY_LOG;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_BALANCE.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.BALANCE RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_BALANCE;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_BLACKOUT_LOG.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.BLACKOUT_LOG RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_BLACKOUT_LOG;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_BRANCH.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.BRANCH RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_BRANCH;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_CALENDAR.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.CALENDAR RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_CALENDAR;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_CASHFLOW.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.CASHFLOW RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_CASHFLOW;

2
MARS_Packages/mrds_elt-dev-database/mrds_elt-dev-database/database/CT_MRDS/SETUP/OU_TMS_CLIENT.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE OU_TMS.CLIENT RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY;
DROP TABLE TMS.T_CLIENT;

Some files were not shown because too many files have changed in this diff Show More