admin/mars
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
aa03dd1616861f06179bc69caf6ac90978989534
mars/MARS_Packages/REL02_POST/MARS-1409/rollback_version/FILE_MANAGER.pkb
Grzegorz Michalski 1c6f552df9 szkielet paczki MARS-1409
2026-02-27 07:32:21 +01:00

2010 lines
106 KiB
Plaintext
Raw Blame History

create or replace PACKAGE BODY CT_MRDS.FILE_MANAGER
AS
----------------------------------------------------------------------------------------------------
-- PRIVATE FUNCTION: NORMALIZE_DATE_FORMAT
----------------------------------------------------------------------------------------------------
/**
* Purpose: Normalize Oracle date format strings for use in external tables
*
* Problem: ISO 8601 formats like 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' fail because
* literal character 'T' must be enclosed in double quotes for Oracle
* external table DATE column definitions.
*
* Solution: Detect unquoted 'T' separator and wrap it in double quotes
*
* Parameters:
* pDateFormat - Original date format from A_COLUMN_DATE_FORMAT table
*
* Returns: Normalized format with quoted 'T' if applicable
*
* Examples:
* Input: 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM'
* Output: 'YYYY-MM-DD"T"HH24:MI:SS.FF3TZH:TZM'
*
* Input: 'DD/MM/YYYY HH24:MI:SS' (no T)
* Output: 'DD/MM/YYYY HH24:MI:SS' (unchanged)
*
* Input: 'YYYY-MM-DD"T"HH24:MI:SS' (already quoted)
* Output: 'YYYY-MM-DD"T"HH24:MI:SS' (unchanged)
*
* Author: Grzegorz Michalski
* Date: 2025-11-27
* Version: 1.0.0 (MARS-1046)
*/
FUNCTION NORMALIZE_DATE_FORMAT(pDateFormat VARCHAR2) RETURN VARCHAR2 IS
vNormalizedFormat VARCHAR2(500);
BEGIN
-- Return NULL if input is NULL
IF pDateFormat IS NULL THEN
RETURN NULL;
END IF;
vNormalizedFormat := pDateFormat;
-- Check if 'T' separator exists and is NOT already quoted
-- Pattern: [YMD]T[HM] (date component + T + time component)
IF INSTR(vNormalizedFormat, '"T"') = 0 AND
REGEXP_LIKE(vNormalizedFormat, '[YMD]T[HM]') THEN
-- Wrap 'T' in double quotes using regex replace
-- Pattern matches: (date format char) + T + (time format char)
-- Replacement: \1 + "T" + \2
vNormalizedFormat := REGEXP_REPLACE(vNormalizedFormat, '([YMD])T([HM])', '\1"T"\2');
END IF;
RETURN vNormalizedFormat;
EXCEPTION
WHEN OTHERS THEN
-- If normalization fails, return original format (safety fallback)
RETURN pDateFormat;
END NORMALIZE_DATE_FORMAT;
----------------------------------------------------------------------------------------------------
FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL
, pSourceFileReceivedKey IN NUMBER DEFAULT NULL
, pSourceFileConfigKey IN NUMBER DEFAULT NULL)
RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pFileUri => '''||nvl(pFileUri,'NULL')||''''
,'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL')
,'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
BEGIN
IF pFileUri IS NOT NULL THEN
SELECT *
INTO vSourceFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE REGEXP_LIKE(pFileUri, A_SOURCE_KEY||'/'||SOURCE_FILE_ID||'/'||TABLE_ID||'/'||SOURCE_FILE_NAME_PATTERN);
ELSIF pSourceFileReceivedKey IS NOT NULL THEN
SELECT T.*
INTO vSourceFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG T, CT_MRDS.A_SOURCE_FILE_RECEIVED R
WHERE T.A_SOURCE_FILE_CONFIG_KEY = R.A_SOURCE_FILE_CONFIG_KEY
AND R.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey;
ELSIF pSourceFileConfigKey IS NOT NULL THEN
SELECT *
INTO vSourceFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY);
END IF;
-- Set global package variable vgSourceFileConfigKey - used in error messages
vgSourceFileConfigKey := vSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY;
EXCEPTION
WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY);
WHEN NO_DATA_FOUND THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI);
WHEN TOO_MANY_ROWS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE);
WHEN OTHERS THEN
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vSourceFileConfig;
END GET_SOURCE_FILE_CONFIG;
----------------------------------------------------------------------------------------------------
FUNCTION GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey IN NUMBER DEFAULT NULL)
--
-- Get source file received info
--
RETURN tSourceFileReceived
IS
vSourceFileReceived tSourceFileReceived;
vBucket VARCHAR2(400);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
BEGIN
SELECT R.A_SOURCE_FILE_RECEIVED_KEY, R.A_SOURCE_FILE_CONFIG_KEY,
'INBOX'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_INBOX,
'ODS'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_ODS,
'QUARANTINE'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_QUARANTINE,
'ARCHIVE'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/' as SOURCE_FILE_PREFIX_ARCHIVE,
R.SOURCE_FILE_NAME,
R.RECEPTION_DATE, R.PROCESSING_STATUS, R.EXTERNAL_TABLE_NAME
INTO vSourceFileReceived
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED R, CT_MRDS.A_SOURCE_FILE_CONFIG T
WHERE R.A_SOURCE_FILE_CONFIG_KEY = T.A_SOURCE_FILE_CONFIG_KEY
AND A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey;
EXCEPTION
WHEN NO_DATA_FOUND THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE);
WHEN TOO_MANY_ROWS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vSourceFileReceived;
END GET_SOURCE_FILE_RECEIVED_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName IN VARCHAR2)
RETURN PLS_INTEGER
--
-- Register a newly received source file A_SOURCE_FILE_RECEIVED
-- This overload automatically determines source file type from the file name
--
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vSourceFileReceivedKey PLS_INTEGER;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO',vParameters);
vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedName);
vSourceFileReceivedKey := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName, vSourceFileConfig);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN vSourceFileReceivedKey;
EXCEPTION
WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY);
WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI);
WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD);
WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END REGISTER_SOURCE_FILE_RECEIVED;
----------------------------------------------------------------------------------------------------
FUNCTION REGISTER_SOURCE_FILE_RECEIVED(
pSourceFileReceivedName IN VARCHAR2
,pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE)
RETURN PLS_INTEGER
--
-- Register a newly received source file A_SOURCE_FILE_RECEIVED
--
IS
vExternalTableName VARCHAR2(200);
vDirName VARCHAR2(1000);
vFileName VARCHAR2(1000);
vChecksum A_SOURCE_FILE_RECEIVED.CHECKSUM%TYPE;
vCreated A_SOURCE_FILE_RECEIVED.CREATED%TYPE;
vBytes A_SOURCE_FILE_RECEIVED.BYTES%TYPE;
vSourceFileReceivedKey PLS_INTEGER;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vRow CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||''''
,'pSourceFileConfig => '||'tSourceFileConfig record type'));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
vDirName := REGEXP_SUBSTR(pSourceFileReceivedName, '(.*/)(.*)', 1, 1, NULL, 1);
-- Remove prefix from file name
vFileName := REGEXP_SUBSTR(pSourceFileReceivedName,'[^/]*$');
ENV_MANAGER.LOG_PROCESS_EVENT('gvCredentialName','DEBUG',ENV_MANAGER.gvCredentialName);
ENV_MANAGER.LOG_PROCESS_EVENT('gvInboxBucketUri','DEBUG',ENV_MANAGER.gvInboxBucketUri);
ENV_MANAGER.LOG_PROCESS_EVENT('vDirName','DEBUG',vDirName);
SELECT
checksum, created, bytes
INTO
vChecksum, vCreated, vBytes
FROM DBMS_CLOUD.LIST_OBJECTS(ENV_MANAGER.gvCredentialName,
ENV_MANAGER.gvInboxBucketUri || vDirName
)
WHERE object_name = vFileName
;
vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL;
vExternalTableName := REPLACE(
REGEXP_SUBSTR(pSourceFileConfig.TEMPLATE_TABLE_NAME||'_'||vSourceFileReceivedKey,
'\..*'),
'.','');
INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED
(A_SOURCE_FILE_RECEIVED_KEY, A_SOURCE_FILE_CONFIG_KEY,
SOURCE_FILE_NAME, RECEPTION_DATE,
PROCESSING_STATUS, EXTERNAL_TABLE_NAME,
CHECKSUM, CREATED, BYTES)
VALUES (vSourceFileReceivedKey, pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY,
vFileName, SYSDATE,
'RECEIVED', vExternalTableName,
vChecksum, vCreated, vBytes);
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vSourceFileReceivedKey;
EXCEPTION
WHEN NO_DATA_FOUND THEN
vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD
||cgBL||' '||'File: '||ENV_MANAGER.gvInboxBucketUri || vDirName || vFileName;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, vgMsgTmp);
WHEN DUP_VAL_ON_INDEX THEN
select * into vRow
from CT_MRDS.A_SOURCE_FILE_RECEIVED
where CHECKSUM = vChecksum
and CREATED = vCreated
and BYTES = vBytes
;
vgMsgTmp := ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED
||cgBL||' '||'Details about existing File: '
||cgBL||' '||'-------------------------'
||cgBL||' '||'A_SOURCE_FILE_RECEIVED_KEY = '||vRow.A_SOURCE_FILE_RECEIVED_KEY
||cgBL||' '||'A_SOURCE_FILE_CONFIG_KEY = '||vRow.A_SOURCE_FILE_CONFIG_KEY
||cgBL||' '||'SOURCE_FILE_NAME = '||vRow.SOURCE_FILE_NAME
||cgBL||' '||'CHECKSUM = '||vRow.CHECKSUM
||cgBL||' '||'CREATED = '||vRow.CREATED
||cgBL||' '||'BYTES = '||vRow.BYTES
||cgBL||' '||'RECEPTION_DATE = '||vRow.RECEPTION_DATE
||cgBL||' '||'PROCESSING_STATUS = '||vRow.PROCESSING_STATUS
||cgBL||' '||'EXTERNAL_TABLE_NAME = '||vRow.EXTERNAL_TABLE_NAME
||cgBL||' '||'-------------------------'
||cgBL||' '||'There cannot be two files with the same values for (CHECKSUM, CREATED, BYTES)'
;
-- vChecksum, vCreated, vBytes
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, vgMsgTmp);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END REGISTER_SOURCE_FILE_RECEIVED;
----------------------------------------------------------------------------------------------------
PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey IN PLS_INTEGER, pStatus IN VARCHAR2)
--
-- Change status of file in the A_SOURCE_FILE_RECEIVED table
--
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL')
,'pStatus => '''||nvl(pStatus, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
SET PROCESSING_STATUS=pStatus
WHERE A_SOURCE_FILE_RECEIVED_KEY=pSourceFileReceivedKey;
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to '||pStatus,'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END SET_SOURCE_FILE_RECEIVED_STATUS;
----------------------------------------------------------------------------------------------------
FUNCTION GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate IN VARCHAR2)
RETURN CLOB
--
-- Create list of columns for DBMS_CLOUD.CREATE_EXTERNAL_TABLE from existing template table
--
IS
vColumnList CLOB;
vTableName VARCHAR2(200);
vSchemaName VARCHAR2(200);
BEGIN
vSchemaName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'.*\.'),'.','');
vTableName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'\..*'),'.','');
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SQLTERMINATOR', True);
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'COLLATION_CLAUSE', 'NEVER');
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'REF_CONSTRAINTS', False);
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'STORAGE', False);
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'TABLESPACE', False);
DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SEGMENT_ATTRIBUTES', False);
vColumnList := RTRIM(
LTRIM(
REGEXP_SUBSTR(DBMS_METADATA.GET_DDL('TABLE', vTableName, vSchemaName),'\(.*\)',1,1,'mn'),
'('),
')');
RETURN vColumnList;
END GET_EXTERNAL_TABLE_COLUMNS;
----------------------------------------------------------------------------------------------------
PROCEDURE CREATE_EXTERNAL_TABLE (
pTableName IN VARCHAR2,
pTemplateTableName IN VARCHAR2,
pPrefix IN VARCHAR2,
pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri,
pFileName IN VARCHAR2 DEFAULT NULL,
pDelimiter IN VARCHAR2 DEFAULT ',',
pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NEW PARAMETER FOR FILE ENCODING
)
--
-- Create external table for a single source file to validate the file structure
--
IS
vTableName VARCHAR2(200);
vColumnList CLOB;
vFieldList CLOB;
vFormat VARCHAR2(200);
vPrefix VARCHAR2(200);
vFileName VARCHAR2(1000);
vFileExtension VARCHAR2(200);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTableName => '''||nvl(pTableName, 'NULL')||''''
,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pPrefix => '''||nvl(pPrefix, 'NULL')||''''
,'pBucketUri => '''||nvl(pBucketUri, 'NULL')||''''
,'pFileName => '''||nvl(pFileName, 'NULL')||''''
,'pDelimiter => '''||nvl(pDelimiter, 'NULL')||''''
,'pEncoding => '''||nvl(pEncoding, 'NULL')||'''' -- MARS-1049: NOWY
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
-- Strip off leading and trailing slashes from prefix
vPrefix := TRIM(BOTH '/' FROM pPrefix);
-- Generate column and field list from template table
GENERATE_EXTERNAL_TABLE_PARAMS (pTemplateTableName, vColumnList, vFieldList);
--vFormat evaluation based on pBucketUri first, then pPrefix
-- Archive bucket should use parquet + implicit partitioning regardless of prefix
IF INSTR(pBucketUri, ENV_MANAGER.gvArchiveBucketName)>0 THEN
vFormat := '{"type": "parquet"
,"implicit_partition_type": "hive"
,"implicit_partition_columns":["PARTITION_YEAR","PARTITION_MONTH"]}';
vColumnList := vColumnList||cgBL||' , "PARTITION_YEAR" varchar2(4)'||cgBL||', "PARTITION_MONTH" varchar2(2)';
vFieldList := NULL;
vFileExtension := '.parquet';
-- For INBOX, ODS, and other ARCHIVE prefixes (not in archive bucket) use CSV
ELSIF SUBSTR(pPrefix,1,5) = 'INBOX' OR SUBSTR(pPrefix,1,3) = 'ODS'
OR SUBSTR(pPrefix,1,7) = 'ARCHIVE'
THEN
-- MARS-1049: Create format with encoding if specified
IF pDelimiter = '|' THEN
IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN
vFormat := json_object(
'delimiter' VALUE '|',
'skipheaders' VALUE '1',
'characterset' VALUE pEncoding
);
ELSE
vFormat := json_object('delimiter' VALUE '|', 'skipheaders' VALUE '1');
END IF;
ELSE
IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN
vFormat := json_object(
'type' VALUE 'CSV',
'skipheaders' VALUE '1',
'ignoremissingcolumns' VALUE 'true',
'characterset' VALUE pEncoding
);
ELSE
vFormat := json_object('type' VALUE 'CSV', 'skipheaders' VALUE '1', 'ignoremissingcolumns' value 'true');
END IF;
END IF;
vFileExtension := '.csv';
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX);
END IF;
-- No filename give: Match all csv files
IF pFileName IS NOT NULL THEN
vFileName := pFileName;
ELSE
vFileName := pBucketUri||vPrefix||'/*'||vFileExtension;
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('pTableName', 'DEBUG', pTableName);
ENV_MANAGER.LOG_PROCESS_EVENT('ENV_MANAGER.vpCredentialName', 'DEBUG', ENV_MANAGER.gvCredentialName);
ENV_MANAGER.LOG_PROCESS_EVENT('vFileName', 'DEBUG', vFileName);
ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', vColumnList);
ENV_MANAGER.LOG_PROCESS_EVENT('vFieldList', 'DEBUG', vFieldList);
ENV_MANAGER.LOG_PROCESS_EVENT('vFormat', 'DEBUG', vFormat);
-- Pre-validation: Check CSV column count for CSV files only
IF SUBSTR(pPrefix,1,5) = 'INBOX' AND pFileName IS NOT NULL THEN
DECLARE
vCsvFirstLine VARCHAR2(4000);
vCsvColCount NUMBER := 0;
vTemplateColCount NUMBER := 0;
vExcessColumns VARCHAR2(2000);
-- Get template column count
CURSOR c_template_count IS
SELECT COUNT(*) as col_count
FROM ALL_TAB_COLUMNS
WHERE OWNER = UPPER(REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.',''))
AND TABLE_NAME = UPPER(REGEXP_REPLACE(pTemplateTableName,'^.*\.',''));
BEGIN
-- Get template column count
FOR rec IN c_template_count LOOP
vTemplateColCount := rec.col_count;
END LOOP;
-- Read first line of CSV to count columns
BEGIN
SELECT UTL_RAW.CAST_TO_VARCHAR2(
DBMS_LOB.SUBSTR(
DBMS_CLOUD.GET_OBJECT(
credential_name => ENV_MANAGER.gvCredentialName,
object_uri => pFileName
),
4000, 1
)
) INTO vCsvFirstLine FROM DUAL;
-- Count commas in header line + 1 for total columns
vCsvColCount := REGEXP_COUNT(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), ',') + 1;
ENV_MANAGER.LOG_PROCESS_EVENT('CSV Column Count: ' || vCsvColCount || ', Template Column Count: ' || vTemplateColCount, 'INFO', vParameters);
-- Check for excess columns
IF vCsvColCount > vTemplateColCount THEN
vgMsgTmp := ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED
||cgBL||'EXCESS COLUMNS DETECTED!'
||cgBL||'CSV file has ' || vCsvColCount || ' columns but template expects only ' || vTemplateColCount
||cgBL||'Excess columns: ' || (vCsvColCount - vTemplateColCount)
||cgBL||'CSV header: ' || SUBSTR(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), 1, 200)
||cgBL||'POSSIBLE SOLUTIONS:'
||cgBL||' 1. Remove excess columns from CSV file before processing'
||cgBL||' 2. Add excess columns to template table: ' || pTemplateTableName;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, vgMsgTmp);
END IF;
EXCEPTION
WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN
RAISE; -- Re-raise the excess columns error
WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN
RAISE; -- Re-raise the validation error
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not perform pre-validation column count check: ' || SQLERRM, 'WARN', vParameters);
-- Continue with normal processing if pre-validation fails
END;
END;
END IF;
DBMS_CLOUD.CREATE_EXTERNAL_TABLE(
TABLE_NAME => pTableName,
CREDENTIAL_NAME => ENV_MANAGER.gvCredentialName,
FILE_URI_LIST => vFileName,
COLUMN_LIST => vColumnList,
FIELD_LIST => vFieldList,
FORMAT => vFormat
);
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN
RAISE; -- Re-raise the excess columns error with specific code -20011
WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN
vgMsgTmp := ENV_MANAGER.MSG_UNKNOWN_PREFIX || ': ' || pPrefix;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, vgMsgTmp);
WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END CREATE_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
PROCEDURE CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER)
--
-- Create external table for a single source file to validate the file structure
--
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vSourceFileReceived tSourceFileReceived;
vTableName VARCHAR2(200);
vFileName VARCHAR2(1000);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey);
vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey);
vTableName := vSourceFileConfig.TEMPLATE_TABLE_NAME;
vFileName := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX||vSourceFileReceived.SOURCE_FILE_NAME;
CREATE_EXTERNAL_TABLE(
pTableName => vSourceFileReceived.EXTERNAL_TABLE_NAME,
pTemplateTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME,
pPrefix => vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX,
pBucketUri => ENV_MANAGER.gvInboxBucketUri,
pFileName => vFileName,
pDelimiter => ',',
pEncoding => vSourceFileConfig.ENCODING -- MARS-1049: NOWY PARAMETR
);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
END CREATE_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey IN NUMBER)
--
-- Check the structure of the received file using DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
--
IS
vSourceFileReceived tSourceFileReceived;
vOperationId NUMBER := -1;
vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE;
vStatus USER_LOAD_OPERATIONS.STATUS%TYPE;
vErrors NUMBER := 0;
vNumRows NUMBER := 0;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey);
ENV_MANAGER.LOG_PROCESS_EVENT('vSourceFileReceived.EXTERNAL_TABLE_NAME: '||vSourceFileReceived.EXTERNAL_TABLE_NAME,'DEBUG', vParameters);
BEGIN
DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(vSourceFileReceived.EXTERNAL_TABLE_NAME, vOperationId);
EXCEPTION
WHEN OTHERS THEN
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_VALIDATION_FAILED, vParameters, 'FILE_MANAGER');
-- Call detailed validation error analysis and log the results
DECLARE
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vValidationLogTable VARCHAR2(200);
vTemplateSchema VARCHAR2(200);
vTemplateTable VARCHAR2(200);
vCsvFileUri VARCHAR2(2000);
vAnalysisResult VARCHAR2(32000);
vFailedOperationId NUMBER;
BEGIN
-- Get source file configuration
vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey);
-- Extract template schema and table from template table name
vTemplateSchema := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'.*\.'),'.','');
vTemplateTable := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'\..*'),'.','');
-- Construct CSV file URI
vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX || vSourceFileReceived.SOURCE_FILE_NAME;
-- Find the failed validation operation ID
SELECT MAX(ID) INTO vFailedOperationId
FROM USER_LOAD_OPERATIONS
WHERE TABLE_NAME = vSourceFileReceived.EXTERNAL_TABLE_NAME
AND TYPE = 'VALIDATE'
AND STATUS != 'COMPLETED';
-- Get validation log table name
IF vFailedOperationId IS NOT NULL THEN
SELECT LOGFILE_TABLE INTO vValidationLogTable
FROM USER_LOAD_OPERATIONS
WHERE ID = vFailedOperationId;
-- Call detailed error analysis
vAnalysisResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS(
pValidationLogTable => vValidationLogTable,
pTemplateSchema => vTemplateSchema,
pTemplateTable => vTemplateTable,
pCsvFileUri => vCsvFileUri
);
-- Log detailed analysis results
ENV_MANAGER.LOG_PROCESS_EVENT('DETAILED VALIDATION ERROR ANALYSIS:', 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(vAnalysisResult, 'ERROR', vParameters);
END IF;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error during validation analysis: ' || SQLERRM, 'ERROR', vParameters);
END;
MOVE_FILE(pSourceFileReceivedKey => pSourceFileReceivedKey, pDestination => 'QUARANTINE');
-- Ensure the status change is committed before raising exception
COMMIT;
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED);
END;
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters);
IF vOperationId = -1
THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID);
END IF;
SELECT BADFILE_TABLE, ROWS_LOADED, STATUS
INTO vBadfileTable, vNumRows, vStatus
FROM USER_LOAD_OPERATIONS
WHERE ID = vOperationId;
-- DBMS_OUTPUT.PUT_LINE(vStatus);
SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATED');
ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to VALIDATED','DEBUG', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters);
SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID);
WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN
vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED;
ENV_MANAGER.LOG_PROCESS_ERROR(vgMsgTmp, vParameters, 'FILE_MANAGER');
SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp);
WHEN OTHERS THEN
IF SQLCODE = -20404 THEN
vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD||cgBL||SQLERRM;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, vgMsgTmp);
ELSIF SQLCODE = -20003 THEN
execute immediate 'select LISTAGG(record, '''||cgBL||''') from (select * from '||REGEXP_SUBSTR(SQLERRM, '"([^"]+)"."([^"]+)"')||' order by rownum desc) where rownum <=2'
into vgMsgTmp;
vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||vgMsgTmp;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp);
-- ELSIF SQLCODE = -20000 THEN
-- TO_DO Add additional info about current config
-- ENV_MANAGER.MSG_FILE_VALIDATION_FAILED := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||FILE_MANAGER.OUTPUT_SOURCE_FILE_CONFIG_INFO( ..config key value.. );
-- dbms_output.put_line(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT, 'ERROR', vParameters);
-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
-- RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT);
ELSE
-- Log complete error details including full stack trace and backtrace
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END IF;
END VALIDATE_SOURCE_FILE_RECEIVED;
----------------------------------------------------------------------------------------------------
FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
RETURN VARCHAR2
--
-- wrapper for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE
--
IS
vOperationId NUMBER := -1;
vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE;
vLogfileTable USER_LOAD_OPERATIONS.LOGFILE_TABLE%TYPE;
vStatus USER_LOAD_OPERATIONS.STATUS%TYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vDetails clob;
TYPE TCURSOR is REF CURSOR;
vCursor TCURSOR;
vQuery VARCHAR2(1000);
vRecord VARCHAR2(10000);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTableName => '''||nvl(pTableName, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(pTableName, vOperationId);
ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters);
IF vOperationId = -1
THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID);
END IF;
SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE
INTO vStatus, vLogfileTable, vBadfileTable
FROM USER_LOAD_OPERATIONS
WHERE ID = vOperationId;
RETURN vStatus;
EXCEPTION
WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID);
WHEN OTHERS THEN
SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE
INTO vStatus, vLogfileTable, vBadfileTable
FROM USER_LOAD_OPERATIONS
WHERE ID = vOperationId;
vQuery := 'select record from (
select
nvl(l.record,''----------------------------------------------------'') as record
,rownum as lp
,max(case when nvl(instr(l.record, ''error'' ),0) > 0 then rownum else 0 end) over (partition by 1) as ENV_MANAGER.ERR_row
from '||vLogfileTable||' l
)
where lp >= ENV_MANAGER.ERR_row
order by rownum';
vDetails := vStatus||cgBL||'----------------------------------------------------'||cgBL;
OPEN vCursor for vQuery;
loop
fetch vCursor into vRecord;
EXIT WHEN vCursor%NOTFOUND;
vDetails := vDetails ||vRecord ||cgBL;
-- for i in loop
-- vDetails := vDetails ||i.record ||cgBL;
end loop;
CLOSE vCursor;
vDetails := vDetails||'More details can be found in below tables:'||cgBL||
' SELECT * FROM USER_LOAD_OPERATIONS WHERE ID = '||vOperationId||';'||cgBL||
' SELECT * FROM '||vLogfileTable||';'||cgBL||
' SELECT * FROM '||vBadfileTable||';'
;
RETURN vDetails;
END VALIDATE_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2)
RETURN VARCHAR2
--
-- Simple
--
IS
vCount PLS_INTEGER;
BEGIN
execute immediate 'select count(1) from '||pTableName into vCount;
IF vCount >= 0
THEN
RETURN 'PASSED';
END IF;
RETURN 'FAILED';
EXCEPTION
WHEN OTHERS THEN
RETURN 'FAILED';
END S_VALIDATE_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
PROCEDURE DROP_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER)
--
-- Drop external table created to validate the file structure
--
IS
vSourceFileReceived tSourceFileReceived;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey);
EXECUTE IMMEDIATE 'DROP TABLE '||vSourceFileReceived.EXTERNAL_TABLE_NAME;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END DROP_EXTERNAL_TABLE;
----------------------------------------------------------------------------------------------------
PROCEDURE COPY_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2)
--
-- Possible pDestination values are: 'ODS' or 'ARCHIVE'
--
IS
vSourceFileReceivedInfo tSourceFileReceived;
vSourceObject VARCHAR2(2000);
vTargetObject VARCHAR2(2000);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
-- vStatus VARCHAR2(20);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pDestination => '''||nvl(pDestination, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey);
IF pDestination = 'ODS' THEN
vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM);
END IF;
DBMS_CLOUD.COPY_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => vSourceObject,
target_object_uri => vTargetObject,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File copied to '||pDestination||' target location','DEBUG', vTargetObject);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END COPY_FILE;
----------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------
PROCEDURE MOVE_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2)
--
-- Possible pDestination values are: 'ODS' or 'ARCHIVE'
--
IS
vSourceFileReceivedInfo tSourceFileReceived;
vSourceObject VARCHAR2(2000);
vTargetObject VARCHAR2(2000);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vStatus VARCHAR2(20);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
'pDestination => '''||nvl(pDestination, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey);
IF pDestination = 'ODS' THEN
vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
vStatus := 'READY_FOR_INGESTION';
ELSIF pDestination = 'QUARANTINE' THEN
vSourceObject := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
vTargetObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_QUARANTINE||vSourceFileReceivedInfo.SOURCE_FILE_NAME;
vStatus := 'VALIDATION_FAILED';
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM);
END IF;
DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName,
source_object_uri => vSourceObject,
target_object_uri => vTargetObject,
target_credential_name => ENV_MANAGER.gvCredentialName
);
ENV_MANAGER.LOG_PROCESS_EVENT('File moved to '||pDestination||' target location','DEBUG', vTargetObject);
SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => vStatus);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN
ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM := ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM
||cgBL||' '||'Possible parameters are: ''ODS'' or ''ARCHIVE'''
||cgBL||' '||'Provided destination parameter: '''||pDestination||'''';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM);
WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END MOVE_FILE;
----------------------------------------------------------------------------------------------------
PROCEDURE DELETE_FOLDER_CONTENTS(pBucketArea IN VARCHAR2, pFolderPrefix IN VARCHAR2)
--
-- Delete all files from specified folder in cloud storage
-- pBucketArea: 'INBOX', 'DATA', 'ARCHIVE'
-- pFolderPrefix: folder path within bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/')
--
IS
vBucketUri VARCHAR2(2000);
vFolderUri VARCHAR2(2000);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vFilesDeleted PLS_INTEGER := 0;
vObjectName VARCHAR2(4000);
vFullObjectUri VARCHAR2(4000);
-- Cursor to list all objects in the folder
CURSOR c_objects IS
SELECT object_name
FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
credential_name => ENV_MANAGER.gvCredentialName,
location_uri => vBucketUri
))
WHERE object_name IS NOT NULL
AND object_name LIKE pFolderPrefix || '%';
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pBucketArea => '''||nvl(pBucketArea, 'NULL')||'''',
'pFolderPrefix => '''||nvl(pFolderPrefix, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- Get bucket URI based on bucket area
vBucketUri := GET_BUCKET_URI(pBucketArea);
ENV_MANAGER.LOG_PROCESS_EVENT('Listing objects in bucket with prefix: ' || pFolderPrefix, 'DEBUG', vBucketUri);
-- List and delete all objects in the folder
FOR obj_rec IN c_objects LOOP
vObjectName := obj_rec.object_name;
vFullObjectUri := vBucketUri || vObjectName;
BEGIN
ENV_MANAGER.LOG_PROCESS_EVENT('Deleting object', 'DEBUG', vFullObjectUri);
DBMS_CLOUD.DELETE_OBJECT(
credential_name => ENV_MANAGER.gvCredentialName,
object_uri => vFullObjectUri
);
vFilesDeleted := vFilesDeleted + 1;
ENV_MANAGER.LOG_PROCESS_EVENT('Object deleted successfully', 'DEBUG', vObjectName);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error deleting object: ' || vObjectName || ' - ' || SQLERRM, 'ERROR', vParameters);
-- Continue with next file instead of stopping the whole process
END;
END LOOP;
ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted: ' || vFilesDeleted, 'INFO', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT('Error in DELETE_FOLDER_CONTENTS: ' || SQLERRM, 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END DELETE_FOLDER_CONTENTS;
----------------------------------------------------------------------------------------------------
PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
--
-- Ubmrella procedure that calls
-- - REGISTER_SOURCE_FILE_RECEIVED
-- - CREATE_EXTERNAL_TABLE
-- - VALIDATE_SOURCE_FILE_RECEIVED
-- - DROP_EXTERNAL_TABLE
-- - MOVE_FILE
IS
vSourceFileId NUMBER;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
vSourceFileId := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName);
CREATE_EXTERNAL_TABLE(vSourceFileId);
VALIDATE_SOURCE_FILE_RECEIVED(vSourceFileId);
DROP_EXTERNAL_TABLE(vSourceFileId);
-- COPY_FILE(vSourceFileId, 'ODS');
MOVE_FILE(vSourceFileId, 'ODS');
SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => vSourceFileId, pStatus => 'READY_FOR_INGESTION');
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
-- -20001
WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY);
-- -20002
WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI);
-- -20003
WHEN ENV_MANAGER.ERR_MULTIPLE_MATCH_FOR_SRCFILE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE);
-- -20004
WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT);
-- -20005
WHEN ENV_MANAGER.ERR_MULTIPLE_COLUMN_DATE_FORMAT THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT);
-- -20006
WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID);
-- -20007
WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE);
-- -20008
WHEN ENV_MANAGER.ERR_MULTI_CONFIG_FOR_RECEIVED_FILE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE);
-- -20009
WHEN ENV_MANAGER.ERR_FILE_NOT_FOUND_ON_CLOUD THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD);
-- -20010
WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_VALIDATION_FAILED), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED);
-- -20011
WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NO_CONFIG_MATCH), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH, ENV_MANAGER.MSG_NO_CONFIG_MATCH);
-- -20012
WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN_PREFIX), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX);
-- -20013
WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_TABLE_NOT_EXISTS), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS);
-- -20014
WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_COLUMN_NOT_EXISTS), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS);
-- -20015
WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE);
-- -20016
WHEN ENV_MANAGER.ERR_MISSING_SOURCE_KEY THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_SOURCE_KEY), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY);
-- -20017
WHEN ENV_MANAGER.ERR_NULL_SOURCE_FILE_CONFIG_KEY THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY, ENV_MANAGER.MSG_NULL_SOURCE_FILE_CONFIG_KEY);
-- -20018
WHEN ENV_MANAGER.ERR_DUPLICATED_SOURCE_KEY THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY);
-- -20019
WHEN ENV_MANAGER.ERR_MISSING_CONTAINER_CONFIG THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG, ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG);
-- -20020
WHEN ENV_MANAGER.ERR_MULTIPLE_CONTAINER_ENTRIES THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES);
-- -20021
WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM);
-- -20022
WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD);
-- -20023
WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED);
-- -20024
WHEN ENV_MANAGER.ERR_WRONG_DATE_TIMESTAMP_FORMAT THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT);
-- -20011
WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN
ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED, vParameters, 'FILE_MANAGER');
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED);
-- -20999
WHEN ENV_MANAGER.ERR_UNKNOWN THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END PROCESS_SOURCE_FILE;
----------------------------------------------------------------------------------------------------
FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2)
RETURN PLS_INTEGER
IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
----
PROCESS_SOURCE_FILE(pSourceFileReceivedName => pSourceFileReceivedName);
----
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
RETURN SQLCODE;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RETURN SQLCODE;
END PROCESS_SOURCE_FILE;
----------------------------------------------------------------------------------------------------
FUNCTION GET_DATE_FORMAT(
pTemplateTableName IN VARCHAR2,
pColumnName IN VARCHAR2
)
RETURN VARCHAR2
IS
vDateFormat A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vColumnName VARCHAR2(200);
vGetDefault BOOLEAN := FALSE;
BEGIN
vColumnName := trim(pColumnName);
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pColumnName => '''||nvl(vColumnName, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
BEGIN
SELECT DATE_FORMAT
INTO vDateFormat
FROM CT_MRDS.A_COLUMN_DATE_FORMAT F
WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName
AND F.COLUMN_NAME = vColumnName;
EXCEPTION
WHEN NO_DATA_FOUND THEN
vGetDefault := TRUE;
WHEN TOO_MANY_ROWS THEN
-- Below error should not happened because:
-- Unique constraint added on table A_COLUMN_DATE_FORMAT on columns: (TEMPLATE_TABLE_NAME, COLUMN_NAME)
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT);
END;
IF vGetDefault THEN
BEGIN
SELECT DATE_FORMAT
INTO vDateFormat
FROM CT_MRDS.A_COLUMN_DATE_FORMAT F
WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName
AND F.COLUMN_NAME = 'DEFAULT';
EXCEPTION
WHEN NO_DATA_FOUND THEN
vDateFormat := ENV_MANAGER.gvDefaultDateFormat;
END;
END IF;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vDateFormat;
END GET_DATE_FORMAT;
----------------------------------------------------------------------------------------------------
PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS (
pTemplateTableName IN VARCHAR2,
pColumnList OUT CLOB,
pFieldList OUT CLOB
)
IS
vSchemaName VARCHAR2(200);
vTableName VARCHAR2(200);
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vMaxColumnNameLength PLS_INTEGER := 0;
vColType varchar2(200);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pColumnList => '''||nvl(pColumnList, 'NULL')||''''
,'pFieldList = '''||nvl(pFieldList, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
vSchemaName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.','');
ENV_MANAGER.LOG_PROCESS_EVENT('vSchemaName','DEBUG', vSchemaName);
vTableName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'\..*'),'.','');
ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG', vTableName);
FOR rec IN (
SELECT
'"'||column_name||'"' as quoted_column_name,
column_name,
data_type,
data_length,
char_length, -- MARS-1056: Added for CHAR/BYTE semantics
char_used, -- MARS-1056: Added for CHAR/BYTE semantics
data_precision,
data_scale,
column_id,
max(length(column_name)+1) over (partition by table_name) as max_column_name_length
FROM all_tab_columns
WHERE table_name = UPPER(vTableName)
AND owner = NVL(UPPER(vSchemaName), USER)
ORDER BY column_id
) LOOP
-- Build the column_list string
rec.quoted_column_name := rpad(rec.quoted_column_name, rec.max_column_name_length+2, ' ');
vColType :=
CASE
-- MARS-1056: Fixed VARCHAR2 definition logic to preserve CHAR/BYTE semantics
WHEN rec.data_type = 'VARCHAR2' THEN
CASE
WHEN rec.char_used = 'C' THEN
rec.quoted_column_name || ' VARCHAR2(' || rec.char_length || ' CHAR)'
WHEN rec.char_used = 'B' THEN
rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ' BYTE)'
ELSE
-- Fallback for NULL char_used (should not occur but handle gracefully)
rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ')'
END
-- Other character types (preserve original logic)
WHEN rec.data_type IN ('CHAR', 'NCHAR', 'NVARCHAR2') THEN
rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')'
WHEN rec.data_type = 'NUMBER' THEN
rec.quoted_column_name || ' ' || rec.data_type ||
CASE
WHEN rec.data_precision IS NOT NULL AND rec.data_scale IS NOT NULL THEN
'(' || rec.data_precision || ',' || rec.data_scale || ')'
WHEN rec.data_precision IS NOT NULL THEN
'(' || rec.data_precision || ')'
ELSE
''
END
WHEN rec.data_type IN ('RAW') THEN
rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')'
WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') IN ('DATE', 'TIMESTAMP') THEN
rec.quoted_column_name || ' ' || rec.data_type
ELSE
rec.quoted_column_name || ' ' || rec.data_type
END;
pColumnList := pColumnList ||vColType ||cgBL|| ',';
-- Build the field_list string
-- Note: field_list uses CHAR() for CSV field definitions - this is correct behavior
pFieldList := pFieldList ||
CASE
WHEN rec.data_type = 'DATE' THEN
-- MARS-1046: DATE format - wrap with NORMALIZE_DATE_FORMAT to fix ISO 8601 'T' separator
rec.quoted_column_name || ' DATE ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
WHEN rec.data_type LIKE 'TIMESTAMP%WITH TIME ZONE' THEN
-- MARS-1046: TIMESTAMP WITH TIME ZONE format for ISO 8601 with fractional seconds and timezone
-- Syntax: column_name CHAR(length) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK "format"
-- Use fixed length of 50 for ISO 8601 format (e.g., "2012-03-02T14:16:23.798+01:00" = 29 chars)
rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN
-- Other TIMESTAMP types (without timezone)
-- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format')
rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN
-- For CSV field definitions, use data_length for CHAR() specification
rec.quoted_column_name || ' CHAR(' || rec.data_length || ')'
ELSE
rec.quoted_column_name
END ||cgBL|| ',';
END LOOP;
-- Remove the trailing comma and space from the strings
pColumnList := ' '||RTRIM(pColumnList, ',');
pFieldList := ' '||RTRIM(pFieldList, ',');
ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', pColumnList);
-- TO_DO !!!
-- Add check if pColumnList/pFieldList is empty or not
-- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-- Output the generated column_list and field_list
ENV_MANAGER.LOG_PROCESS_EVENT('column_list' ,'DEBUG',pColumnList);
ENV_MANAGER.LOG_PROCESS_EVENT('field_list' ,'DEBUG',pFieldList);
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
EXCEPTION
-- WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN
-- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters);
-- RAISE_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GENERATE_EXTERNAL_TABLE_PARAMS;
----------------------------------------------------------------------------------------------------
PROCEDURE ADD_SOURCE (
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE,
pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
INSERT INTO CT_MRDS.A_SOURCE(A_SOURCE_KEY, SOURCE_NAME) VALUES (pSourceKey, pSourceName);
COMMIT;
EXCEPTION
WHEN DUP_VAL_ON_INDEX THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END ADD_SOURCE;
----------------------------------------------------------------------------------------------------
PROCEDURE DELETE_SOURCE_CASCADE (
pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
vSharedTemplateCount PLS_INTEGER := 0;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceKey => '''||nvl(pSourceKey, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
-- First pass: Delete container files (those that have CONTAINER_FILE_KEY set)
for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG
where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NOT NULL
ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop
-- Delete processed file records
delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for container config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters);
-- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats
IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN
SELECT COUNT(DISTINCT A_SOURCE_KEY)
INTO vSharedTemplateCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME
AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted
-- Only delete date formats if template table is not shared with other sources
IF vSharedTemplateCount = 0 THEN
delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters);
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters);
END IF;
END IF;
-- Delete container file configuration
delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted container A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters);
end loop;
COMMIT; -- Commit container deletions
-- Second pass: Delete parent files (those that do NOT have CONTAINER_FILE_KEY set)
for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG
where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NULL
ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop
-- Delete processed file records
delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for parent config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters);
-- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats
IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN
SELECT COUNT(DISTINCT A_SOURCE_KEY)
INTO vSharedTemplateCount
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME
AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted
-- Only delete date formats if template table is not shared with other sources
IF vSharedTemplateCount = 0 THEN
delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters);
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters);
END IF;
END IF;
-- Delete parent file configuration
delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted parent A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters);
end loop;
COMMIT; -- Commit parent deletions
-- Delete source system record
DELETE FROM CT_MRDS.A_SOURCE where A_SOURCE_KEY = pSourceKey;
ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE record for source key: '||pSourceKey,'DEBUG', vParameters);
COMMIT; -- Final commit for source deletion
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END DELETE_SOURCE_CASCADE;
----------------------------------------------------------------------------------------------------
FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY (
pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
) RETURN PLS_INTEGER
IS
vSourceFileConfigKey PLS_INTEGER;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileId => '||nvl(to_char(pSourceFileId), 'NULL')));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT A_SOURCE_FILE_CONFIG_KEY
INTO vSourceFileConfigKey
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_ID = pSourceFileId
AND SOURCE_FILE_TYPE = 'CONTAINER';
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
RETURN vSourceFileConfigKey;
EXCEPTION
WHEN NO_DATA_FOUND THEN
ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG := 'No match in A_SOURCE_FILE_CONFIG where SOURCE_FILE_TYPE=''CONTAINER'' and SOURCE_FILE_ID = '''||pSourceFileId||'''';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG, 'WARNING', vParameters);
RETURN NULL;
WHEN TOO_MANY_ROWS THEN
ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES := 'GET_CONTAINER_SOURCE_FILE_CONFIG_KEY: Multiple SOURCE_FILE_TYPE=''CONTAINER'' matches for SOURCE_FILE_ID: '||pSourceFileId||' in A_SOURCE_FILE_CONFIG';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END GET_CONTAINER_SOURCE_FILE_CONFIG_KEY;
----------------------------------------------------------------------------------------------------
FUNCTION GET_SOURCE_FILE_CONFIG_KEY (
pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT'
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE
) RETURN PLS_INTEGER
IS
vSourceFileConfigKey PLS_INTEGER;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileType => '''||nvl(pSourceFileType, 'NULL')||''''
,'pSourceFileId => '''||nvl(pSourceFileId, 'NULL')||''''
,'pTableId => '''||nvl(pTableId, 'NULL')||''''));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
SELECT A_SOURCE_FILE_CONFIG_KEY
INTO vSourceFileConfigKey
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE SOURCE_FILE_TYPE = pSourceFileType
AND SOURCE_FILE_ID = pSourceFileId
AND TABLE_ID = pTableId;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG', vParameters);
RETURN vSourceFileConfigKey;
EXCEPTION
WHEN NO_DATA_FOUND THEN
RETURN NULL;
WHEN TOO_MANY_ROWS THEN
vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE
||cgBL||' '||'GET_SOURCE_FILE_CONFIG_KEY: Multiple matches in A_SOURCE_FILE_CONFIG table WHERE'
||cgBL||' '||'SOURCE_FILE_TYPE: '||pSourceFileType
||cgBL||' '||'SOURCE_FILE_ID: '||pSourceFileId
||cgBL||' '||'TABLE_ID: '||pTableId;
ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, vgMsgTmp);
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN);
END GET_SOURCE_FILE_CONFIG_KEY;
----------------------------------------------------------------------------------------------------
PROCEDURE ADD_SOURCE_FILE_CONFIG (
pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE
,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE
,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE
,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE
,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE
,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL
,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL
,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL
,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049: NOWY PARAMETR
) IS
vSourceFileConfigKey PLS_INTEGER;
vSourceKeyExists PLS_INTEGER := 0;
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => '''||nvl(to_char(pSourceKey), 'NULL')||''''
,'pSourceFileType => '''||nvl(to_char(pSourceFileType), 'NULL')||''''
,'pSourceFileId => '''||nvl(to_char(pSourceFileId), 'NULL')||''''
,'pSourceFileDesc => '''||nvl(to_char(pSourceFileDesc), 'NULL')||''''
,'pSourceFileNamePattern => '''||nvl(to_char(pSourceFileNamePattern), 'NULL')||''''
,'pTableId => '''||nvl(to_char(pTableId), 'NULL')||''''
,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||''''
,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||''''
,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING)
VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding);
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
EXCEPTION
WHEN OTHERS THEN
IF SQLCODE = -2291 THEN
ENV_MANAGER.MSG_MISSING_SOURCE_KEY := 'The Source with A_SOURCE_KEY: '''||pSourceKey||''' not found in parent table A_SOURCE.'
||cgBL||'First add a record to A_SOURCE:'
||cgBL||'call file_manager.add_source(pSourceKey => '''||pSourceKey||''', pSourceName => ''...'')';
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_SOURCE_KEY, 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY);
ELSE
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END IF;
END ADD_SOURCE_FILE_CONFIG;
----------------------------------------------------------------------------------------------------
PROCEDURE ADD_COLUMN_DATE_FORMAT (
pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE
,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE
,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE
) IS
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
,'pColumnName => '''||nvl(pColumnName, 'NULL')||''''
,'pDateFormat => '''||nvl(pDateFormat, 'NULL')||''''
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
INSERT INTO CT_MRDS.A_COLUMN_DATE_FORMAT(TEMPLATE_TABLE_NAME, COLUMN_NAME, DATE_FORMAT)
VALUES (pTemplateTableName, pColumnName, pDateFormat);
COMMIT;
ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN, 'ERROR', vParameters);
ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END ADD_COLUMN_DATE_FORMAT;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2)
RETURN VARCHAR2
IS
BEGIN
CASE pBucketArea
WHEN 'INBOX' THEN RETURN ENV_MANAGER.gvInboxBucketUri;
WHEN 'ODS' THEN RETURN ENV_MANAGER.gvDataBucketUri;
WHEN 'DATA' THEN RETURN ENV_MANAGER.gvDataBucketUri;
WHEN 'ARCHIVE' THEN RETURN ENV_MANAGER.gvArchiveBucketUri;
ELSE
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_INVALID_BUCKET_AREA,
ENV_MANAGER.MSG_INVALID_BUCKET_AREA || ' Provided: ''' || pBucketArea || '''');
END CASE;
END;
----------------------------------------------------------------------------------------------------
FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO (
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
) RETURN VARCHAR2
----
-- Function to get info about File Configuration entry
IS
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vContainerFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vCount PLS_INTEGER := 0;
addColHeader BOOLEAN := TRUE;
vMsgTmp VARCHAR2(32000):= '';
CURSOR cColumnFormat(vTableName A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE) IS
SELECT *
FROM CT_MRDS.A_COLUMN_DATE_FORMAT
WHERE TEMPLATE_TABLE_NAME = vTableName;
vColumnDateFormat A_COLUMN_DATE_FORMAT%ROWTYPE;
FUNCTION FORMAT_CONFIG( pSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE,
pTypeInfo VARCHAR2 DEFAULT 'File Configuration',
pLevel VARCHAR2 DEFAULT '')
RETURN VARCHAR2
IS
vMsg VARCHAR2(9999);
BEGIN
vMsg := ''
||cgBL||pLevel||''||'Details about '||pTypeInfo||':'
||cgBL||pLevel||''||'--------------------------------'
||cgBL||pLevel||'A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY
||cgBL||pLevel||'A_SOURCE_KEY = '||pSourceFileConfig.A_SOURCE_KEY
||cgBL||pLevel||'SOURCE_FILE_TYPE = '||pSourceFileConfig.SOURCE_FILE_TYPE
||cgBL||pLevel||'SOURCE_FILE_ID = '||pSourceFileConfig.SOURCE_FILE_ID
||cgBL||pLevel||'SOURCE_FILE_DESC = '||pSourceFileConfig.SOURCE_FILE_DESC
||cgBL||pLevel||'SOURCE_FILE_NAME_PATTERN = '||pSourceFileConfig.SOURCE_FILE_NAME_PATTERN
||cgBL||pLevel||'TABLE_ID = '||pSourceFileConfig.TABLE_ID
||cgBL||pLevel||'TEMPLATE_TABLE_NAME = '||pSourceFileConfig.TEMPLATE_TABLE_NAME
||cgBL||pLevel||'CONTAINER_FILE_KEY = '||pSourceFileConfig.CONTAINER_FILE_KEY
||cgBL||pLevel||'ODS_SCHEMA_NAME = '||pSourceFileConfig.ODS_SCHEMA_NAME
||cgBL||pLevel||'ARCHIVE_THRESHOLD_DAYS = '||pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS
||cgBL||pLevel||'ARCHIVE_THRESHOLD_FILES_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT
||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM
||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT
||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS
||cgBL||pLevel||''||'--------------------------------'
;
RETURN vMsg;
END FORMAT_CONFIG;
BEGIN
vMsgTmp := '';
-- Get Main Config
BEGIN
SELECT *
INTO vSourceFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
vMsgTmp := FORMAT_CONFIG(pSourceFileConfig => vSourceFileConfig, pTypeInfo => 'File Configuration');
EXCEPTION
WHEN NO_DATA_FOUND THEN
vMsgTmp := 'There is no config entry in A_SOURCE_FILE_CONFIG where A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfigKey;
RETURN vMsgTmp;
END;
-- Get Container Config
IF pIncludeContainerInfo > 0 THEN
BEGIN
SELECT *
INTO vContainerFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG
WHERE A_SOURCE_FILE_CONFIG_KEY = vSourceFileConfig.CONTAINER_FILE_KEY;
vMsgTmp := vMsgTmp || cgBL || FORMAT_CONFIG(pSourceFileConfig => vContainerFileConfig, pTypeInfo => 'related Container Config', pLevel => ' ');
EXCEPTION
WHEN NO_DATA_FOUND THEN
vMsgTmp := vMsgTmp|| cgBL || 'There is no CONTAINER config entry in A_SOURCE_FILE_CONFIG.';
-- RETURN vMsgTmp;
END;
END IF;
-- Get Column Date Format Config
IF pIncludeColumnFormatInfo > 0 THEN
BEGIN
OPEN cColumnFormat(vTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME);
LOOP
FETCH cColumnFormat INTO vColumnDateFormat;
IF cColumnFormat%FOUND AND addColHeader THEN
vCount := 1;
vMsgTmp := vMsgTmp||cgBL|| cgBL || ' Column Date Format config entries:';
vMsgTmp := vMsgTmp||cgBL||''||' --------------------------------';
addColHeader := FALSE;
END IF;
EXIT WHEN cColumnFormat%NOTFOUND;
vMsgTmp := vMsgTmp
||cgBL||' TEMPLATE_TABLE_NAME = '||vColumnDateFormat.TEMPLATE_TABLE_NAME
||cgBL||' COLUMN_NAME = '||vColumnDateFormat.COLUMN_NAME
||cgBL||' DATE_FORMAT = '||vColumnDateFormat.DATE_FORMAT
||cgBL||''||' --------------------------------';
END LOOP;
If vCount=0 THEN
vMsgTmp := vMsgTmp || cgBL || 'There is no Column Date Format config entries in A_COLUMN_DATE_FORMAT where TEMPLATE_TABLE_NAME = '||NVL(vSourceFileConfig.TEMPLATE_TABLE_NAME,'NULL');
END IF;
CLOSE cColumnFormat;
END;
END IF;
RETURN vMsgTmp;
EXCEPTION
WHEN OTHERS THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GET_DET_SOURCE_FILE_CONFIG_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO (
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE
,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1
,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1
,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1
) RETURN VARCHAR2
----
-- Function to get info about File Received entry
IS
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
vMsgTmp VARCHAR2(32000):= '';
BEGIN
BEGIN
SELECT *
INTO vSourceFileReceived
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey;
vMsgTmp := ''
||cgBL||''||'Details about received file:'
||cgBL||''||'--------------------------------'
||cgBL||'A_SOURCE_FILE_RECEIVED_KEY = '||vSourceFileReceived.A_SOURCE_FILE_RECEIVED_KEY
||cgBL||'A_SOURCE_FILE_CONFIG_KEY = '||vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY
||cgBL||'SOURCE_FILE_NAME = '||vSourceFileReceived.SOURCE_FILE_NAME
||cgBL||'CHECKSUM = '||vSourceFileReceived.CHECKSUM
||cgBL||'CREATED = '||vSourceFileReceived.CREATED
||cgBL||'BYTES = '||vSourceFileReceived.BYTES
||cgBL||'RECEPTION_DATE = '||vSourceFileReceived.RECEPTION_DATE
||cgBL||'PROCESSING_STATUS = '||vSourceFileReceived.PROCESSING_STATUS
||cgBL||'EXTERNAL_TABLE_NAME = '||vSourceFileReceived.EXTERNAL_TABLE_NAME
||cgBL||'PARTITION_YEAR = '||vSourceFileReceived.PARTITION_YEAR
||cgBL||'PARTITION_MONTH = '||vSourceFileReceived.PARTITION_MONTH
||cgBL||''||'--------------------------------'
;
EXCEPTION
WHEN NO_DATA_FOUND THEN
vMsgTmp := 'There is no data in A_SOURCE_FILE_RECEIVED where A_SOURCE_FILE_RECEIVED_KEY = '||pSourceFileReceivedKey;
RETURN vMsgTmp;
END;
IF pIncludeConfigInfo>0 THEN
vMsgTmp := vMsgTmp || cgBL || CT_MRDS.FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO(vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY,pIncludeContainerInfo,pIncludeColumnFormatInfo);
END IF;
RETURN vMsgTmp;
EXCEPTION
WHEN OTHERS THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GET_DET_SOURCE_FILE_RECEIVED_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_DET_USER_LOAD_OPERATIONS (
pOperationId PLS_INTEGER
) RETURN VARCHAR2
----
-- Function to get info about File Received entry
IS
vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE;
vMsgTmp VARCHAR2(32000):= '';
BEGIN
BEGIN
SELECT *
INTO vUserLoadOperations
FROM USER_LOAD_OPERATIONS
WHERE id = pOperationId;
vMsgTmp := ''
||''||'Details about USER_LOAD_OPERATIONS where ID = '||pOperationId
||cgBL||''||'--------------------------------'
||cgBL||'ID = '||vUserLoadOperations.ID
||cgBL||'TYPE = '||vUserLoadOperations.TYPE
||cgBL||'SID = '||vUserLoadOperations.SID
||cgBL||'SERIAL# = '||vUserLoadOperations.SERIAL#
||cgBL||'START_TIME = '||vUserLoadOperations.START_TIME
||cgBL||'UPDATE_TIME = '||vUserLoadOperations.UPDATE_TIME
||cgBL||'STATUS = '||vUserLoadOperations.STATUS
||cgBL||'OWNER_NAME = '||vUserLoadOperations.OWNER_NAME
||cgBL||'TABLE_NAME = '||vUserLoadOperations.TABLE_NAME
||cgBL||'PARTITION_NAME = '||vUserLoadOperations.PARTITION_NAME
||cgBL||'SUBPARTITION_NAME = '||vUserLoadOperations.SUBPARTITION_NAME
||cgBL||'FILE_URI_LIST = '||vUserLoadOperations.FILE_URI_LIST
||cgBL||'ROWS_LOADED = '||vUserLoadOperations.ROWS_LOADED
||cgBL||'LOGFILE_TABLE = '||vUserLoadOperations.LOGFILE_TABLE
||cgBL||'BADFILE_TABLE = '||vUserLoadOperations.BADFILE_TABLE
||cgBL||'STATUS_TABLE = '||vUserLoadOperations.STATUS_TABLE
||cgBL||'TEMPEXT_TABLE = '||vUserLoadOperations.TEMPEXT_TABLE
||cgBL||'CREDENTIAL_NAME = '||vUserLoadOperations.CREDENTIAL_NAME
||cgBL||'EXPIRATION_TIME = '||vUserLoadOperations.EXPIRATION_TIME
||cgBL||''||'--------------------------------'
;
EXCEPTION
WHEN NO_DATA_FOUND THEN
vMsgTmp := 'There is no data in USER_LOAD_OPERATIONS where ID = '||pOperationId;
RETURN vMsgTmp;
END;
RETURN vMsgTmp;
EXCEPTION
WHEN OTHERS THEN
RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
END GET_DET_USER_LOAD_OPERATIONS;
----------------------------------------------------------------------------------------------------
FUNCTION ANALYZE_VALIDATION_ERRORS(
pSourceFileReceivedKey IN NUMBER
) RETURN VARCHAR2
IS
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
vValidationLogTable VARCHAR2(128);
vTemplateSchema VARCHAR2(128);
vTemplateTable VARCHAR2(128);
vCsvFileUri VARCHAR2(4000);
vParameters VARCHAR2(4000);
vResult VARCHAR2(32000);
BEGIN
vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
'pSourceFileReceivedKey => ' || pSourceFileReceivedKey
));
ENV_MANAGER.LOG_PROCESS_EVENT('Start', 'DEBUG', vParameters);
-- Get file and config information
BEGIN
-- Get source file received data first
SELECT *
INTO vSourceFileReceived
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
WHERE sfr.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey;
-- Get source file config data
SELECT *
INTO vSourceFileConfig
FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc
WHERE sfc.A_SOURCE_FILE_CONFIG_KEY = vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY;
EXCEPTION
WHEN NO_DATA_FOUND THEN
ENV_MANAGER.LOG_PROCESS_ERROR('Source file or config not found for key: ' || pSourceFileReceivedKey, vParameters);
RETURN 'Error: Source file with key ' || pSourceFileReceivedKey || ' not found in A_SOURCE_FILE_RECEIVED table';
END;
-- Extract template schema and table from template table name
vTemplateSchema := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '^([^.]+)');
vTemplateTable := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '\.(.+)$', 1, 1, NULL, 1);
-- Build CSV file URI
vCsvFileUri := ENV_MANAGER.gvInboxBucketUri ||
'INBOX/' || vSourceFileConfig.A_SOURCE_KEY || '/' ||
vSourceFileConfig.SOURCE_FILE_ID || '/' ||
vSourceFileConfig.TABLE_ID || '/' ||
vSourceFileReceived.SOURCE_FILE_NAME;
-- Find validation log table (most recent one with errors)
-- Look for validation log tables related to this external table
BEGIN
SELECT table_name
INTO vValidationLogTable
FROM (
SELECT table_name,
REGEXP_SUBSTR(table_name, '\$([0-9]+)_', 1, 1, NULL, 1) as log_number
FROM USER_TABLES
WHERE table_name LIKE 'VALIDATE$%_LOG'
ORDER BY TO_NUMBER(log_number) DESC
)
WHERE ROWNUM = 1;
EXCEPTION
WHEN NO_DATA_FOUND THEN
-- If no validation log tables found, use a default name
vValidationLogTable := 'VALIDATE$999_LOG';
END;
ENV_MANAGER.LOG_PROCESS_EVENT('Calling ENV_MANAGER.ANALYZE_VALIDATION_ERRORS with parameters: ' ||
'LogTable=' || vValidationLogTable ||
', Schema=' || vTemplateSchema ||
', Table=' || vTemplateTable ||
', URI=' || SUBSTR(vCsvFileUri, 1, 100) || '...', 'DEBUG', vParameters);
-- Call the main function with derived parameters
vResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS(
pValidationLogTable => vValidationLogTable,
pTemplateSchema => vTemplateSchema,
pTemplateTable => vTemplateTable,
pCsvFileUri => vCsvFileUri
);
ENV_MANAGER.LOG_PROCESS_EVENT('End', 'DEBUG', vParameters);
RETURN vResult;
EXCEPTION
WHEN OTHERS THEN
ENV_MANAGER.LOG_PROCESS_ERROR('Error in ANALYZE_VALIDATION_ERRORS: ' || SQLERRM, vParameters);
RETURN 'Error analyzing validation errors: ' || SQLERRM ||
cgBL || 'Source File Key: ' || pSourceFileReceivedKey ||
cgBL || 'Check A_SOURCE_FILE_RECEIVED and A_SOURCE_FILE_CONFIG tables for data integrity.';
END ANALYZE_VALIDATION_ERRORS;
----------------------------------------------------------------------------------------------------
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION
RETURN VARCHAR2
IS
BEGIN
RETURN PACKAGE_VERSION;
END GET_VERSION;
----------------------------------------------------------------------------------------------------
FUNCTION GET_BUILD_INFO
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO(
pPackageName => 'FILE_MANAGER',
pVersion => PACKAGE_VERSION,
pBuildDate => PACKAGE_BUILD_DATE,
pAuthor => PACKAGE_AUTHOR
);
END GET_BUILD_INFO;
----------------------------------------------------------------------------------------------------
FUNCTION GET_VERSION_HISTORY
RETURN VARCHAR2
IS
BEGIN
RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY(
pPackageName => 'FILE_MANAGER',
pVersionHistory => VERSION_HISTORY
);
END GET_VERSION_HISTORY;
----------------------------------------------------------------------------------------------------
END;
/
/
Reference in New Issue View Git Blame Copy Permalink