create or replace PACKAGE BODY CT_MRDS.FILE_MANAGER AS ---------------------------------------------------------------------------------------------------- -- PRIVATE FUNCTION: NORMALIZE_DATE_FORMAT ---------------------------------------------------------------------------------------------------- /** * Purpose: Normalize Oracle date format strings for use in external tables * * Problem: ISO 8601 formats like 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' fail because * literal character 'T' must be enclosed in double quotes for Oracle * external table DATE column definitions. * * Solution: Detect unquoted 'T' separator and wrap it in double quotes * * Parameters: * pDateFormat - Original date format from A_COLUMN_DATE_FORMAT table * * Returns: Normalized format with quoted 'T' if applicable * * Examples: * Input: 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' * Output: 'YYYY-MM-DD"T"HH24:MI:SS.FF3TZH:TZM' * * Input: 'DD/MM/YYYY HH24:MI:SS' (no T) * Output: 'DD/MM/YYYY HH24:MI:SS' (unchanged) * * Input: 'YYYY-MM-DD"T"HH24:MI:SS' (already quoted) * Output: 'YYYY-MM-DD"T"HH24:MI:SS' (unchanged) * * Author: Grzegorz Michalski * Date: 2025-11-27 * Version: 1.0.0 (MARS-1046) */ FUNCTION NORMALIZE_DATE_FORMAT(pDateFormat VARCHAR2) RETURN VARCHAR2 IS vNormalizedFormat VARCHAR2(500); BEGIN -- Return NULL if input is NULL IF pDateFormat IS NULL THEN RETURN NULL; END IF; vNormalizedFormat := pDateFormat; -- Check if 'T' separator exists and is NOT already quoted -- Pattern: [YMD]T[HM] (date component + T + time component) IF INSTR(vNormalizedFormat, '"T"') = 0 AND REGEXP_LIKE(vNormalizedFormat, '[YMD]T[HM]') THEN -- Wrap 'T' in double quotes using regex replace -- Pattern matches: (date format char) + T + (time format char) -- Replacement: \1 + "T" + \2 vNormalizedFormat := REGEXP_REPLACE(vNormalizedFormat, '([YMD])T([HM])', '\1"T"\2'); END IF; RETURN vNormalizedFormat; EXCEPTION WHEN OTHERS THEN -- If normalization fails, return original format (safety fallback) RETURN pDateFormat; END NORMALIZE_DATE_FORMAT; ---------------------------------------------------------------------------------------------------- FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri IN VARCHAR2 DEFAULT NULL , pSourceFileReceivedKey IN NUMBER DEFAULT NULL , pSourceFileConfigKey IN NUMBER DEFAULT NULL) RETURN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pFileUri => '''||nvl(pFileUri,'NULL')||'''' ,'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') ,'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); BEGIN IF pFileUri IS NOT NULL THEN SELECT * INTO vSourceFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE REGEXP_LIKE(pFileUri, A_SOURCE_KEY||'/'||SOURCE_FILE_ID||'/'||TABLE_ID||'/'||SOURCE_FILE_NAME_PATTERN); ELSIF pSourceFileReceivedKey IS NOT NULL THEN SELECT T.* INTO vSourceFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG T, CT_MRDS.A_SOURCE_FILE_RECEIVED R WHERE T.A_SOURCE_FILE_CONFIG_KEY = R.A_SOURCE_FILE_CONFIG_KEY AND R.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; ELSIF pSourceFileConfigKey IS NOT NULL THEN SELECT * INTO vSourceFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; ELSE RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); END IF; -- Set global package variable vgSourceFileConfigKey - used in error messages vgSourceFileConfigKey := vSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY; EXCEPTION WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); WHEN NO_DATA_FOUND THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); WHEN TOO_MANY_ROWS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); WHEN OTHERS THEN -- Log complete error details including full stack trace and backtrace ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); RETURN vSourceFileConfig; END GET_SOURCE_FILE_CONFIG; ---------------------------------------------------------------------------------------------------- FUNCTION GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey IN NUMBER DEFAULT NULL) -- -- Get source file received info -- RETURN tSourceFileReceived IS vSourceFileReceived tSourceFileReceived; vBucket VARCHAR2(400); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); BEGIN SELECT R.A_SOURCE_FILE_RECEIVED_KEY, R.A_SOURCE_FILE_CONFIG_KEY, 'INBOX'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_INBOX, 'ODS'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_ODS, 'QUARANTINE'||'/'||T.A_SOURCE_KEY||'/'||T.TABLE_ID||'/' as SOURCE_FILE_PREFIX_QUARANTINE, 'ARCHIVE'||'/'||T.A_SOURCE_KEY||'/'||T.SOURCE_FILE_ID||'/' as SOURCE_FILE_PREFIX_ARCHIVE, R.SOURCE_FILE_NAME, R.RECEPTION_DATE, R.PROCESSING_STATUS, R.EXTERNAL_TABLE_NAME INTO vSourceFileReceived FROM CT_MRDS.A_SOURCE_FILE_RECEIVED R, CT_MRDS.A_SOURCE_FILE_CONFIG T WHERE R.A_SOURCE_FILE_CONFIG_KEY = T.A_SOURCE_FILE_CONFIG_KEY AND A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; EXCEPTION WHEN NO_DATA_FOUND THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); WHEN TOO_MANY_ROWS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); RETURN vSourceFileReceived; END GET_SOURCE_FILE_RECEIVED_INFO; ---------------------------------------------------------------------------------------------------- FUNCTION REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName IN VARCHAR2) RETURN PLS_INTEGER -- -- Register a newly received source file A_SOURCE_FILE_RECEIVED -- This overload automatically determines source file type from the file name -- IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vSourceFileReceivedKey PLS_INTEGER; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO',vParameters); vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedName); vSourceFileReceivedKey := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName, vSourceFileConfig); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); RETURN vSourceFileReceivedKey; EXCEPTION WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END REGISTER_SOURCE_FILE_RECEIVED; ---------------------------------------------------------------------------------------------------- FUNCTION REGISTER_SOURCE_FILE_RECEIVED( pSourceFileReceivedName IN VARCHAR2 ,pSourceFileConfig IN CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE) RETURN PLS_INTEGER -- -- Register a newly received source file A_SOURCE_FILE_RECEIVED -- IS vExternalTableName VARCHAR2(200); vDirName VARCHAR2(1000); vFileName VARCHAR2(1000); vChecksum A_SOURCE_FILE_RECEIVED.CHECKSUM%TYPE; vCreated A_SOURCE_FILE_RECEIVED.CREATED%TYPE; vBytes A_SOURCE_FILE_RECEIVED.BYTES%TYPE; vSourceFileReceivedKey PLS_INTEGER; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vRow CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '''||nvl(pSourceFileReceivedName, 'NULL')||'''' ,'pSourceFileConfig => '||'tSourceFileConfig record type')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); vDirName := REGEXP_SUBSTR(pSourceFileReceivedName, '(.*/)(.*)', 1, 1, NULL, 1); -- Remove prefix from file name vFileName := REGEXP_SUBSTR(pSourceFileReceivedName,'[^/]*$'); ENV_MANAGER.LOG_PROCESS_EVENT('gvCredentialName','DEBUG',ENV_MANAGER.gvCredentialName); ENV_MANAGER.LOG_PROCESS_EVENT('gvInboxBucketUri','DEBUG',ENV_MANAGER.gvInboxBucketUri); ENV_MANAGER.LOG_PROCESS_EVENT('vDirName','DEBUG',vDirName); SELECT checksum, created, bytes INTO vChecksum, vCreated, vBytes FROM DBMS_CLOUD.LIST_OBJECTS(ENV_MANAGER.gvCredentialName, ENV_MANAGER.gvInboxBucketUri || vDirName ) WHERE object_name = vFileName ; vSourceFileReceivedKey := CT_MRDS.A_SOURCE_FILE_RECEIVED_KEY_SEQ.NEXTVAL; vExternalTableName := REPLACE( REGEXP_SUBSTR(pSourceFileConfig.TEMPLATE_TABLE_NAME||'_'||vSourceFileReceivedKey, '\..*'), '.',''); INSERT INTO CT_MRDS.A_SOURCE_FILE_RECEIVED (A_SOURCE_FILE_RECEIVED_KEY, A_SOURCE_FILE_CONFIG_KEY, SOURCE_FILE_NAME, RECEPTION_DATE, PROCESSING_STATUS, EXTERNAL_TABLE_NAME, CHECKSUM, CREATED, BYTES) VALUES (vSourceFileReceivedKey, pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY, vFileName, SYSDATE, 'RECEIVED', vExternalTableName, vChecksum, vCreated, vBytes); COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); RETURN vSourceFileReceivedKey; EXCEPTION WHEN NO_DATA_FOUND THEN vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD ||cgBL||' '||'File: '||ENV_MANAGER.gvInboxBucketUri || vDirName || vFileName; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, vgMsgTmp); WHEN DUP_VAL_ON_INDEX THEN select * into vRow from CT_MRDS.A_SOURCE_FILE_RECEIVED where CHECKSUM = vChecksum and CREATED = vCreated and BYTES = vBytes ; vgMsgTmp := ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED ||cgBL||' '||'Details about existing File: ' ||cgBL||' '||'-------------------------' ||cgBL||' '||'A_SOURCE_FILE_RECEIVED_KEY = '||vRow.A_SOURCE_FILE_RECEIVED_KEY ||cgBL||' '||'A_SOURCE_FILE_CONFIG_KEY = '||vRow.A_SOURCE_FILE_CONFIG_KEY ||cgBL||' '||'SOURCE_FILE_NAME = '||vRow.SOURCE_FILE_NAME ||cgBL||' '||'CHECKSUM = '||vRow.CHECKSUM ||cgBL||' '||'CREATED = '||vRow.CREATED ||cgBL||' '||'BYTES = '||vRow.BYTES ||cgBL||' '||'RECEPTION_DATE = '||vRow.RECEPTION_DATE ||cgBL||' '||'PROCESSING_STATUS = '||vRow.PROCESSING_STATUS ||cgBL||' '||'EXTERNAL_TABLE_NAME = '||vRow.EXTERNAL_TABLE_NAME ||cgBL||' '||'-------------------------' ||cgBL||' '||'There cannot be two files with the same values for (CHECKSUM, CREATED, BYTES)' ; -- vChecksum, vCreated, vBytes ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, vgMsgTmp); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END REGISTER_SOURCE_FILE_RECEIVED; ---------------------------------------------------------------------------------------------------- PROCEDURE SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey IN PLS_INTEGER, pStatus IN VARCHAR2) -- -- Change status of file in the A_SOURCE_FILE_RECEIVED table -- IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey),'NULL') ,'pStatus => '''||nvl(pStatus, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED SET PROCESSING_STATUS=pStatus WHERE A_SOURCE_FILE_RECEIVED_KEY=pSourceFileReceivedKey; COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to '||pStatus,'INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END SET_SOURCE_FILE_RECEIVED_STATUS; ---------------------------------------------------------------------------------------------------- FUNCTION GET_EXTERNAL_TABLE_COLUMNS(pTargetTableTemplate IN VARCHAR2) RETURN CLOB -- -- Create list of columns for DBMS_CLOUD.CREATE_EXTERNAL_TABLE from existing template table -- IS vColumnList CLOB; vTableName VARCHAR2(200); vSchemaName VARCHAR2(200); BEGIN vSchemaName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'.*\.'),'.',''); vTableName := REPLACE(REGEXP_SUBSTR(pTargetTableTemplate,'\..*'),'.',''); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SQLTERMINATOR', True); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'COLLATION_CLAUSE', 'NEVER'); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'REF_CONSTRAINTS', False); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'STORAGE', False); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'TABLESPACE', False); DBMS_METADATA.SET_TRANSFORM_PARAM(-1, 'SEGMENT_ATTRIBUTES', False); vColumnList := RTRIM( LTRIM( REGEXP_SUBSTR(DBMS_METADATA.GET_DDL('TABLE', vTableName, vSchemaName),'\(.*\)',1,1,'mn'), '('), ')'); RETURN vColumnList; END GET_EXTERNAL_TABLE_COLUMNS; ---------------------------------------------------------------------------------------------------- PROCEDURE CREATE_EXTERNAL_TABLE ( pTableName IN VARCHAR2, pTemplateTableName IN VARCHAR2, pPrefix IN VARCHAR2, pBucketUri IN VARCHAR2 DEFAULT ENV_MANAGER.gvInboxBucketUri, pFileName IN VARCHAR2 DEFAULT NULL, pDelimiter IN VARCHAR2 DEFAULT ',', pEncoding IN VARCHAR2 DEFAULT NULL -- MARS-1049: NEW PARAMETER FOR FILE ENCODING ) -- -- Create external table for a single source file to validate the file structure -- IS vTableName VARCHAR2(200); vColumnList CLOB; vFieldList CLOB; vFormat VARCHAR2(200); vPrefix VARCHAR2(200); vFileName VARCHAR2(1000); vFileExtension VARCHAR2(200); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTableName => '''||nvl(pTableName, 'NULL')||'''' ,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pPrefix => '''||nvl(pPrefix, 'NULL')||'''' ,'pBucketUri => '''||nvl(pBucketUri, 'NULL')||'''' ,'pFileName => '''||nvl(pFileName, 'NULL')||'''' ,'pDelimiter => '''||nvl(pDelimiter, 'NULL')||'''' ,'pEncoding => '''||nvl(pEncoding, 'NULL')||'''' -- MARS-1049: NOWY )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); -- Strip off leading and trailing slashes from prefix vPrefix := TRIM(BOTH '/' FROM pPrefix); -- Generate column and field list from template table GENERATE_EXTERNAL_TABLE_PARAMS (pTemplateTableName, vColumnList, vFieldList); --vFormat evaluation based on pBucketUri first, then pPrefix -- Archive bucket should use parquet + implicit partitioning regardless of prefix IF INSTR(pBucketUri, ENV_MANAGER.gvArchiveBucketName)>0 THEN vFormat := '{"type": "parquet" ,"implicit_partition_type": "hive" ,"implicit_partition_columns":["PARTITION_YEAR","PARTITION_MONTH"]}'; vColumnList := vColumnList||cgBL||' , "PARTITION_YEAR" varchar2(4)'||cgBL||', "PARTITION_MONTH" varchar2(2)'; vFieldList := NULL; vFileExtension := '.parquet'; -- For INBOX, ODS, and other ARCHIVE prefixes (not in archive bucket) use CSV ELSIF SUBSTR(pPrefix,1,5) = 'INBOX' OR SUBSTR(pPrefix,1,3) = 'ODS' OR SUBSTR(pPrefix,1,7) = 'ARCHIVE' THEN -- MARS-1049: Create format with encoding if specified IF pDelimiter = '|' THEN IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN vFormat := json_object( 'delimiter' VALUE '|', 'skipheaders' VALUE '1', 'characterset' VALUE pEncoding ); ELSE vFormat := json_object('delimiter' VALUE '|', 'skipheaders' VALUE '1'); END IF; ELSE IF pEncoding IS NOT NULL AND LENGTH(TRIM(pEncoding)) > 0 THEN vFormat := json_object( 'type' VALUE 'CSV', 'skipheaders' VALUE '1', 'ignoremissingcolumns' VALUE 'true', 'characterset' VALUE pEncoding ); ELSE vFormat := json_object('type' VALUE 'CSV', 'skipheaders' VALUE '1', 'ignoremissingcolumns' value 'true'); END IF; END IF; vFileExtension := '.csv'; ELSE RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); END IF; -- No filename give: Match all csv files IF pFileName IS NOT NULL THEN vFileName := pFileName; ELSE vFileName := pBucketUri||vPrefix||'/*'||vFileExtension; END IF; ENV_MANAGER.LOG_PROCESS_EVENT('pTableName', 'DEBUG', pTableName); ENV_MANAGER.LOG_PROCESS_EVENT('ENV_MANAGER.vpCredentialName', 'DEBUG', ENV_MANAGER.gvCredentialName); ENV_MANAGER.LOG_PROCESS_EVENT('vFileName', 'DEBUG', vFileName); ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', vColumnList); ENV_MANAGER.LOG_PROCESS_EVENT('vFieldList', 'DEBUG', vFieldList); ENV_MANAGER.LOG_PROCESS_EVENT('vFormat', 'DEBUG', vFormat); -- Pre-validation: Check CSV column count for CSV files only IF SUBSTR(pPrefix,1,5) = 'INBOX' AND pFileName IS NOT NULL THEN DECLARE vCsvFirstLine VARCHAR2(4000); vCsvColCount NUMBER := 0; vTemplateColCount NUMBER := 0; vExcessColumns VARCHAR2(2000); -- Get template column count CURSOR c_template_count IS SELECT COUNT(*) as col_count FROM ALL_TAB_COLUMNS WHERE OWNER = UPPER(REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.','')) AND TABLE_NAME = UPPER(REGEXP_REPLACE(pTemplateTableName,'^.*\.','')); BEGIN -- Get template column count FOR rec IN c_template_count LOOP vTemplateColCount := rec.col_count; END LOOP; -- Read first line of CSV to count columns BEGIN SELECT UTL_RAW.CAST_TO_VARCHAR2( DBMS_LOB.SUBSTR( DBMS_CLOUD.GET_OBJECT( credential_name => ENV_MANAGER.gvCredentialName, object_uri => pFileName ), 4000, 1 ) ) INTO vCsvFirstLine FROM DUAL; -- Count commas in header line + 1 for total columns vCsvColCount := REGEXP_COUNT(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), ',') + 1; ENV_MANAGER.LOG_PROCESS_EVENT('CSV Column Count: ' || vCsvColCount || ', Template Column Count: ' || vTemplateColCount, 'INFO', vParameters); -- Check for excess columns IF vCsvColCount > vTemplateColCount THEN vgMsgTmp := ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED ||cgBL||'EXCESS COLUMNS DETECTED!' ||cgBL||'CSV file has ' || vCsvColCount || ' columns but template expects only ' || vTemplateColCount ||cgBL||'Excess columns: ' || (vCsvColCount - vTemplateColCount) ||cgBL||'CSV header: ' || SUBSTR(REGEXP_SUBSTR(vCsvFirstLine, '[^'||chr(10)||']*'), 1, 200) ||cgBL||'POSSIBLE SOLUTIONS:' ||cgBL||' 1. Remove excess columns from CSV file before processing' ||cgBL||' 2. Add excess columns to template table: ' || pTemplateTableName; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, vgMsgTmp); END IF; EXCEPTION WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN RAISE; -- Re-raise the excess columns error WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN RAISE; -- Re-raise the validation error WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT('Warning: Could not perform pre-validation column count check: ' || SQLERRM, 'WARN', vParameters); -- Continue with normal processing if pre-validation fails END; END; END IF; DBMS_CLOUD.CREATE_EXTERNAL_TABLE( TABLE_NAME => pTableName, CREDENTIAL_NAME => ENV_MANAGER.gvCredentialName, FILE_URI_LIST => vFileName, COLUMN_LIST => vColumnList, FIELD_LIST => vFieldList, FORMAT => vFormat ); ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); EXCEPTION WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN RAISE; -- Re-raise the excess columns error with specific code -20011 WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN vgMsgTmp := ENV_MANAGER.MSG_UNKNOWN_PREFIX || ': ' || pPrefix; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, vgMsgTmp); WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END CREATE_EXTERNAL_TABLE; ---------------------------------------------------------------------------------------------------- PROCEDURE CREATE_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) -- -- Create external table for a single source file to validate the file structure -- IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vSourceFileReceived tSourceFileReceived; vTableName VARCHAR2(200); vFileName VARCHAR2(1000); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); vTableName := vSourceFileConfig.TEMPLATE_TABLE_NAME; vFileName := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX||vSourceFileReceived.SOURCE_FILE_NAME; CREATE_EXTERNAL_TABLE( pTableName => vSourceFileReceived.EXTERNAL_TABLE_NAME, pTemplateTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME, pPrefix => vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX, pBucketUri => ENV_MANAGER.gvInboxBucketUri, pFileName => vFileName, pDelimiter => ',', pEncoding => vSourceFileConfig.ENCODING -- MARS-1049: NOWY PARAMETR ); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); END CREATE_EXTERNAL_TABLE; ---------------------------------------------------------------------------------------------------- PROCEDURE VALIDATE_SOURCE_FILE_RECEIVED(pSourceFileReceivedKey IN NUMBER) -- -- Check the structure of the received file using DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE -- IS vSourceFileReceived tSourceFileReceived; vOperationId NUMBER := -1; vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; vErrors NUMBER := 0; vNumRows NUMBER := 0; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); ENV_MANAGER.LOG_PROCESS_EVENT('vSourceFileReceived.EXTERNAL_TABLE_NAME: '||vSourceFileReceived.EXTERNAL_TABLE_NAME,'DEBUG', vParameters); BEGIN DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(vSourceFileReceived.EXTERNAL_TABLE_NAME, vOperationId); EXCEPTION WHEN OTHERS THEN -- Log complete error details including full stack trace and backtrace ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_VALIDATION_FAILED, vParameters, 'FILE_MANAGER'); -- Call detailed validation error analysis and log the results DECLARE vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vValidationLogTable VARCHAR2(200); vTemplateSchema VARCHAR2(200); vTemplateTable VARCHAR2(200); vCsvFileUri VARCHAR2(2000); vAnalysisResult VARCHAR2(32000); vFailedOperationId NUMBER; BEGIN -- Get source file configuration vSourceFileConfig := GET_SOURCE_FILE_CONFIG(pSourceFileReceivedKey => pSourceFileReceivedKey); -- Extract template schema and table from template table name vTemplateSchema := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'.*\.'),'.',''); vTemplateTable := REPLACE(REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME,'\..*'),'.',''); -- Construct CSV file URI vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || vSourceFileReceived.SOURCE_FILE_PREFIX_INBOX || vSourceFileReceived.SOURCE_FILE_NAME; -- Find the failed validation operation ID SELECT MAX(ID) INTO vFailedOperationId FROM USER_LOAD_OPERATIONS WHERE TABLE_NAME = vSourceFileReceived.EXTERNAL_TABLE_NAME AND TYPE = 'VALIDATE' AND STATUS != 'COMPLETED'; -- Get validation log table name IF vFailedOperationId IS NOT NULL THEN SELECT LOGFILE_TABLE INTO vValidationLogTable FROM USER_LOAD_OPERATIONS WHERE ID = vFailedOperationId; -- Call detailed error analysis vAnalysisResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( pValidationLogTable => vValidationLogTable, pTemplateSchema => vTemplateSchema, pTemplateTable => vTemplateTable, pCsvFileUri => vCsvFileUri ); -- Log detailed analysis results ENV_MANAGER.LOG_PROCESS_EVENT('DETAILED VALIDATION ERROR ANALYSIS:', 'ERROR', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT(vAnalysisResult, 'ERROR', vParameters); END IF; EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT('Error during validation analysis: ' || SQLERRM, 'ERROR', vParameters); END; MOVE_FILE(pSourceFileReceivedKey => pSourceFileReceivedKey, pDestination => 'QUARANTINE'); -- Ensure the status change is committed before raising exception COMMIT; RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); END; ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); IF vOperationId = -1 THEN RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); END IF; SELECT BADFILE_TABLE, ROWS_LOADED, STATUS INTO vBadfileTable, vNumRows, vStatus FROM USER_LOAD_OPERATIONS WHERE ID = vOperationId; -- MARS-1409: Extract and validate A_WORKFLOW_HISTORY_KEY from external table DECLARE vWorkflowHistoryKey NUMBER; vWorkflowKeyCount NUMBER; vWorkflowKeyDistinct NUMBER; vDynamicSQL VARCHAR2(1000); BEGIN -- Build dynamic SQL to count distinct A_WORKFLOW_HISTORY_KEY values vDynamicSQL := 'SELECT COUNT(*), COUNT(DISTINCT A_WORKFLOW_HISTORY_KEY) FROM ' || vSourceFileReceived.EXTERNAL_TABLE_NAME; ENV_MANAGER.LOG_PROCESS_EVENT('MARS-1409: Extracting A_WORKFLOW_HISTORY_KEY from external table', 'DEBUG', vParameters); -- Count total rows and distinct workflow keys EXECUTE IMMEDIATE vDynamicSQL INTO vWorkflowKeyCount, vWorkflowKeyDistinct; ENV_MANAGER.LOG_PROCESS_EVENT('MARS-1409: Total rows: ' || vWorkflowKeyCount || ', Distinct A_WORKFLOW_HISTORY_KEY values: ' || vWorkflowKeyDistinct, 'DEBUG', vParameters); -- Validate workflow key presence and uniqueness IF vWorkflowKeyDistinct = 0 OR vWorkflowKeyDistinct IS NULL THEN -- No A_WORKFLOW_HISTORY_KEY found or all values are NULL vgMsgTmp := ENV_MANAGER.MSG_WORKFLOW_KEY_NULL || ' [File: ' || vSourceFileReceived.SOURCE_FILE_NAME || ']'; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WORKFLOW_KEY_NULL, vgMsgTmp); ELSIF vWorkflowKeyDistinct > 1 THEN -- Multiple different A_WORKFLOW_HISTORY_KEY values found vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_WORKFLOW_KEYS || ' [Found: ' || vWorkflowKeyDistinct || ' distinct values in file: ' || vSourceFileReceived.SOURCE_FILE_NAME || ']'; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_WORKFLOW_KEYS, vgMsgTmp); ELSE -- Exactly one A_WORKFLOW_HISTORY_KEY value - fetch and save it vDynamicSQL := 'SELECT DISTINCT A_WORKFLOW_HISTORY_KEY FROM ' || vSourceFileReceived.EXTERNAL_TABLE_NAME; EXECUTE IMMEDIATE vDynamicSQL INTO vWorkflowHistoryKey; ENV_MANAGER.LOG_PROCESS_EVENT('MARS-1409: Extracted A_WORKFLOW_HISTORY_KEY: ' || vWorkflowHistoryKey, 'DEBUG', vParameters); -- Update A_SOURCE_FILE_RECEIVED with workflow history key UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED SET A_WORKFLOW_HISTORY_KEY = vWorkflowHistoryKey WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; ENV_MANAGER.LOG_PROCESS_EVENT('MARS-1409: Updated A_SOURCE_FILE_RECEIVED with A_WORKFLOW_HISTORY_KEY: ' || vWorkflowHistoryKey, 'INFO', vParameters); END IF; EXCEPTION WHEN ENV_MANAGER.ERR_WORKFLOW_KEY_NULL THEN SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE; WHEN ENV_MANAGER.ERR_MULTIPLE_WORKFLOW_KEYS THEN SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE; WHEN OTHERS THEN vgMsgTmp := 'MARS-1409: Error extracting A_WORKFLOW_HISTORY_KEY: ' || SQLERRM; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); END; -- DBMS_OUTPUT.PUT_LINE(vStatus); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATED'); ENV_MANAGER.LOG_PROCESS_EVENT('File status changed to VALIDATED','DEBUG', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED; ENV_MANAGER.LOG_PROCESS_ERROR(vgMsgTmp, vParameters, 'FILE_MANAGER'); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); WHEN ENV_MANAGER.ERR_WORKFLOW_KEY_NULL THEN vgMsgTmp := ENV_MANAGER.MSG_WORKFLOW_KEY_NULL; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WORKFLOW_KEY_NULL, vgMsgTmp); WHEN ENV_MANAGER.ERR_MULTIPLE_WORKFLOW_KEYS THEN vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_WORKFLOW_KEYS; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => 'VALIDATION_FAILED'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_WORKFLOW_KEYS, vgMsgTmp); WHEN OTHERS THEN IF SQLCODE = -20404 THEN vgMsgTmp := ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD||cgBL||SQLERRM; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, vgMsgTmp); ELSIF SQLCODE = -20003 THEN execute immediate 'select LISTAGG(record, '''||cgBL||''') from (select * from '||REGEXP_SUBSTR(SQLERRM, '"([^"]+)"."([^"]+)"')||' order by rownum desc) where rownum <=2' into vgMsgTmp; vgMsgTmp := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||vgMsgTmp; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, vgMsgTmp); -- ELSIF SQLCODE = -20000 THEN -- TO_DO Add additional info about current config -- ENV_MANAGER.MSG_FILE_VALIDATION_FAILED := ENV_MANAGER.MSG_FILE_VALIDATION_FAILED||cgBL||SQLERRM||cgBL||FILE_MANAGER.OUTPUT_SOURCE_FILE_CONFIG_INFO( ..config key value.. ); -- dbms_output.put_line(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); -- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT, 'ERROR', vParameters); -- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); -- RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); ELSE -- Log complete error details including full stack trace and backtrace ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_UNKNOWN, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END IF; END VALIDATE_SOURCE_FILE_RECEIVED; ---------------------------------------------------------------------------------------------------- FUNCTION VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) RETURN VARCHAR2 -- -- wrapper for DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE -- IS vOperationId NUMBER := -1; vBadfileTable USER_LOAD_OPERATIONS.BADFILE_TABLE%TYPE; vLogfileTable USER_LOAD_OPERATIONS.LOGFILE_TABLE%TYPE; vStatus USER_LOAD_OPERATIONS.STATUS%TYPE; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vDetails clob; TYPE TCURSOR is REF CURSOR; vCursor TCURSOR; vQuery VARCHAR2(1000); vRecord VARCHAR2(10000); BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTableName => '''||nvl(pTableName, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); DBMS_CLOUD.VALIDATE_EXTERNAL_TABLE(pTableName, vOperationId); ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of validation: '||vOperationId,'DEBUG', vParameters); IF vOperationId = -1 THEN RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); END IF; SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE INTO vStatus, vLogfileTable, vBadfileTable FROM USER_LOAD_OPERATIONS WHERE ID = vOperationId; RETURN vStatus; EXCEPTION WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); WHEN OTHERS THEN SELECT decode(STATUS, 'COMPLETED', 'PASSED', STATUS), LOGFILE_TABLE, BADFILE_TABLE INTO vStatus, vLogfileTable, vBadfileTable FROM USER_LOAD_OPERATIONS WHERE ID = vOperationId; vQuery := 'select record from ( select nvl(l.record,''----------------------------------------------------'') as record ,rownum as lp ,max(case when nvl(instr(l.record, ''error'' ),0) > 0 then rownum else 0 end) over (partition by 1) as ENV_MANAGER.ERR_row from '||vLogfileTable||' l ) where lp >= ENV_MANAGER.ERR_row order by rownum'; vDetails := vStatus||cgBL||'----------------------------------------------------'||cgBL; OPEN vCursor for vQuery; loop fetch vCursor into vRecord; EXIT WHEN vCursor%NOTFOUND; vDetails := vDetails ||vRecord ||cgBL; -- for i in loop -- vDetails := vDetails ||i.record ||cgBL; end loop; CLOSE vCursor; vDetails := vDetails||'More details can be found in below tables:'||cgBL|| ' SELECT * FROM USER_LOAD_OPERATIONS WHERE ID = '||vOperationId||';'||cgBL|| ' SELECT * FROM '||vLogfileTable||';'||cgBL|| ' SELECT * FROM '||vBadfileTable||';' ; RETURN vDetails; END VALIDATE_EXTERNAL_TABLE; ---------------------------------------------------------------------------------------------------- FUNCTION S_VALIDATE_EXTERNAL_TABLE(pTableName IN VARCHAR2) RETURN VARCHAR2 -- -- Simple -- IS vCount PLS_INTEGER; BEGIN execute immediate 'select count(1) from '||pTableName into vCount; IF vCount >= 0 THEN RETURN 'PASSED'; END IF; RETURN 'FAILED'; EXCEPTION WHEN OTHERS THEN RETURN 'FAILED'; END S_VALIDATE_EXTERNAL_TABLE; ---------------------------------------------------------------------------------------------------- PROCEDURE DROP_EXTERNAL_TABLE(pSourceFileReceivedKey IN NUMBER) -- -- Drop external table created to validate the file structure -- IS vSourceFileReceived tSourceFileReceived; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileReceived := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); EXECUTE IMMEDIATE 'DROP TABLE '||vSourceFileReceived.EXTERNAL_TABLE_NAME; ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END DROP_EXTERNAL_TABLE; ---------------------------------------------------------------------------------------------------- PROCEDURE COPY_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) -- -- Possible pDestination values are: 'ODS' or 'ARCHIVE' -- IS vSourceFileReceivedInfo tSourceFileReceived; vSourceObject VARCHAR2(2000); vTargetObject VARCHAR2(2000); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; -- vStatus VARCHAR2(20); BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); IF pDestination = 'ODS' THEN vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; ELSE RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); END IF; DBMS_CLOUD.COPY_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, source_object_uri => vSourceObject, target_object_uri => vTargetObject, target_credential_name => ENV_MANAGER.gvCredentialName ); ENV_MANAGER.LOG_PROCESS_EVENT('File copied to '||pDestination||' target location','DEBUG', vTargetObject); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END COPY_FILE; ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- PROCEDURE MOVE_FILE(pSourceFileReceivedKey IN NUMBER, pDestination IN VARCHAR2) -- -- Possible pDestination values are: 'ODS' or 'ARCHIVE' -- IS vSourceFileReceivedInfo tSourceFileReceived; vSourceObject VARCHAR2(2000); vTargetObject VARCHAR2(2000); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vStatus VARCHAR2(20); BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedKey => ' ||nvl(to_char(pSourceFileReceivedKey), 'NULL'), 'pDestination => '''||nvl(pDestination, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); vSourceFileReceivedInfo := GET_SOURCE_FILE_RECEIVED_INFO(pSourceFileReceivedKey); IF pDestination = 'ODS' THEN vSourceObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; vTargetObject := ENV_MANAGER.gvDataBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_ODS ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; vStatus := 'READY_FOR_INGESTION'; ELSIF pDestination = 'QUARANTINE' THEN vSourceObject := ENV_MANAGER.gvInboxBucketUri ||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_INBOX ||vSourceFileReceivedInfo.SOURCE_FILE_NAME; vTargetObject := ENV_MANAGER.gvInboxBucketUri||vSourceFileReceivedInfo.SOURCE_FILE_PREFIX_QUARANTINE||vSourceFileReceivedInfo.SOURCE_FILE_NAME; vStatus := 'VALIDATION_FAILED'; ELSE RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); END IF; DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, source_object_uri => vSourceObject, target_object_uri => vTargetObject, target_credential_name => ENV_MANAGER.gvCredentialName ); ENV_MANAGER.LOG_PROCESS_EVENT('File moved to '||pDestination||' target location','DEBUG', vTargetObject); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => pSourceFileReceivedKey, pStatus => vStatus); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM := ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM ||cgBL||' '||'Possible parameters are: ''ODS'' or ''ARCHIVE''' ||cgBL||' '||'Provided destination parameter: '''||pDestination||''''; ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END MOVE_FILE; ---------------------------------------------------------------------------------------------------- PROCEDURE DELETE_FOLDER_CONTENTS(pBucketArea IN VARCHAR2, pFolderPrefix IN VARCHAR2) -- -- Delete all files from specified folder in cloud storage -- pBucketArea: 'INBOX', 'DATA', 'ARCHIVE' -- pFolderPrefix: folder path within bucket (e.g., 'C2D/UC_DISSEM/UC_NMA_DISSEM/') -- IS vBucketUri VARCHAR2(2000); vFolderUri VARCHAR2(2000); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vFilesDeleted PLS_INTEGER := 0; vObjectName VARCHAR2(4000); vFullObjectUri VARCHAR2(4000); -- Cursor to list all objects in the folder CURSOR c_objects IS SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => ENV_MANAGER.gvCredentialName, location_uri => vBucketUri )) WHERE object_name IS NOT NULL AND object_name LIKE pFolderPrefix || '%'; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pBucketArea => '''||nvl(pBucketArea, 'NULL')||'''', 'pFolderPrefix => '''||nvl(pFolderPrefix, 'NULL')||'''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); -- Get bucket URI based on bucket area vBucketUri := GET_BUCKET_URI(pBucketArea); ENV_MANAGER.LOG_PROCESS_EVENT('Listing objects in bucket with prefix: ' || pFolderPrefix, 'DEBUG', vBucketUri); -- List and delete all objects in the folder FOR obj_rec IN c_objects LOOP vObjectName := obj_rec.object_name; vFullObjectUri := vBucketUri || vObjectName; BEGIN ENV_MANAGER.LOG_PROCESS_EVENT('Deleting object', 'DEBUG', vFullObjectUri); DBMS_CLOUD.DELETE_OBJECT( credential_name => ENV_MANAGER.gvCredentialName, object_uri => vFullObjectUri ); vFilesDeleted := vFilesDeleted + 1; ENV_MANAGER.LOG_PROCESS_EVENT('Object deleted successfully', 'DEBUG', vObjectName); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT('Error deleting object: ' || vObjectName || ' - ' || SQLERRM, 'ERROR', vParameters); -- Continue with next file instead of stopping the whole process END; END LOOP; ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted: ' || vFilesDeleted, 'INFO', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT('Error in DELETE_FOLDER_CONTENTS: ' || SQLERRM, 'ERROR', vParameters); ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END DELETE_FOLDER_CONTENTS; ---------------------------------------------------------------------------------------------------- PROCEDURE PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) -- -- Ubmrella procedure that calls -- - REGISTER_SOURCE_FILE_RECEIVED -- - CREATE_EXTERNAL_TABLE -- - VALIDATE_SOURCE_FILE_RECEIVED -- - DROP_EXTERNAL_TABLE -- - MOVE_FILE IS vSourceFileId NUMBER; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ---- vSourceFileId := REGISTER_SOURCE_FILE_RECEIVED(pSourceFileReceivedName); CREATE_EXTERNAL_TABLE(vSourceFileId); VALIDATE_SOURCE_FILE_RECEIVED(vSourceFileId); DROP_EXTERNAL_TABLE(vSourceFileId); -- COPY_FILE(vSourceFileId, 'ODS'); MOVE_FILE(vSourceFileId, 'ODS'); SET_SOURCE_FILE_RECEIVED_STATUS(pSourceFileReceivedKey => vSourceFileId, pStatus => 'READY_FOR_INGESTION'); ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION -- -20001 WHEN ENV_MANAGER.ERR_EMPTY_FILEURI_AND_RECKEY THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EMPTY_FILEURI_AND_RECKEY, ENV_MANAGER.MSG_EMPTY_FILEURI_AND_RECKEY); -- -20002 WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH_FOR_FILEURI THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH_FOR_FILEURI, ENV_MANAGER.MSG_NO_CONFIG_MATCH_FOR_FILEURI); -- -20003 WHEN ENV_MANAGER.ERR_MULTIPLE_MATCH_FOR_SRCFILE THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE); -- -20004 WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT); -- -20005 WHEN ENV_MANAGER.ERR_MULTIPLE_COLUMN_DATE_FORMAT THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); -- -20006 WHEN ENV_MANAGER.ERR_DIDNT_GET_LOAD_OPERATION_ID THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DIDNT_GET_LOAD_OPERATION_ID, ENV_MANAGER.MSG_DIDNT_GET_LOAD_OPERATION_ID); -- -20007 WHEN ENV_MANAGER.ERR_NO_CONFIG_FOR_RECEIVED_FILE THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_NO_CONFIG_FOR_RECEIVED_FILE); -- -20008 WHEN ENV_MANAGER.ERR_MULTI_CONFIG_FOR_RECEIVED_FILE THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTI_CONFIG_FOR_RECEIVED_FILE, ENV_MANAGER.MSG_MULTI_CONFIG_FOR_RECEIVED_FILE); -- -20009 WHEN ENV_MANAGER.ERR_FILE_NOT_FOUND_ON_CLOUD THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_FOUND_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_FOUND_ON_CLOUD); -- -20010 WHEN ENV_MANAGER.ERR_FILE_VALIDATION_FAILED THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_VALIDATION_FAILED), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_VALIDATION_FAILED, ENV_MANAGER.MSG_FILE_VALIDATION_FAILED); -- -20011 WHEN ENV_MANAGER.ERR_NO_CONFIG_MATCH THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NO_CONFIG_MATCH), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NO_CONFIG_MATCH, ENV_MANAGER.MSG_NO_CONFIG_MATCH); -- -20012 WHEN ENV_MANAGER.ERR_UNKNOWN_PREFIX THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN_PREFIX), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN_PREFIX, ENV_MANAGER.MSG_UNKNOWN_PREFIX); -- -20013 WHEN ENV_MANAGER.ERR_TABLE_NOT_EXISTS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_TABLE_NOT_EXISTS), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_TABLE_NOT_EXISTS, ENV_MANAGER.MSG_TABLE_NOT_EXISTS); -- -20014 WHEN ENV_MANAGER.ERR_COLUMN_NOT_EXISTS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_COLUMN_NOT_EXISTS), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_COLUMN_NOT_EXISTS, ENV_MANAGER.MSG_COLUMN_NOT_EXISTS); -- -20015 WHEN ENV_MANAGER.ERR_UNSUPPORTED_DATA_TYPE THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNSUPPORTED_DATA_TYPE, ENV_MANAGER.MSG_UNSUPPORTED_DATA_TYPE); -- -20016 WHEN ENV_MANAGER.ERR_MISSING_SOURCE_KEY THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_SOURCE_KEY), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); -- -20017 WHEN ENV_MANAGER.ERR_NULL_SOURCE_FILE_CONFIG_KEY THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NULL_SOURCE_FILE_CONFIG_KEY, ENV_MANAGER.MSG_NULL_SOURCE_FILE_CONFIG_KEY); -- -20018 WHEN ENV_MANAGER.ERR_DUPLICATED_SOURCE_KEY THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); -- -20019 WHEN ENV_MANAGER.ERR_MISSING_CONTAINER_CONFIG THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_CONTAINER_CONFIG, ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG); -- -20020 WHEN ENV_MANAGER.ERR_MULTIPLE_CONTAINER_ENTRIES THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); -- -20021 WHEN ENV_MANAGER.ERR_WRONG_DESTINATION_PARAM THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DESTINATION_PARAM, ENV_MANAGER.MSG_WRONG_DESTINATION_PARAM); -- -20022 WHEN ENV_MANAGER.ERR_FILE_NOT_EXISTS_ON_CLOUD THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_NOT_EXISTS_ON_CLOUD, ENV_MANAGER.MSG_FILE_NOT_EXISTS_ON_CLOUD); -- -20023 WHEN ENV_MANAGER.ERR_FILE_ALREADY_REGISTERED THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_FILE_ALREADY_REGISTERED, ENV_MANAGER.MSG_FILE_ALREADY_REGISTERED); -- -20024 WHEN ENV_MANAGER.ERR_WRONG_DATE_TIMESTAMP_FORMAT THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_WRONG_DATE_TIMESTAMP_FORMAT, ENV_MANAGER.MSG_WRONG_DATE_TIMESTAMP_FORMAT); -- -20011 WHEN ENV_MANAGER.ERR_EXCESS_COLUMNS_DETECTED THEN ENV_MANAGER.LOG_PROCESS_ERROR(ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED, vParameters, 'FILE_MANAGER'); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXCESS_COLUMNS_DETECTED, ENV_MANAGER.MSG_EXCESS_COLUMNS_DETECTED); -- -20999 WHEN ENV_MANAGER.ERR_UNKNOWN THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> ENV_MANAGER.CODE_UNKNOWN), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END PROCESS_SOURCE_FILE; ---------------------------------------------------------------------------------------------------- FUNCTION PROCESS_SOURCE_FILE(pSourceFileReceivedName IN VARCHAR2) RETURN PLS_INTEGER IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileReceivedName => '||nvl(pSourceFileReceivedName, 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); ---- PROCESS_SOURCE_FILE(pSourceFileReceivedName => pSourceFileReceivedName); ---- ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); RETURN SQLCODE; EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RETURN SQLCODE; END PROCESS_SOURCE_FILE; ---------------------------------------------------------------------------------------------------- FUNCTION GET_DATE_FORMAT( pTemplateTableName IN VARCHAR2, pColumnName IN VARCHAR2 ) RETURN VARCHAR2 IS vDateFormat A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vColumnName VARCHAR2(200); vGetDefault BOOLEAN := FALSE; BEGIN vColumnName := trim(pColumnName); vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pColumnName => '''||nvl(vColumnName, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); BEGIN SELECT DATE_FORMAT INTO vDateFormat FROM CT_MRDS.A_COLUMN_DATE_FORMAT F WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName AND F.COLUMN_NAME = vColumnName; EXCEPTION WHEN NO_DATA_FOUND THEN vGetDefault := TRUE; WHEN TOO_MANY_ROWS THEN -- Below error should not happened because: -- Unique constraint added on table A_COLUMN_DATE_FORMAT on columns: (TEMPLATE_TABLE_NAME, COLUMN_NAME) RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_COLUMN_DATE_FORMAT, ENV_MANAGER.MSG_MULTIPLE_COLUMN_DATE_FORMAT); END; IF vGetDefault THEN BEGIN SELECT DATE_FORMAT INTO vDateFormat FROM CT_MRDS.A_COLUMN_DATE_FORMAT F WHERE F.TEMPLATE_TABLE_NAME = pTemplateTableName AND F.COLUMN_NAME = 'DEFAULT'; EXCEPTION WHEN NO_DATA_FOUND THEN vDateFormat := ENV_MANAGER.gvDefaultDateFormat; END; END IF; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); RETURN vDateFormat; END GET_DATE_FORMAT; ---------------------------------------------------------------------------------------------------- PROCEDURE GENERATE_EXTERNAL_TABLE_PARAMS ( pTemplateTableName IN VARCHAR2, pColumnList OUT CLOB, pFieldList OUT CLOB ) IS vSchemaName VARCHAR2(200); vTableName VARCHAR2(200); vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vMaxColumnNameLength PLS_INTEGER := 0; vColType varchar2(200); BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pColumnList => '''||nvl(pColumnList, 'NULL')||'''' ,'pFieldList = '''||nvl(pFieldList, 'NULL')||'''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); vSchemaName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'.*\.'),'.',''); ENV_MANAGER.LOG_PROCESS_EVENT('vSchemaName','DEBUG', vSchemaName); vTableName := REPLACE(REGEXP_SUBSTR(pTemplateTableName,'\..*'),'.',''); ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG', vTableName); FOR rec IN ( SELECT '"'||column_name||'"' as quoted_column_name, column_name, data_type, data_length, char_length, -- MARS-1056: Added for CHAR/BYTE semantics char_used, -- MARS-1056: Added for CHAR/BYTE semantics data_precision, data_scale, column_id, max(length(column_name)+1) over (partition by table_name) as max_column_name_length FROM all_tab_columns WHERE table_name = UPPER(vTableName) AND owner = NVL(UPPER(vSchemaName), USER) ORDER BY column_id ) LOOP -- Build the column_list string rec.quoted_column_name := rpad(rec.quoted_column_name, rec.max_column_name_length+2, ' '); vColType := CASE -- MARS-1056: Fixed VARCHAR2 definition logic to preserve CHAR/BYTE semantics WHEN rec.data_type = 'VARCHAR2' THEN CASE WHEN rec.char_used = 'C' THEN rec.quoted_column_name || ' VARCHAR2(' || rec.char_length || ' CHAR)' WHEN rec.char_used = 'B' THEN rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ' BYTE)' ELSE -- Fallback for NULL char_used (should not occur but handle gracefully) rec.quoted_column_name || ' VARCHAR2(' || rec.data_length || ')' END -- Other character types (preserve original logic) -- MARS-1468: Fixed CHAR to use char_used/char_length (same as VARCHAR2 fix in MARS-1056) WHEN rec.data_type = 'CHAR' THEN CASE WHEN rec.char_used = 'C' THEN rec.quoted_column_name || ' CHAR(' || rec.char_length || ' CHAR)' WHEN rec.char_used = 'B' THEN rec.quoted_column_name || ' CHAR(' || rec.data_length || ' BYTE)' ELSE rec.quoted_column_name || ' CHAR(' || rec.data_length || ')' END -- MARS-1468: NCHAR/NVARCHAR2 - use char_length (data_length stores bytes in AL16UTF16, e.g. NCHAR(1) => data_length=2 but char_length=1) WHEN rec.data_type IN ('NCHAR', 'NVARCHAR2') THEN rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.char_length || ')' WHEN rec.data_type = 'NUMBER' THEN rec.quoted_column_name || ' ' || rec.data_type || CASE WHEN rec.data_precision IS NOT NULL AND rec.data_scale IS NOT NULL THEN '(' || rec.data_precision || ',' || rec.data_scale || ')' WHEN rec.data_precision IS NOT NULL THEN '(' || rec.data_precision || ')' ELSE '' END WHEN rec.data_type IN ('RAW') THEN rec.quoted_column_name || ' ' || rec.data_type || '(' || rec.data_length || ')' WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') IN ('DATE', 'TIMESTAMP') THEN rec.quoted_column_name || ' ' || rec.data_type ELSE rec.quoted_column_name || ' ' || rec.data_type END; pColumnList := pColumnList ||vColType ||cgBL|| ','; -- Build the field_list string -- Note: field_list uses CHAR() for CSV field definitions - this is correct behavior pFieldList := pFieldList || CASE WHEN rec.data_type = 'DATE' THEN -- MARS-1046: DATE format - wrap with NORMALIZE_DATE_FORMAT to fix ISO 8601 'T' separator rec.quoted_column_name || ' DATE ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) WHEN rec.data_type LIKE 'TIMESTAMP%WITH TIME ZONE' THEN -- MARS-1046: TIMESTAMP WITH TIME ZONE format for ISO 8601 with fractional seconds and timezone -- Syntax: column_name CHAR(length) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK "format" -- Use fixed length of 50 for ISO 8601 format (e.g., "2012-03-02T14:16:23.798+01:00" = 29 chars) rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN -- Other TIMESTAMP types (without timezone) -- SQL*Loader syntax: CHAR(length) DATE_FORMAT TIMESTAMP MASK "format" (not: TIMESTAMP 'format') rec.quoted_column_name || ' CHAR(35) DATE_FORMAT TIMESTAMP MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39) WHEN rec.data_type IN ('VARCHAR2', 'CHAR') THEN -- MARS-1468: For CHAR use char_length when char semantics (C), otherwise data_length rec.quoted_column_name || ' CHAR(' || CASE WHEN rec.char_used = 'C' THEN rec.char_length ELSE rec.data_length END || ')' WHEN rec.data_type IN ('NCHAR', 'NVARCHAR2') THEN -- For CSV field definitions, use data_length for NCHAR/NVARCHAR2 rec.quoted_column_name || ' CHAR(' || rec.data_length || ')' ELSE rec.quoted_column_name END ||cgBL|| ','; END LOOP; -- Remove the trailing comma and space from the strings pColumnList := ' '||RTRIM(pColumnList, ','); pFieldList := ' '||RTRIM(pFieldList, ','); ENV_MANAGER.LOG_PROCESS_EVENT('vColumnList', 'DEBUG', pColumnList); -- TO_DO !!! -- Add check if pColumnList/pFieldList is empty or not -- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -- Output the generated column_list and field_list ENV_MANAGER.LOG_PROCESS_EVENT('column_list' ,'DEBUG',pColumnList); ENV_MANAGER.LOG_PROCESS_EVENT('field_list' ,'DEBUG',pFieldList); ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); EXCEPTION -- WHEN ENV_MANAGER.ERR_MISSING_COLUMN_DATE_FORMAT THEN -- ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_COLUMN_DATE_FORMAT, 'ERROR', vParameters); -- RAISE_ERROR(ENV_MANAGER.CODE_MISSING_COLUMN_DATE_FORMAT); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END GENERATE_EXTERNAL_TABLE_PARAMS; ---------------------------------------------------------------------------------------------------- PROCEDURE ADD_SOURCE ( pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE, pSourceName IN CT_MRDS.A_SOURCE.SOURCE_NAME%TYPE ) IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN INSERT INTO CT_MRDS.A_SOURCE(A_SOURCE_KEY, SOURCE_NAME) VALUES (pSourceKey, pSourceName); COMMIT; EXCEPTION WHEN DUP_VAL_ON_INDEX THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DUPLICATED_SOURCE_KEY, ENV_MANAGER.MSG_DUPLICATED_SOURCE_KEY); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END ADD_SOURCE; ---------------------------------------------------------------------------------------------------- PROCEDURE DELETE_SOURCE_CASCADE ( pSourceKey IN CT_MRDS.A_SOURCE.A_SOURCE_KEY%TYPE ) IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; vSharedTemplateCount PLS_INTEGER := 0; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceKey => '''||nvl(pSourceKey, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); -- First pass: Delete container files (those that have CONTAINER_FILE_KEY set) for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NOT NULL ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop -- Delete processed file records delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for container config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN SELECT COUNT(DISTINCT A_SOURCE_KEY) INTO vSharedTemplateCount FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted -- Only delete date formats if template table is not shared with other sources IF vSharedTemplateCount = 0 THEN delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); ELSE ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); END IF; END IF; -- Delete container file configuration delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted container A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); end loop; COMMIT; -- Commit container deletions -- Second pass: Delete parent files (those that do NOT have CONTAINER_FILE_KEY set) for rec in (select A_SOURCE_FILE_CONFIG_KEY, TEMPLATE_TABLE_NAME from CT_MRDS.A_SOURCE_FILE_CONFIG where A_SOURCE_KEY = pSourceKey AND CONTAINER_FILE_KEY IS NULL ORDER BY A_SOURCE_FILE_CONFIG_KEY) loop -- Delete processed file records delete from CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE_FILE_RECEIVED records for parent config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); -- Check if TEMPLATE_TABLE_NAME is shared with other source systems before deleting date formats IF rec.TEMPLATE_TABLE_NAME IS NOT NULL THEN SELECT COUNT(DISTINCT A_SOURCE_KEY) INTO vSharedTemplateCount FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME AND A_SOURCE_KEY <> pSourceKey; -- Exclude current source being deleted -- Only delete date formats if template table is not shared with other sources IF vSharedTemplateCount = 0 THEN delete from CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = rec.TEMPLATE_TABLE_NAME; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_COLUMN_DATE_FORMAT records for template: '||rec.TEMPLATE_TABLE_NAME,'DEBUG', vParameters); ELSE ENV_MANAGER.LOG_PROCESS_EVENT('Skipping A_COLUMN_DATE_FORMAT deletion - template table '||rec.TEMPLATE_TABLE_NAME||' is shared with '||vSharedTemplateCount||' other source systems','WARNING', vParameters); END IF; END IF; -- Delete parent file configuration delete from CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = rec.A_SOURCE_FILE_CONFIG_KEY; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted parent A_SOURCE_FILE_CONFIG record for config key: '||rec.A_SOURCE_FILE_CONFIG_KEY,'DEBUG', vParameters); end loop; COMMIT; -- Commit parent deletions -- Delete source system record DELETE FROM CT_MRDS.A_SOURCE where A_SOURCE_KEY = pSourceKey; ENV_MANAGER.LOG_PROCESS_EVENT('Deleted A_SOURCE record for source key: '||pSourceKey,'DEBUG', vParameters); COMMIT; -- Final commit for source deletion ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END DELETE_SOURCE_CASCADE; ---------------------------------------------------------------------------------------------------- FUNCTION GET_CONTAINER_SOURCE_FILE_CONFIG_KEY ( pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE ) RETURN PLS_INTEGER IS vSourceFileConfigKey PLS_INTEGER; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileId => '||nvl(to_char(pSourceFileId), 'NULL'))); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); SELECT A_SOURCE_FILE_CONFIG_KEY INTO vSourceFileConfigKey FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_ID = pSourceFileId AND SOURCE_FILE_TYPE = 'CONTAINER'; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); RETURN vSourceFileConfigKey; EXCEPTION WHEN NO_DATA_FOUND THEN ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG := 'No match in A_SOURCE_FILE_CONFIG where SOURCE_FILE_TYPE=''CONTAINER'' and SOURCE_FILE_ID = '''||pSourceFileId||''''; ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_CONTAINER_CONFIG, 'WARNING', vParameters); RETURN NULL; WHEN TOO_MANY_ROWS THEN ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES := 'GET_CONTAINER_SOURCE_FILE_CONFIG_KEY: Multiple SOURCE_FILE_TYPE=''CONTAINER'' matches for SOURCE_FILE_ID: '||pSourceFileId||' in A_SOURCE_FILE_CONFIG'; ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_CONTAINER_ENTRIES, ENV_MANAGER.MSG_MULTIPLE_CONTAINER_ENTRIES); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END GET_CONTAINER_SOURCE_FILE_CONFIG_KEY; ---------------------------------------------------------------------------------------------------- FUNCTION GET_SOURCE_FILE_CONFIG_KEY ( pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE DEFAULT 'INPUT' ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE ) RETURN PLS_INTEGER IS vSourceFileConfigKey PLS_INTEGER; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileType => '''||nvl(pSourceFileType, 'NULL')||'''' ,'pSourceFileId => '''||nvl(pSourceFileId, 'NULL')||'''' ,'pTableId => '''||nvl(pTableId, 'NULL')||'''')); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); SELECT A_SOURCE_FILE_CONFIG_KEY INTO vSourceFileConfigKey FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE SOURCE_FILE_TYPE = pSourceFileType AND SOURCE_FILE_ID = pSourceFileId AND TABLE_ID = pTableId; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG', vParameters); RETURN vSourceFileConfigKey; EXCEPTION WHEN NO_DATA_FOUND THEN RETURN NULL; WHEN TOO_MANY_ROWS THEN vgMsgTmp := ENV_MANAGER.MSG_MULTIPLE_MATCH_FOR_SRCFILE ||cgBL||' '||'GET_SOURCE_FILE_CONFIG_KEY: Multiple matches in A_SOURCE_FILE_CONFIG table WHERE' ||cgBL||' '||'SOURCE_FILE_TYPE: '||pSourceFileType ||cgBL||' '||'SOURCE_FILE_ID: '||pSourceFileId ||cgBL||' '||'TABLE_ID: '||pTableId; ENV_MANAGER.LOG_PROCESS_EVENT(vgMsgTmp, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MULTIPLE_MATCH_FOR_SRCFILE, vgMsgTmp); WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.MSG_UNKNOWN); END GET_SOURCE_FILE_CONFIG_KEY; ---------------------------------------------------------------------------------------------------- PROCEDURE ADD_SOURCE_FILE_CONFIG ( pSourceKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY%TYPE ,pSourceFileType IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE ,pSourceFileId IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID%TYPE ,pSourceFileDesc IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC%TYPE ,pSourceFileNamePattern IN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN%TYPE ,pTableId IN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID%TYPE DEFAULT NULL ,pTemplateTableName IN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME%TYPE DEFAULT NULL ,pContainerFileKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY%TYPE DEFAULT NULL ,pEncoding IN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING%TYPE DEFAULT NULL -- MARS-1049 ,pIsWorkflowSuccessRequired IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_WORKFLOW_SUCCESS_REQUIRED%TYPE DEFAULT 'Y' -- MARS-1409 ,pIsArchiveEnabled IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED%TYPE DEFAULT 'N' -- MARS-828 ,pIsKeepInTrash IN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEPT_IN_TRASH%TYPE DEFAULT 'Y' -- MARS-828 ,pArchivalStrategy IN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY%TYPE DEFAULT 'THRESHOLD_BASED' -- MARS-828 ,pMinimumAgeMonths IN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS%TYPE DEFAULT 0 -- MARS-828 ) IS vSourceFileConfigKey PLS_INTEGER; vSourceKeyExists PLS_INTEGER := 0; vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceKey => '''||nvl(to_char(pSourceKey), 'NULL')||'''' ,'pSourceFileType => '''||nvl(to_char(pSourceFileType), 'NULL')||'''' ,'pSourceFileId => '''||nvl(to_char(pSourceFileId), 'NULL')||'''' ,'pSourceFileDesc => '''||nvl(to_char(pSourceFileDesc), 'NULL')||'''' ,'pSourceFileNamePattern => '''||nvl(to_char(pSourceFileNamePattern), 'NULL')||'''' ,'pTableId => '''||nvl(to_char(pTableId), 'NULL')||'''' ,'pTemplateTableName => '''||nvl(to_char(pTemplateTableName), 'NULL')||'''' ,'pContainerFileKey => '''||nvl(to_char(pContainerFileKey), 'NULL')||'''' ,'pEncoding => '''||nvl(to_char(pEncoding), 'NULL')||'''' -- MARS-1049: NOWY ,'pIsWorkflowSuccessRequired => '''||nvl(to_char(pIsWorkflowSuccessRequired), 'NULL')||'''' -- MARS-1409 ,'pIsArchiveEnabled => '''||nvl(to_char(pIsArchiveEnabled), 'NULL')||'''' -- MARS-828 ,'pIsKeepInTrash => '''||nvl(to_char(pIsKeepInTrash), 'NULL')||'''' -- MARS-828 ,'pArchivalStrategy => '''||nvl(to_char(pArchivalStrategy), 'NULL')||'''' -- MARS-828 ,'pMinimumAgeMonths => '''||nvl(to_char(pMinimumAgeMonths), 'NULL')||'''' -- MARS-828 )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); INSERT INTO CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_KEY, SOURCE_FILE_TYPE, SOURCE_FILE_ID, SOURCE_FILE_DESC, SOURCE_FILE_NAME_PATTERN, TABLE_ID, TEMPLATE_TABLE_NAME, CONTAINER_FILE_KEY, ENCODING, IS_WORKFLOW_SUCCESS_REQUIRED, IS_ARCHIVE_ENABLED, IS_KEPT_IN_TRASH, ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS) VALUES (pSourceKey, pSourceFileType, pSourceFileId, pSourceFileDesc, pSourceFileNamePattern, pTableId, pTemplateTableName, pContainerFileKey, pEncoding, pIsWorkflowSuccessRequired, pIsArchiveEnabled, pIsKeepInTrash, pArchivalStrategy, pMinimumAgeMonths); COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); EXCEPTION WHEN OTHERS THEN IF SQLCODE = -2291 THEN ENV_MANAGER.MSG_MISSING_SOURCE_KEY := 'The Source with A_SOURCE_KEY: '''||pSourceKey||''' not found in parent table A_SOURCE.' ||cgBL||'First add a record to A_SOURCE:' ||cgBL||'call file_manager.add_source(pSourceKey => '''||pSourceKey||''', pSourceName => ''...'')'; ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_MISSING_SOURCE_KEY, 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MISSING_SOURCE_KEY, ENV_MANAGER.MSG_MISSING_SOURCE_KEY); ELSE ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END IF; END ADD_SOURCE_FILE_CONFIG; ---------------------------------------------------------------------------------------------------- PROCEDURE ADD_COLUMN_DATE_FORMAT ( pTemplateTableName IN CT_MRDS.A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE ,pColumnName IN CT_MRDS.A_COLUMN_DATE_FORMAT.COLUMN_NAME%TYPE ,pDateFormat IN CT_MRDS.A_COLUMN_DATE_FORMAT.DATE_FORMAT%TYPE ) IS vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||'''' ,'pColumnName => '''||nvl(pColumnName, 'NULL')||'''' ,'pDateFormat => '''||nvl(pDateFormat, 'NULL')||'''' )); ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters); INSERT INTO CT_MRDS.A_COLUMN_DATE_FORMAT(TEMPLATE_TABLE_NAME, COLUMN_NAME, DATE_FORMAT) VALUES (pTemplateTableName, pColumnName, pDateFormat); COMMIT; ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters); EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END ADD_COLUMN_DATE_FORMAT; ---------------------------------------------------------------------------------------------------- FUNCTION GET_BUCKET_URI(pBucketArea VARCHAR2) RETURN VARCHAR2 IS BEGIN CASE pBucketArea WHEN 'INBOX' THEN RETURN ENV_MANAGER.gvInboxBucketUri; WHEN 'ODS' THEN RETURN ENV_MANAGER.gvDataBucketUri; WHEN 'DATA' THEN RETURN ENV_MANAGER.gvDataBucketUri; WHEN 'ARCHIVE' THEN RETURN ENV_MANAGER.gvArchiveBucketUri; ELSE RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_INVALID_BUCKET_AREA, ENV_MANAGER.MSG_INVALID_BUCKET_AREA || ' Provided: ''' || pBucketArea || ''''); END CASE; END; ---------------------------------------------------------------------------------------------------- FUNCTION GET_DET_SOURCE_FILE_CONFIG_INFO ( pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 ) RETURN VARCHAR2 ---- -- Function to get info about File Configuration entry IS vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vContainerFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vCount PLS_INTEGER := 0; addColHeader BOOLEAN := TRUE; vMsgTmp VARCHAR2(32000):= ''; CURSOR cColumnFormat(vTableName A_COLUMN_DATE_FORMAT.TEMPLATE_TABLE_NAME%TYPE) IS SELECT * FROM CT_MRDS.A_COLUMN_DATE_FORMAT WHERE TEMPLATE_TABLE_NAME = vTableName; vColumnDateFormat A_COLUMN_DATE_FORMAT%ROWTYPE; FUNCTION FORMAT_CONFIG( pSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE, pTypeInfo VARCHAR2 DEFAULT 'File Configuration', pLevel VARCHAR2 DEFAULT '') RETURN VARCHAR2 IS vMsg VARCHAR2(9999); BEGIN vMsg := '' ||cgBL||pLevel||''||'Details about '||pTypeInfo||':' ||cgBL||pLevel||''||'--------------------------------' ||cgBL||pLevel||'A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfig.A_SOURCE_FILE_CONFIG_KEY ||cgBL||pLevel||'A_SOURCE_KEY = '||pSourceFileConfig.A_SOURCE_KEY ||cgBL||pLevel||'SOURCE_FILE_TYPE = '||pSourceFileConfig.SOURCE_FILE_TYPE ||cgBL||pLevel||'SOURCE_FILE_ID = '||pSourceFileConfig.SOURCE_FILE_ID ||cgBL||pLevel||'SOURCE_FILE_DESC = '||pSourceFileConfig.SOURCE_FILE_DESC ||cgBL||pLevel||'SOURCE_FILE_NAME_PATTERN = '||pSourceFileConfig.SOURCE_FILE_NAME_PATTERN ||cgBL||pLevel||'TABLE_ID = '||pSourceFileConfig.TABLE_ID ||cgBL||pLevel||'TEMPLATE_TABLE_NAME = '||pSourceFileConfig.TEMPLATE_TABLE_NAME ||cgBL||pLevel||'CONTAINER_FILE_KEY = '||pSourceFileConfig.CONTAINER_FILE_KEY ||cgBL||pLevel||'ODS_SCHEMA_NAME = '||pSourceFileConfig.ODS_SCHEMA_NAME ||cgBL||pLevel||'ARCHIVE_THRESHOLD_DAYS = '||pSourceFileConfig.ARCHIVE_THRESHOLD_DAYS ||cgBL||pLevel||'ARCHIVE_THRESHOLD_FILES_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_FILES_COUNT ||cgBL||pLevel||'ARCHIVE_THRESHOLD_BYTES_SUM = '||pSourceFileConfig.ARCHIVE_THRESHOLD_BYTES_SUM ||cgBL||pLevel||'ARCHIVE_THRESHOLD_ROWS_COUNT = '||pSourceFileConfig.ARCHIVE_THRESHOLD_ROWS_COUNT ||cgBL||pLevel||'HOURS_TO_EXPIRE_STATISTICS = '||pSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS ||cgBL||pLevel||'ENCODING = '||pSourceFileConfig.ENCODING ||cgBL||pLevel||'IS_ARCHIVE_ENABLED = '||pSourceFileConfig.IS_ARCHIVE_ENABLED ||cgBL||pLevel||'IS_KEPT_IN_TRASH = '||pSourceFileConfig.IS_KEPT_IN_TRASH ||cgBL||pLevel||'ARCHIVAL_STRATEGY = '||pSourceFileConfig.ARCHIVAL_STRATEGY ||cgBL||pLevel||'MINIMUM_AGE_MONTHS = '||pSourceFileConfig.MINIMUM_AGE_MONTHS ||cgBL||pLevel||'IS_WORKFLOW_SUCCESS_REQUIRED = '||pSourceFileConfig.IS_WORKFLOW_SUCCESS_REQUIRED ||cgBL||pLevel||''||'--------------------------------' ; RETURN vMsg; END FORMAT_CONFIG; BEGIN vMsgTmp := ''; -- Get Main Config BEGIN SELECT * INTO vSourceFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey; vMsgTmp := FORMAT_CONFIG(pSourceFileConfig => vSourceFileConfig, pTypeInfo => 'File Configuration'); EXCEPTION WHEN NO_DATA_FOUND THEN vMsgTmp := 'There is no config entry in A_SOURCE_FILE_CONFIG where A_SOURCE_FILE_CONFIG_KEY = '||pSourceFileConfigKey; RETURN vMsgTmp; END; -- Get Container Config IF pIncludeContainerInfo > 0 THEN BEGIN SELECT * INTO vContainerFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG WHERE A_SOURCE_FILE_CONFIG_KEY = vSourceFileConfig.CONTAINER_FILE_KEY; vMsgTmp := vMsgTmp || cgBL || FORMAT_CONFIG(pSourceFileConfig => vContainerFileConfig, pTypeInfo => 'related Container Config', pLevel => ' '); EXCEPTION WHEN NO_DATA_FOUND THEN vMsgTmp := vMsgTmp|| cgBL || 'There is no CONTAINER config entry in A_SOURCE_FILE_CONFIG.'; -- RETURN vMsgTmp; END; END IF; -- Get Column Date Format Config IF pIncludeColumnFormatInfo > 0 THEN BEGIN OPEN cColumnFormat(vTableName => vSourceFileConfig.TEMPLATE_TABLE_NAME); LOOP FETCH cColumnFormat INTO vColumnDateFormat; IF cColumnFormat%FOUND AND addColHeader THEN vCount := 1; vMsgTmp := vMsgTmp||cgBL|| cgBL || ' Column Date Format config entries:'; vMsgTmp := vMsgTmp||cgBL||''||' --------------------------------'; addColHeader := FALSE; END IF; EXIT WHEN cColumnFormat%NOTFOUND; vMsgTmp := vMsgTmp ||cgBL||' TEMPLATE_TABLE_NAME = '||vColumnDateFormat.TEMPLATE_TABLE_NAME ||cgBL||' COLUMN_NAME = '||vColumnDateFormat.COLUMN_NAME ||cgBL||' DATE_FORMAT = '||vColumnDateFormat.DATE_FORMAT ||cgBL||''||' --------------------------------'; END LOOP; If vCount=0 THEN vMsgTmp := vMsgTmp || cgBL || 'There is no Column Date Format config entries in A_COLUMN_DATE_FORMAT where TEMPLATE_TABLE_NAME = '||NVL(vSourceFileConfig.TEMPLATE_TABLE_NAME,'NULL'); END IF; CLOSE cColumnFormat; END; END IF; RETURN vMsgTmp; EXCEPTION WHEN OTHERS THEN RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END GET_DET_SOURCE_FILE_CONFIG_INFO; ---------------------------------------------------------------------------------------------------- FUNCTION GET_DET_SOURCE_FILE_RECEIVED_INFO ( pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE ,pIncludeConfigInfo IN PLS_INTEGER DEFAULT 1 ,pIncludeContainerInfo IN PLS_INTEGER DEFAULT 1 ,pIncludeColumnFormatInfo IN PLS_INTEGER DEFAULT 1 ) RETURN VARCHAR2 ---- -- Function to get info about File Received entry IS vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; vMsgTmp VARCHAR2(32000):= ''; BEGIN BEGIN SELECT * INTO vSourceFileReceived FROM CT_MRDS.A_SOURCE_FILE_RECEIVED WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; vMsgTmp := '' ||cgBL||''||'Details about received file:' ||cgBL||''||'--------------------------------' ||cgBL||'A_SOURCE_FILE_RECEIVED_KEY = '||vSourceFileReceived.A_SOURCE_FILE_RECEIVED_KEY ||cgBL||'A_SOURCE_FILE_CONFIG_KEY = '||vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY ||cgBL||'SOURCE_FILE_NAME = '||vSourceFileReceived.SOURCE_FILE_NAME ||cgBL||'CHECKSUM = '||vSourceFileReceived.CHECKSUM ||cgBL||'CREATED = '||vSourceFileReceived.CREATED ||cgBL||'BYTES = '||vSourceFileReceived.BYTES ||cgBL||'RECEPTION_DATE = '||vSourceFileReceived.RECEPTION_DATE ||cgBL||'PROCESSING_STATUS = '||vSourceFileReceived.PROCESSING_STATUS ||cgBL||'EXTERNAL_TABLE_NAME = '||vSourceFileReceived.EXTERNAL_TABLE_NAME ||cgBL||'PARTITION_YEAR = '||vSourceFileReceived.PARTITION_YEAR ||cgBL||'PARTITION_MONTH = '||vSourceFileReceived.PARTITION_MONTH ||cgBL||''||'--------------------------------' ; EXCEPTION WHEN NO_DATA_FOUND THEN vMsgTmp := 'There is no data in A_SOURCE_FILE_RECEIVED where A_SOURCE_FILE_RECEIVED_KEY = '||pSourceFileReceivedKey; RETURN vMsgTmp; END; IF pIncludeConfigInfo>0 THEN vMsgTmp := vMsgTmp || cgBL || CT_MRDS.FILE_MANAGER.GET_DET_SOURCE_FILE_CONFIG_INFO(vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY,pIncludeContainerInfo,pIncludeColumnFormatInfo); END IF; RETURN vMsgTmp; EXCEPTION WHEN OTHERS THEN RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END GET_DET_SOURCE_FILE_RECEIVED_INFO; ---------------------------------------------------------------------------------------------------- FUNCTION GET_DET_USER_LOAD_OPERATIONS ( pOperationId PLS_INTEGER ) RETURN VARCHAR2 ---- -- Function to get info about File Received entry IS vUserLoadOperations USER_LOAD_OPERATIONS%ROWTYPE; vMsgTmp VARCHAR2(32000):= ''; BEGIN BEGIN SELECT * INTO vUserLoadOperations FROM USER_LOAD_OPERATIONS WHERE id = pOperationId; vMsgTmp := '' ||''||'Details about USER_LOAD_OPERATIONS where ID = '||pOperationId ||cgBL||''||'--------------------------------' ||cgBL||'ID = '||vUserLoadOperations.ID ||cgBL||'TYPE = '||vUserLoadOperations.TYPE ||cgBL||'SID = '||vUserLoadOperations.SID ||cgBL||'SERIAL# = '||vUserLoadOperations.SERIAL# ||cgBL||'START_TIME = '||vUserLoadOperations.START_TIME ||cgBL||'UPDATE_TIME = '||vUserLoadOperations.UPDATE_TIME ||cgBL||'STATUS = '||vUserLoadOperations.STATUS ||cgBL||'OWNER_NAME = '||vUserLoadOperations.OWNER_NAME ||cgBL||'TABLE_NAME = '||vUserLoadOperations.TABLE_NAME ||cgBL||'PARTITION_NAME = '||vUserLoadOperations.PARTITION_NAME ||cgBL||'SUBPARTITION_NAME = '||vUserLoadOperations.SUBPARTITION_NAME ||cgBL||'FILE_URI_LIST = '||vUserLoadOperations.FILE_URI_LIST ||cgBL||'ROWS_LOADED = '||vUserLoadOperations.ROWS_LOADED ||cgBL||'LOGFILE_TABLE = '||vUserLoadOperations.LOGFILE_TABLE ||cgBL||'BADFILE_TABLE = '||vUserLoadOperations.BADFILE_TABLE ||cgBL||'STATUS_TABLE = '||vUserLoadOperations.STATUS_TABLE ||cgBL||'TEMPEXT_TABLE = '||vUserLoadOperations.TEMPEXT_TABLE ||cgBL||'CREDENTIAL_NAME = '||vUserLoadOperations.CREDENTIAL_NAME ||cgBL||'EXPIRATION_TIME = '||vUserLoadOperations.EXPIRATION_TIME ||cgBL||''||'--------------------------------' ; EXCEPTION WHEN NO_DATA_FOUND THEN vMsgTmp := 'There is no data in USER_LOAD_OPERATIONS where ID = '||pOperationId; RETURN vMsgTmp; END; RETURN vMsgTmp; EXCEPTION WHEN OTHERS THEN RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END GET_DET_USER_LOAD_OPERATIONS; ---------------------------------------------------------------------------------------------------- FUNCTION ANALYZE_VALIDATION_ERRORS( pSourceFileReceivedKey IN NUMBER ) RETURN VARCHAR2 IS vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; vValidationLogTable VARCHAR2(128); vTemplateSchema VARCHAR2(128); vTemplateTable VARCHAR2(128); vCsvFileUri VARCHAR2(4000); vParameters VARCHAR2(4000); vResult VARCHAR2(32000); BEGIN vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( 'pSourceFileReceivedKey => ' || pSourceFileReceivedKey )); ENV_MANAGER.LOG_PROCESS_EVENT('Start', 'DEBUG', vParameters); -- Get file and config information BEGIN -- Get source file received data first SELECT * INTO vSourceFileReceived FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr WHERE sfr.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey; -- Get source file config data SELECT * INTO vSourceFileConfig FROM CT_MRDS.A_SOURCE_FILE_CONFIG sfc WHERE sfc.A_SOURCE_FILE_CONFIG_KEY = vSourceFileReceived.A_SOURCE_FILE_CONFIG_KEY; EXCEPTION WHEN NO_DATA_FOUND THEN ENV_MANAGER.LOG_PROCESS_ERROR('Source file or config not found for key: ' || pSourceFileReceivedKey, vParameters); RETURN 'Error: Source file with key ' || pSourceFileReceivedKey || ' not found in A_SOURCE_FILE_RECEIVED table'; END; -- Extract template schema and table from template table name vTemplateSchema := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '^([^.]+)'); vTemplateTable := REGEXP_SUBSTR(vSourceFileConfig.TEMPLATE_TABLE_NAME, '\.(.+)$', 1, 1, NULL, 1); -- Build CSV file URI vCsvFileUri := ENV_MANAGER.gvInboxBucketUri || 'INBOX/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.SOURCE_FILE_ID || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; -- Find validation log table (most recent one with errors) -- Look for validation log tables related to this external table BEGIN SELECT table_name INTO vValidationLogTable FROM ( SELECT table_name, REGEXP_SUBSTR(table_name, '\$([0-9]+)_', 1, 1, NULL, 1) as log_number FROM USER_TABLES WHERE table_name LIKE 'VALIDATE$%_LOG' ORDER BY TO_NUMBER(log_number) DESC ) WHERE ROWNUM = 1; EXCEPTION WHEN NO_DATA_FOUND THEN -- If no validation log tables found, use a default name vValidationLogTable := 'VALIDATE$999_LOG'; END; ENV_MANAGER.LOG_PROCESS_EVENT('Calling ENV_MANAGER.ANALYZE_VALIDATION_ERRORS with parameters: ' || 'LogTable=' || vValidationLogTable || ', Schema=' || vTemplateSchema || ', Table=' || vTemplateTable || ', URI=' || SUBSTR(vCsvFileUri, 1, 100) || '...', 'DEBUG', vParameters); -- Call the main function with derived parameters vResult := ENV_MANAGER.ANALYZE_VALIDATION_ERRORS( pValidationLogTable => vValidationLogTable, pTemplateSchema => vTemplateSchema, pTemplateTable => vTemplateTable, pCsvFileUri => vCsvFileUri ); ENV_MANAGER.LOG_PROCESS_EVENT('End', 'DEBUG', vParameters); RETURN vResult; EXCEPTION WHEN OTHERS THEN ENV_MANAGER.LOG_PROCESS_ERROR('Error in ANALYZE_VALIDATION_ERRORS: ' || SQLERRM, vParameters); RETURN 'Error analyzing validation errors: ' || SQLERRM || cgBL || 'Source File Key: ' || pSourceFileReceivedKey || cgBL || 'Check A_SOURCE_FILE_RECEIVED and A_SOURCE_FILE_CONFIG tables for data integrity.'; END ANALYZE_VALIDATION_ERRORS; ---------------------------------------------------------------------------------------------------- -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION ---------------------------------------------------------------------------------------------------- FUNCTION GET_VERSION RETURN VARCHAR2 IS BEGIN RETURN PACKAGE_VERSION; END GET_VERSION; ---------------------------------------------------------------------------------------------------- FUNCTION GET_BUILD_INFO RETURN VARCHAR2 IS BEGIN RETURN ENV_MANAGER.GET_PACKAGE_VERSION_INFO( pPackageName => 'FILE_MANAGER', pVersion => PACKAGE_VERSION, pBuildDate => PACKAGE_BUILD_DATE, pAuthor => PACKAGE_AUTHOR ); END GET_BUILD_INFO; ---------------------------------------------------------------------------------------------------- FUNCTION GET_VERSION_HISTORY RETURN VARCHAR2 IS BEGIN RETURN ENV_MANAGER.FORMAT_VERSION_HISTORY( pPackageName => 'FILE_MANAGER', pVersionHistory => VERSION_HISTORY ); END GET_VERSION_HISTORY; ---------------------------------------------------------------------------------------------------- END; / /