Add MARS-835-PREHOOK scripts to update and rollback A_SOURCE_FILE_RECEIVED table structure
This commit is contained in:
Grzegorz Michalski
@@ -0,0 +1,30 @@
|
||||
-- ====================================================================
|
||||
-- A_SOURCE_FILE_RECEIVED Table
|
||||
-- ====================================================================
|
||||
-- Purpose: Track received files and their processing status
|
||||
-- ====================================================================
|
||||
|
||||
CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED (
|
||||
A_SOURCE_FILE_RECEIVED_KEY NUMBER(38,0) NOT NULL ENABLE,
|
||||
A_SOURCE_FILE_CONFIG_KEY NUMBER(38,0) NOT NULL ENABLE,
|
||||
SOURCE_FILE_NAME VARCHAR2(1000) NOT NULL,
|
||||
CHECKSUM VARCHAR2(128),
|
||||
CREATED TIMESTAMP(6) WITH TIME ZONE,
|
||||
BYTES NUMBER,
|
||||
RECEPTION_DATE DATE NOT NULL,
|
||||
PROCESSING_STATUS VARCHAR2(200),
|
||||
EXTERNAL_TABLE_NAME VARCHAR2(200),
|
||||
PARTITION_YEAR VARCHAR2(4),
|
||||
PARTITION_MONTH VARCHAR2(2),
|
||||
ARCH_PATH VARCHAR2(1000),
|
||||
PROCESS_NAME VARCHAR2(200),
|
||||
CONSTRAINT A_SOURCE_FILE_RECEIVED_PK PRIMARY KEY (A_SOURCE_FILE_RECEIVED_KEY),
|
||||
CONSTRAINT ASFR_A_SOURCE_FILE_CONFIG_KEY_FK FOREIGN KEY(A_SOURCE_FILE_CONFIG_KEY) REFERENCES CT_MRDS.A_SOURCE_FILE_CONFIG(A_SOURCE_FILE_CONFIG_KEY),
|
||||
CONSTRAINT A_SOURCE_FILE_RECEIVED_CHK CHECK (PROCESSING_STATUS IN ('RECEIVED', 'VALIDATED', 'READY_FOR_INGESTION', 'INGESTED', 'ARCHIVED'))
|
||||
) TABLESPACE "DATA";
|
||||
|
||||
-- Unique index for file identification (workaround for TIMESTAMP WITH TIMEZONE constraint limitation)
|
||||
CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1
|
||||
ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE;
|
||||
@@ -606,6 +606,7 @@ AS
|
||||
pTemplateTableName IN VARCHAR2 default NULL,
|
||||
pMaxFileSize IN NUMBER default 104857600,
|
||||
pRegisterExport IN BOOLEAN default FALSE,
|
||||
pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
@@ -638,6 +639,7 @@ AS
|
||||
,'pTemplateTableName => '''||nvl(pTemplateTableName, 'NULL')||''''
|
||||
,'pMaxFileSize => '''||nvl(TO_CHAR(pMaxFileSize), 'NULL')||''''
|
||||
,'pRegisterExport => '''||CASE WHEN pRegisterExport THEN 'TRUE' ELSE 'FALSE' END||''''
|
||||
,'pProcessName => '''||nvl(pProcessName, 'NULL')||''''
|
||||
,'pCredentialName => '''||nvl(pCredentialName, 'NULL')||''''
|
||||
));
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
@@ -853,7 +855,8 @@ AS
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
ARCH_PATH,
|
||||
PROCESS_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
NVL(vConfigKey, -1), -- Use config key if found, otherwise -1
|
||||
@@ -865,7 +868,8 @@ AS
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for single-file exports
|
||||
NULL, -- PARTITION_MONTH not used for single-file exports
|
||||
NULL -- ARCH_FILE_NAME not used for single-file exports
|
||||
NULL, -- ARCH_PATH not used for single-file exports
|
||||
pProcessName -- Process name from parameter
|
||||
);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || vBytes || ' bytes', 'INFO', vParameters);
|
||||
@@ -887,7 +891,8 @@ AS
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
ARCH_PATH,
|
||||
PROCESS_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
NVL(vConfigKey, -1), -- Use config key if found, otherwise -1
|
||||
@@ -896,7 +901,8 @@ AS
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for single-file exports
|
||||
NULL, -- PARTITION_MONTH not used for single-file exports
|
||||
NULL -- ARCH_FILE_NAME not used for single-file exports
|
||||
NULL, -- ARCH_PATH not used for single-file exports
|
||||
pProcessName -- Process name from parameter
|
||||
);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Registered file without metadata: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vSanitizedFileName, 'INFO', vParameters);
|
||||
@@ -1198,6 +1204,7 @@ AS
|
||||
pTemplateTableName IN VARCHAR2 default NULL,
|
||||
pMaxFileSize IN NUMBER default 104857600,
|
||||
pRegisterExport IN BOOLEAN default FALSE,
|
||||
pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
)
|
||||
IS
|
||||
@@ -1544,7 +1551,8 @@ AS
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
ARCH_PATH,
|
||||
PROCESS_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
vConfigKey, -- Config key from A_SOURCE_FILE_CONFIG lookup
|
||||
@@ -1556,7 +1564,8 @@ AS
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for CSV exports
|
||||
NULL, -- PARTITION_MONTH not used for CSV exports
|
||||
NULL -- ARCH_FILE_NAME not used for CSV exports
|
||||
NULL, -- ARCH_PATH not used for CSV exports
|
||||
pProcessName -- Process name from parameter
|
||||
);
|
||||
|
||||
ENV_MANAGER.LOG_PROCESS_EVENT('Registered file: FileReceivedKey=' || vSourceFileReceivedKey || ', File=' || vActualFileName || ', Size=' || vBytes || ' bytes', 'DEBUG', vParameters);
|
||||
@@ -1578,7 +1587,8 @@ AS
|
||||
PROCESSING_STATUS,
|
||||
PARTITION_YEAR,
|
||||
PARTITION_MONTH,
|
||||
ARCH_FILE_NAME
|
||||
ARCH_PATH,
|
||||
PROCESS_NAME
|
||||
) VALUES (
|
||||
vSourceFileReceivedKey,
|
||||
vConfigKey, -- Config key from A_SOURCE_FILE_CONFIG lookup
|
||||
@@ -1587,7 +1597,8 @@ AS
|
||||
'INGESTED',
|
||||
NULL, -- PARTITION_YEAR not used for CSV exports
|
||||
NULL, -- PARTITION_MONTH not used for CSV exports
|
||||
NULL -- ARCH_FILE_NAME not used for CSV exports
|
||||
NULL, -- ARCH_PATH not used for CSV exports
|
||||
pProcessName -- Process name from parameter
|
||||
);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
@@ -81,6 +81,7 @@ AS
|
||||
* - NULL = use source table columns in natural order
|
||||
* @param pMaxFileSize - Maximum file size in bytes (default 104857600 = 100MB, min 10MB, max 1GB)
|
||||
* @param pRegisterExport - When TRUE, registers exported CSV file in A_SOURCE_FILE_RECEIVED table
|
||||
* @param pProcessName - Process name stored in PROCESS_NAME column (default 'DATA_EXPORTER')
|
||||
* @example
|
||||
* begin
|
||||
* DATA_EXPORTER.EXPORT_TABLE_DATA(
|
||||
@@ -106,6 +107,7 @@ AS
|
||||
pTemplateTableName IN VARCHAR2 default NULL,
|
||||
pMaxFileSize IN NUMBER default 104857600,
|
||||
pRegisterExport IN BOOLEAN default FALSE,
|
||||
pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
@@ -162,6 +164,7 @@ AS
|
||||
* File naming pattern: {pFileName}_YYYYMM.csv or {TABLENAME}_YYYYMM.csv (if pFileName is NULL)
|
||||
* When pRegisterExport=TRUE, successfully exported files are registered in:
|
||||
* - CT_MRDS.A_SOURCE_FILE_RECEIVED (tracks file location, size, checksum, and metadata)
|
||||
* @param pProcessName - Process name stored in PROCESS_NAME column (default 'DATA_EXPORTER')
|
||||
* @example
|
||||
* begin
|
||||
* -- With custom filename
|
||||
@@ -207,6 +210,7 @@ AS
|
||||
pTemplateTableName IN VARCHAR2 default NULL,
|
||||
pMaxFileSize IN NUMBER default 104857600,
|
||||
pRegisterExport IN BOOLEAN default FALSE,
|
||||
pProcessName IN VARCHAR2 default 'DATA_EXPORTER',
|
||||
pCredentialName IN VARCHAR2 default ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user