Add TRASH folder management procedures for restoring and purging files
This commit is contained in:
Grzegorz Michalski
@@ -499,6 +499,411 @@ AS
|
|||||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||||
END GATHER_TABLE_STAT;
|
END GATHER_TABLE_STAT;
|
||||||
|
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
-- TRASH FOLDER MANAGEMENT PROCEDURES
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
)
|
||||||
|
IS
|
||||||
|
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||||
|
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||||
|
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||||
|
vTrashPath VARCHAR2(1000);
|
||||||
|
vOdsPath VARCHAR2(1000);
|
||||||
|
vFilesRestored PLS_INTEGER := 0;
|
||||||
|
vFilesUpdated PLS_INTEGER := 0;
|
||||||
|
vRestoreLevel VARCHAR2(50);
|
||||||
|
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||||
|
BEGIN
|
||||||
|
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||||
|
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||||
|
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||||
|
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||||
|
));
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||||
|
|
||||||
|
-- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||||
|
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||||
|
vRestoreLevel := 'LEVEL_3_SINGLE_FILE';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||||
|
|
||||||
|
-- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||||
|
BEGIN
|
||||||
|
SELECT r.*, c.*
|
||||||
|
INTO vSourceFileReceived, vSourceFileConfig
|
||||||
|
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||||
|
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||||
|
WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||||
|
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
EXCEPTION
|
||||||
|
WHEN NO_DATA_FOUND THEN
|
||||||
|
RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||||
|
END;
|
||||||
|
|
||||||
|
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||||
|
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||||
|
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath);
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.MOVE_OBJECT(
|
||||||
|
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
source_object_uri => vTrashPath,
|
||||||
|
target_object_uri => vOdsPath,
|
||||||
|
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||||
|
);
|
||||||
|
vFilesRestored := 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'INGESTED',
|
||||||
|
ARCH_FILE_NAME = NULL,
|
||||||
|
PARTITION_YEAR = NULL,
|
||||||
|
PARTITION_MONTH = NULL
|
||||||
|
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||||
|
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||||
|
vRestoreLevel := 'LEVEL_2_CONFIG_FILES';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||||
|
|
||||||
|
-- LEVEL 2: Restore all files for specific pSourceFileConfigKey
|
||||||
|
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||||
|
|
||||||
|
FOR file_rec IN (
|
||||||
|
SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME
|
||||||
|
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||||
|
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||||
|
) LOOP
|
||||||
|
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||||
|
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.MOVE_OBJECT(
|
||||||
|
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
source_object_uri => vTrashPath,
|
||||||
|
target_object_uri => vOdsPath,
|
||||||
|
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||||
|
);
|
||||||
|
vFilesRestored := vFilesRestored + 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'INGESTED',
|
||||||
|
ARCH_FILE_NAME = NULL,
|
||||||
|
PARTITION_YEAR = NULL,
|
||||||
|
PARTITION_MONTH = NULL
|
||||||
|
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||||
|
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSIF pRestoreAll THEN
|
||||||
|
vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status');
|
||||||
|
|
||||||
|
-- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||||
|
FOR file_rec IN (
|
||||||
|
SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME,
|
||||||
|
c.A_SOURCE_KEY, c.TABLE_ID
|
||||||
|
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||||
|
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||||
|
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||||
|
) LOOP
|
||||||
|
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||||
|
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.MOVE_OBJECT(
|
||||||
|
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
source_object_uri => vTrashPath,
|
||||||
|
target_object_uri => vOdsPath,
|
||||||
|
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||||
|
);
|
||||||
|
vFilesRestored := vFilesRestored + 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'INGESTED',
|
||||||
|
ARCH_FILE_NAME = NULL,
|
||||||
|
PARTITION_YEAR = NULL,
|
||||||
|
PARTITION_MONTH = NULL
|
||||||
|
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSE
|
||||||
|
RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE');
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO');
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO');
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
ROLLBACK;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||||
|
END RESTORE_FILE_FROM_TRASH;
|
||||||
|
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
PROCEDURE PURGE_TRASH_FOLDER (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
)
|
||||||
|
IS
|
||||||
|
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||||
|
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||||
|
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||||
|
vTrashLocationUri VARCHAR2(1000);
|
||||||
|
vFilesDeleted PLS_INTEGER := 0;
|
||||||
|
vFilesUpdated PLS_INTEGER := 0;
|
||||||
|
vPurgeLevel VARCHAR2(50);
|
||||||
|
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||||
|
BEGIN
|
||||||
|
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||||
|
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||||
|
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||||
|
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||||
|
));
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||||
|
|
||||||
|
-- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||||
|
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||||
|
vPurgeLevel := 'LEVEL_3_SINGLE_FILE';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||||
|
|
||||||
|
-- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||||
|
BEGIN
|
||||||
|
SELECT r.*, c.*
|
||||||
|
INTO vSourceFileReceived, vSourceFileConfig
|
||||||
|
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||||
|
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||||
|
WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||||
|
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
EXCEPTION
|
||||||
|
WHEN NO_DATA_FOUND THEN
|
||||||
|
RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||||
|
END;
|
||||||
|
|
||||||
|
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
object_uri => vTrashLocationUri
|
||||||
|
);
|
||||||
|
vFilesDeleted := 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||||
|
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||||
|
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||||
|
vPurgeLevel := 'LEVEL_2_CONFIG_FILES';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||||
|
|
||||||
|
-- LEVEL 2: Delete all files for specific pSourceFileConfigKey
|
||||||
|
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||||
|
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
|
||||||
|
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri);
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
FOR trash_file IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
location_uri => vTrashLocationUri
|
||||||
|
)
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
object_uri => vTrashLocationUri || trash_file.object_name
|
||||||
|
);
|
||||||
|
vFilesDeleted := vFilesDeleted + 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||||
|
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||||
|
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSIF pPurgeAll THEN
|
||||||
|
vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status');
|
||||||
|
|
||||||
|
-- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||||
|
FOR config_rec IN (
|
||||||
|
SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID
|
||||||
|
FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||||
|
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||||
|
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||||
|
) LOOP
|
||||||
|
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/';
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri);
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
FOR trash_file IN (
|
||||||
|
SELECT object_name
|
||||||
|
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||||
|
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
location_uri => vTrashLocationUri
|
||||||
|
)
|
||||||
|
) LOOP
|
||||||
|
BEGIN
|
||||||
|
DBMS_CLOUD.DELETE_OBJECT(
|
||||||
|
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||||
|
object_uri => vTrashLocationUri || trash_file.object_name
|
||||||
|
);
|
||||||
|
vFilesDeleted := vFilesDeleted + 1;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM);
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||||
|
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||||
|
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||||
|
vFilesUpdated := SQL%ROWCOUNT;
|
||||||
|
|
||||||
|
ELSE
|
||||||
|
RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE');
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO');
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO');
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
ROLLBACK;
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||||
|
END PURGE_TRASH_FOLDER;
|
||||||
|
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
FUNCTION PURGE_TRASH_FOLDER (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
) RETURN PLS_INTEGER
|
||||||
|
IS
|
||||||
|
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||||
|
BEGIN
|
||||||
|
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||||
|
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||||
|
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||||
|
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||||
|
));
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||||
|
----
|
||||||
|
PURGE_TRASH_FOLDER(
|
||||||
|
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||||
|
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||||
|
pPurgeAll => pPurgeAll
|
||||||
|
);
|
||||||
|
----
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||||
|
RETURN SQLCODE;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RETURN SQLCODE;
|
||||||
|
END PURGE_TRASH_FOLDER;
|
||||||
|
|
||||||
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
) RETURN PLS_INTEGER
|
||||||
|
IS
|
||||||
|
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||||
|
BEGIN
|
||||||
|
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||||
|
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||||
|
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||||
|
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||||
|
));
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||||
|
----
|
||||||
|
RESTORE_FILE_FROM_TRASH(
|
||||||
|
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||||
|
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||||
|
pRestoreAll => pRestoreAll
|
||||||
|
);
|
||||||
|
----
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||||
|
RETURN SQLCODE;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN OTHERS THEN
|
||||||
|
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||||
|
RETURN SQLCODE;
|
||||||
|
END RESTORE_FILE_FROM_TRASH;
|
||||||
|
|
||||||
----------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------------
|
||||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||||
----------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -84,6 +84,77 @@ AS
|
|||||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||||
) RETURN PLS_INTEGER;
|
) RETURN PLS_INTEGER;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @name RESTORE_FILE_FROM_TRASH
|
||||||
|
* @desc Restores files from TRASH folder back to ODS at three different granularity levels.
|
||||||
|
* Moves files from TRASH subfolder back to ODS subfolder in DATA bucket.
|
||||||
|
* Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata.
|
||||||
|
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||||
|
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||||
|
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||||
|
* @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345);
|
||||||
|
* @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341);
|
||||||
|
* @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE);
|
||||||
|
**/
|
||||||
|
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @name RESTORE_FILE_FROM_TRASH
|
||||||
|
* @desc Function overload for RESTORE_FILE_FROM_TRASH procedure.
|
||||||
|
* Returns SQLCODE for Python library integration.
|
||||||
|
* Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result.
|
||||||
|
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||||
|
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||||
|
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||||
|
* @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||||
|
* @ex_rslt 0 (success) or error code
|
||||||
|
**/
|
||||||
|
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
) RETURN PLS_INTEGER;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @name PURGE_TRASH_FOLDER
|
||||||
|
* @desc Deletes files from TRASH folder at three different granularity levels.
|
||||||
|
* Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files.
|
||||||
|
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||||
|
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||||
|
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||||
|
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||||
|
* @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345);
|
||||||
|
* @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341);
|
||||||
|
* @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE);
|
||||||
|
**/
|
||||||
|
PROCEDURE PURGE_TRASH_FOLDER (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @name PURGE_TRASH_FOLDER
|
||||||
|
* @desc Function overload for PURGE_TRASH_FOLDER procedure.
|
||||||
|
* Returns SQLCODE for Python library integration.
|
||||||
|
* Calls the main PURGE_TRASH_FOLDER procedure and captures execution result.
|
||||||
|
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||||
|
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||||
|
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||||
|
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||||
|
* @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||||
|
* @ex_rslt 0 (success) or error code
|
||||||
|
**/
|
||||||
|
FUNCTION PURGE_TRASH_FOLDER (
|
||||||
|
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||||
|
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||||
|
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||||
|
) RETURN PLS_INTEGER;
|
||||||
|
|
||||||
---------------------------------------------------------------------------------------------------------------------------
|
---------------------------------------------------------------------------------------------------------------------------
|
||||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||||
---------------------------------------------------------------------------------------------------------------------------
|
---------------------------------------------------------------------------------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user