Add TRASH folder management procedures for restoring and purging files
This commit is contained in:
Grzegorz Michalski
@@ -499,6 +499,411 @@ AS
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END GATHER_TABLE_STAT;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- TRASH FOLDER MANAGEMENT PROCEDURES
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vTrashPath VARCHAR2(1000);
|
||||
vOdsPath VARCHAR2(1000);
|
||||
vFilesRestored PLS_INTEGER := 0;
|
||||
vFilesUpdated PLS_INTEGER := 0;
|
||||
vRestoreLevel VARCHAR2(50);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||
vRestoreLevel := 'LEVEL_3_SINGLE_FILE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
|
||||
-- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||
BEGIN
|
||||
SELECT r.*, c.*
|
||||
INTO vSourceFileReceived, vSourceFileConfig
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
END;
|
||||
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath);
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_FILE_NAME = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||
vRestoreLevel := 'LEVEL_2_CONFIG_FILES';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||
|
||||
-- LEVEL 2: Restore all files for specific pSourceFileConfigKey
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
|
||||
FOR file_rec IN (
|
||||
SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := vFilesRestored + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_FILE_NAME = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pRestoreAll THEN
|
||||
vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status');
|
||||
|
||||
-- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||
FOR file_rec IN (
|
||||
SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME,
|
||||
c.A_SOURCE_KEY, c.TABLE_ID
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.MOVE_OBJECT(
|
||||
source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
source_object_uri => vTrashPath,
|
||||
target_object_uri => vOdsPath,
|
||||
target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName
|
||||
);
|
||||
vFilesRestored := vFilesRestored + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'INGESTED',
|
||||
ARCH_FILE_NAME = NULL,
|
||||
PARTITION_YEAR = NULL,
|
||||
PARTITION_MONTH = NULL
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE');
|
||||
END IF;
|
||||
|
||||
COMMIT;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END RESTORE_FILE_FROM_TRASH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
PROCEDURE PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
)
|
||||
IS
|
||||
vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
|
||||
vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE;
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
vTrashLocationUri VARCHAR2(1000);
|
||||
vFilesDeleted PLS_INTEGER := 0;
|
||||
vFilesUpdated PLS_INTEGER := 0;
|
||||
vPurgeLevel VARCHAR2(50);
|
||||
cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
|
||||
-- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1)
|
||||
IF pSourceFileReceivedKey IS NOT NULL THEN
|
||||
vPurgeLevel := 'LEVEL_3_SINGLE_FILE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
|
||||
-- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY
|
||||
BEGIN
|
||||
SELECT r.*, c.*
|
||||
INTO vSourceFileReceived, vSourceFileConfig
|
||||
FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
EXCEPTION
|
||||
WHEN NO_DATA_FOUND THEN
|
||||
RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey);
|
||||
END;
|
||||
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME;
|
||||
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri
|
||||
);
|
||||
vFilesDeleted := 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pSourceFileConfigKey IS NOT NULL THEN
|
||||
vPurgeLevel := 'LEVEL_2_CONFIG_FILES';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey);
|
||||
|
||||
-- LEVEL 2: Delete all files for specific pSourceFileConfigKey
|
||||
vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/';
|
||||
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri);
|
||||
|
||||
BEGIN
|
||||
FOR trash_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vTrashLocationUri
|
||||
)
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri || trash_file.object_name
|
||||
);
|
||||
vFilesDeleted := vFilesDeleted + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM);
|
||||
END;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey
|
||||
AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSIF pPurgeAll THEN
|
||||
vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status');
|
||||
|
||||
-- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations
|
||||
FOR config_rec IN (
|
||||
SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID
|
||||
FROM CT_MRDS.A_SOURCE_FILE_CONFIG c
|
||||
JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
|
||||
WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'
|
||||
) LOOP
|
||||
vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/';
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri);
|
||||
|
||||
BEGIN
|
||||
FOR trash_file IN (
|
||||
SELECT object_name
|
||||
FROM DBMS_CLOUD.LIST_OBJECTS(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
location_uri => vTrashLocationUri
|
||||
)
|
||||
) LOOP
|
||||
BEGIN
|
||||
DBMS_CLOUD.DELETE_OBJECT(
|
||||
credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName,
|
||||
object_uri => vTrashLocationUri || trash_file.object_name
|
||||
);
|
||||
vFilesDeleted := vFilesDeleted + 1;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
END;
|
||||
END LOOP;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM);
|
||||
END;
|
||||
END LOOP;
|
||||
|
||||
UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED
|
||||
SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED'
|
||||
WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED';
|
||||
vFilesUpdated := SQL%ROWCOUNT;
|
||||
|
||||
ELSE
|
||||
RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE');
|
||||
END IF;
|
||||
|
||||
COMMIT;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO');
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
ROLLBACK;
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
|
||||
END PURGE_TRASH_FOLDER;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
PURGE_TRASH_FOLDER(
|
||||
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||
pPurgeAll => pPurgeAll
|
||||
);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END PURGE_TRASH_FOLDER;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER
|
||||
IS
|
||||
vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
|
||||
BEGIN
|
||||
vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST(
|
||||
'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'),
|
||||
'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'),
|
||||
'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END
|
||||
));
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
|
||||
----
|
||||
RESTORE_FILE_FROM_TRASH(
|
||||
pSourceFileReceivedKey => pSourceFileReceivedKey,
|
||||
pSourceFileConfigKey => pSourceFileConfigKey,
|
||||
pRestoreAll => pRestoreAll
|
||||
);
|
||||
----
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
|
||||
RETURN SQLCODE;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
|
||||
RETURN SQLCODE;
|
||||
END RESTORE_FILE_FROM_TRASH;
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -84,6 +84,77 @@ AS
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Restores files from TRASH folder back to ODS at three different granularity levels.
|
||||
* Moves files from TRASH subfolder back to ODS subfolder in DATA bucket.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341);
|
||||
* @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE);
|
||||
**/
|
||||
PROCEDURE RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name RESTORE_FILE_FROM_TRASH
|
||||
* @desc Function overload for RESTORE_FILE_FROM_TRASH procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority)
|
||||
* @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION RESTORE_FILE_FROM_TRASH (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pRestoreAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Deletes files from TRASH folder at three different granularity levels.
|
||||
* Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345);
|
||||
* @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341);
|
||||
* @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE);
|
||||
**/
|
||||
PROCEDURE PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
/**
|
||||
* @name PURGE_TRASH_FOLDER
|
||||
* @desc Function overload for PURGE_TRASH_FOLDER procedure.
|
||||
* Returns SQLCODE for Python library integration.
|
||||
* Calls the main PURGE_TRASH_FOLDER procedure and captures execution result.
|
||||
* WARNING: This operation is irreversible - files are permanently deleted from TRASH.
|
||||
* @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority)
|
||||
* @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority)
|
||||
* @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority)
|
||||
* @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL;
|
||||
* @ex_rslt 0 (success) or error code
|
||||
**/
|
||||
FUNCTION PURGE_TRASH_FOLDER (
|
||||
pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL,
|
||||
pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL,
|
||||
pPurgeAll IN BOOLEAN DEFAULT FALSE
|
||||
) RETURN PLS_INTEGER;
|
||||
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
-- PACKAGE VERSION MANAGEMENT FUNCTIONS
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user