From 86535048fcd570f64d502f02084b4860cd319e9a Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Tue, 10 Feb 2026 13:53:12 +0100 Subject: [PATCH] Add TRASH folder management procedures for restoring and purging files --- .../MARS-828/new_version/FILE_ARCHIVER.pkb | 405 ++++++++++++++++++ .../MARS-828/new_version/FILE_ARCHIVER.pkg | 71 +++ 2 files changed, 476 insertions(+) diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb index 953d383..529ec6e 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkb @@ -499,6 +499,411 @@ AS RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); END GATHER_TABLE_STAT; + ---------------------------------------------------------------------------------------------------- + -- TRASH FOLDER MANAGEMENT PROCEDURES + ---------------------------------------------------------------------------------------------------- + + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashPath VARCHAR2(1000); + vOdsPath VARCHAR2(1000); + vFilesRestored PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vRestoreLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine restore level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 3 - Single file restoration','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Restore single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT r.*, c.* + INTO vSourceFileReceived, vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20101, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restoring single file from TRASH to ODS','INFO', 'Source: ' || vTrashPath || cgBL || 'Target: ' || vOdsPath); + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File successfully moved from TRASH to ODS','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file from TRASH to ODS','ERROR', vTrashPath); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20103, 'Failed to restore file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vRestoreLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 2 - Configuration-based restoration','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Restore all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + + FOR file_rec IN ( + SELECT A_SOURCE_FILE_RECEIVED_KEY, SOURCE_FILE_NAME + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pRestoreAll THEN + vRestoreLevel := 'LEVEL_1_GLOBAL_RESTORE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Restore level: LEVEL 1 - Global TRASH restoration','INFO', 'Restoring ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Restore all files with ARCHIVED_AND_TRASHED status across all configurations + FOR file_rec IN ( + SELECT r.A_SOURCE_FILE_RECEIVED_KEY, r.SOURCE_FILE_NAME, + c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + vOdsPath := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/' || file_rec.A_SOURCE_KEY || '/' || file_rec.TABLE_ID || '/' || file_rec.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.MOVE_OBJECT( + source_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + source_object_uri => vTrashPath, + target_object_uri => vOdsPath, + target_credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName + ); + vFilesRestored := vFilesRestored + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH','DEBUG', file_rec.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH','ERROR', file_rec.SOURCE_FILE_NAME); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'INGESTED', + ARCH_FILE_NAME = NULL, + PARTITION_YEAR = NULL, + PARTITION_MONTH = NULL + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20104, 'No restore level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pRestoreAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files restored from TRASH: ' || vFilesRestored,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to INGESTED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH restoration completed successfully','INFO', 'Level: ' || vRestoreLevel || ', Files restored: ' || vFilesRestored || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END RESTORE_FILE_FROM_TRASH; + + ---------------------------------------------------------------------------------------------------- + + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) + IS + vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE; + vSourceFileReceived CT_MRDS.A_SOURCE_FILE_RECEIVED%ROWTYPE; + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + vTrashLocationUri VARCHAR2(1000); + vFilesDeleted PLS_INTEGER := 0; + vFilesUpdated PLS_INTEGER := 0; + vPurgeLevel VARCHAR2(50); + cgBL CONSTANT VARCHAR2(2) := CHR(13)||CHR(10); + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + + -- Determine purge level (priority: LEVEL 3 > LEVEL 2 > LEVEL 1) + IF pSourceFileReceivedKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_3_SINGLE_FILE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 3 - Single file deletion','INFO', 'A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + + -- LEVEL 3: Delete single file by A_SOURCE_FILE_RECEIVED_KEY + BEGIN + SELECT r.*, c.* + INTO vSourceFileReceived, vSourceFileConfig + FROM CT_MRDS.A_SOURCE_FILE_RECEIVED r + JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + EXCEPTION + WHEN NO_DATA_FOUND THEN + RAISE_APPLICATION_ERROR(-20301, 'File not found or status is not ARCHIVED_AND_TRASHED for A_SOURCE_FILE_RECEIVED_KEY: ' || pSourceFileReceivedKey); + END; + + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/' || vSourceFileReceived.SOURCE_FILE_NAME; + + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri + ); + vFilesDeleted := 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Single file deleted from TRASH','INFO', vSourceFileReceived.SOURCE_FILE_NAME); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20302, 'Failed to delete single file from TRASH: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_RECEIVED_KEY = pSourceFileReceivedKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pSourceFileConfigKey IS NOT NULL THEN + vPurgeLevel := 'LEVEL_2_CONFIG_FILES'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 2 - Configuration-based deletion','INFO', 'pSourceFileConfigKey: ' || pSourceFileConfigKey); + + -- LEVEL 2: Delete all files for specific pSourceFileConfigKey + vSourceFileConfig := CT_MRDS.FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey); + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || vSourceFileConfig.A_SOURCE_KEY || '/' || vSourceFileConfig.TABLE_ID || '/'; + + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purging TRASH folder for configuration','INFO', 'Location: ' || vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list or delete TRASH files','ERROR', vTrashLocationUri); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(-20303, 'Failed to purge TRASH folder for configuration: ' || SQLERRM); + END; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey + AND PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSIF pPurgeAll THEN + vPurgeLevel := 'LEVEL_1_GLOBAL_PURGE'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Purge level: LEVEL 1 - Global TRASH purge','INFO', 'Deleting ALL files with ARCHIVED_AND_TRASHED status'); + + -- LEVEL 1: Delete all files with ARCHIVED_AND_TRASHED status across all configurations + FOR config_rec IN ( + SELECT DISTINCT c.A_SOURCE_FILE_CONFIG_KEY, c.A_SOURCE_KEY, c.TABLE_ID + FROM CT_MRDS.A_SOURCE_FILE_CONFIG c + JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r ON r.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY + WHERE r.PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED' + ) LOOP + vTrashLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'TRASH/' || config_rec.A_SOURCE_KEY || '/' || config_rec.TABLE_ID || '/'; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Processing TRASH location','DEBUG', vTrashLocationUri); + + BEGIN + FOR trash_file IN ( + SELECT object_name + FROM DBMS_CLOUD.LIST_OBJECTS( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + location_uri => vTrashLocationUri + ) + ) LOOP + BEGIN + DBMS_CLOUD.DELETE_OBJECT( + credential_name => CT_MRDS.ENV_MANAGER.gvCredentialName, + object_uri => vTrashLocationUri || trash_file.object_name + ); + vFilesDeleted := vFilesDeleted + 1; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('File deleted from TRASH','DEBUG', trash_file.object_name); + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to delete file from TRASH','ERROR', trash_file.object_name); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + END; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Failed to list TRASH files','WARNING', vTrashLocationUri || ' - ' || SQLERRM); + END; + END LOOP; + + UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED + SET PROCESSING_STATUS = 'ARCHIVED_AND_PURGED' + WHERE PROCESSING_STATUS = 'ARCHIVED_AND_TRASHED'; + vFilesUpdated := SQL%ROWCOUNT; + + ELSE + RAISE_APPLICATION_ERROR(-20304, 'No purge level specified. Provide pSourceFileReceivedKey, pSourceFileConfigKey, or set pPurgeAll=TRUE'); + END IF; + + COMMIT; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total files deleted from TRASH: ' || vFilesDeleted,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Total file records updated to ARCHIVED_AND_PURGED: ' || vFilesUpdated,'INFO'); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('TRASH folder purge completed successfully','INFO', 'Level: ' || vPurgeLevel || ', Files deleted: ' || vFilesDeleted || ', Records updated: ' || vFilesUpdated); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO', vParameters); + EXCEPTION + WHEN OTHERS THEN + ROLLBACK; + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RAISE_APPLICATION_ERROR(CT_MRDS.ENV_MANAGER.CODE_UNKNOWN, CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE)); + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pPurgeAll => '||CASE WHEN pPurgeAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + PURGE_TRASH_FOLDER( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pPurgeAll => pPurgeAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END PURGE_TRASH_FOLDER; + + ---------------------------------------------------------------------------------------------------- + + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER + IS + vParameters CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE; + BEGIN + vParameters := CT_MRDS.ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST( + 'pSourceFileReceivedKey => '||nvl(to_char(pSourceFileReceivedKey), 'NULL'), + 'pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL'), + 'pRestoreAll => '||CASE WHEN pRestoreAll THEN 'TRUE' ELSE 'FALSE' END + )); + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters); + ---- + RESTORE_FILE_FROM_TRASH( + pSourceFileReceivedKey => pSourceFileReceivedKey, + pSourceFileConfigKey => pSourceFileConfigKey, + pRestoreAll => pRestoreAll + ); + ---- + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters); + RETURN SQLCODE; + EXCEPTION + WHEN OTHERS THEN + CT_MRDS.ENV_MANAGER.LOG_PROCESS_EVENT(CT_MRDS.ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters); + RETURN SQLCODE; + END RESTORE_FILE_FROM_TRASH; + ---------------------------------------------------------------------------------------------------- -- PACKAGE VERSION MANAGEMENT FUNCTIONS IMPLEMENTATION ---------------------------------------------------------------------------------------------------- diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg index ff6ed36..4e2e199 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/FILE_ARCHIVER.pkg @@ -84,6 +84,77 @@ AS pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE ) RETURN PLS_INTEGER; + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Restores files from TRASH folder back to ODS at three different granularity levels. + * Moves files from TRASH subfolder back to ODS subfolder in DATA bucket. + * Updates status from ARCHIVED_AND_TRASHED to INGESTED and clears archival metadata. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Restore single file: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345); + * @example -- Restore all files for config: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileConfigKey => 341); + * @example -- Restore all TRASH globally: CALL FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pRestoreAll => TRUE); + **/ + PROCEDURE RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name RESTORE_FILE_FROM_TRASH + * @desc Function overload for RESTORE_FILE_FROM_TRASH procedure. + * Returns SQLCODE for Python library integration. + * Calls the main RESTORE_FILE_FROM_TRASH procedure and captures execution result. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to restore by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Restore all files for specific configuration key (medium priority) + * @param pRestoreAll - (LEVEL 1) When TRUE, restore ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.RESTORE_FILE_FROM_TRASH(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION RESTORE_FILE_FROM_TRASH ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pRestoreAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + + /** + * @name PURGE_TRASH_FOLDER + * @desc Deletes files from TRASH folder at three different granularity levels. + * Updates status from ARCHIVED_AND_TRASHED to ARCHIVED_AND_PURGED for all affected files. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example -- Delete single file: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345); + * @example -- Delete all files for config: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileConfigKey => 341); + * @example -- Delete all TRASH globally: CALL FILE_ARCHIVER.PURGE_TRASH_FOLDER(pPurgeAll => TRUE); + **/ + PROCEDURE PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ); + + /** + * @name PURGE_TRASH_FOLDER + * @desc Function overload for PURGE_TRASH_FOLDER procedure. + * Returns SQLCODE for Python library integration. + * Calls the main PURGE_TRASH_FOLDER procedure and captures execution result. + * WARNING: This operation is irreversible - files are permanently deleted from TRASH. + * @param pSourceFileReceivedKey - (LEVEL 3) Specific file to delete by A_SOURCE_FILE_RECEIVED_KEY (highest priority) + * @param pSourceFileConfigKey - (LEVEL 2) Delete all files for specific configuration key (medium priority) + * @param pPurgeAll - (LEVEL 1) When TRUE, delete ALL files with ARCHIVED_AND_TRASHED status (lowest priority) + * @example SELECT FILE_ARCHIVER.PURGE_TRASH_FOLDER(pSourceFileReceivedKey => 12345) FROM DUAL; + * @ex_rslt 0 (success) or error code + **/ + FUNCTION PURGE_TRASH_FOLDER ( + pSourceFileReceivedKey IN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY%TYPE DEFAULT NULL, + pSourceFileConfigKey IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE DEFAULT NULL, + pPurgeAll IN BOOLEAN DEFAULT FALSE + ) RETURN PLS_INTEGER; + --------------------------------------------------------------------------------------------------------------------------- -- PACKAGE VERSION MANAGEMENT FUNCTIONS ---------------------------------------------------------------------------------------------------------------------------