-- =====================================================================================
-- Script: 00_MARS_835_pre_check_existing_files.sql
-- Purpose: Display existing archive files in DATA and HIST buckets before export
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- MARS Issue: MARS-835
-- Target Locations: mrds_data_dev/ODS/CSDB/, mrds_hist_dev/ARCHIVE/CSDB/
-- =====================================================================================

SET SERVEROUTPUT ON SIZE UNLIMITED;
SET FEEDBACK ON;
SET VERIFY OFF;
SET LINESIZE 200;

PROMPT =====================================================================================
PROMPT MARS-835 Pre-Check: Listing existing CSDB files in DATA and HIST buckets
PROMPT =====================================================================================

DECLARE
    vDataBucketUri VARCHAR2(500);
    vHistBucketUri VARCHAR2(500);
    vCredentialName VARCHAR2(100);
    vFileCount NUMBER := 0;
    vTotalFiles NUMBER := 0;
    
    TYPE t_folder_list IS TABLE OF VARCHAR2(200);
    vDataFolders t_folder_list;
    vHistFolders t_folder_list;
BEGIN
    -- Get bucket URIs and credential from FILE_MANAGER configuration
    vDataBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA');
    vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
    vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
    
    DBMS_OUTPUT.PUT_LINE('CHECK TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
    DBMS_OUTPUT.PUT_LINE('DATA Bucket URI: ' || vDataBucketUri);
    DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
    DBMS_OUTPUT.PUT_LINE('Credential: ' || vCredentialName);
    DBMS_OUTPUT.PUT_LINE('');
    
    -- Initialize folder lists for CSDB tables
    -- DATA bucket folders (CSV format - only DEBT and DEBT_DAILY)
    vDataFolders := t_folder_list(
        'ODS/CSDB/CSDB_DEBT/',
        'ODS/CSDB/CSDB_DEBT_DAILY/'
    );
    
    -- HIST bucket folders (Parquet format - all 6 tables)
    vHistFolders := t_folder_list(
        'ARCHIVE/CSDB/CSDB_DEBT/',
        'ARCHIVE/CSDB/CSDB_DEBT_DAILY/',
        'ARCHIVE/CSDB/CSDB_INSTR_RAT_FULL/',
        'ARCHIVE/CSDB/CSDB_INSTR_DESC_FULL/',
        'ARCHIVE/CSDB/CSDB_ISSUER_RAT_FULL/',
        'ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/'
    );
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Checking DATA Bucket (CSV format - last 6 months)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
    -- Check DATA bucket folders
    FOR i IN 1..vDataFolders.COUNT LOOP
        vFileCount := 0;
        
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('Folder: ' || vDataFolders(i));
        DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
        
        BEGIN
            FOR rec IN (
                SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
                FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
                    credential_name => vCredentialName,
                    location_uri => vDataBucketUri || vDataFolders(i)
                ))
                WHERE object_name LIKE '%.csv'
                ORDER BY created DESC
            ) LOOP
                vFileCount := vFileCount + 1;
                vTotalFiles := vTotalFiles + 1;
                DBMS_OUTPUT.PUT_LINE('  [' || vFileCount || '] ' || rec.object_name || 
                                   ' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
            END LOOP;
            
            IF vFileCount = 0 THEN
                DBMS_OUTPUT.PUT_LINE('  No CSV files found (OK - clean folder)');
            ELSE
                DBMS_OUTPUT.PUT_LINE('  >> WARNING: Found ' || vFileCount || ' existing CSV files');
            END IF;
        EXCEPTION
            WHEN OTHERS THEN
                IF SQLCODE = -20404 THEN
                    DBMS_OUTPUT.PUT_LINE('  Folder does not exist (OK - will be created during export)');
                ELSE
                    DBMS_OUTPUT.PUT_LINE('  ERROR checking folder: ' || SQLERRM);
                END IF;
        END;
    END LOOP;
    
    DBMS_OUTPUT.PUT_LINE('');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket (Parquet format with Hive partitioning)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
    -- Check HIST bucket folders
    FOR i IN 1..vHistFolders.COUNT LOOP
        vFileCount := 0;
        
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('Folder: ' || vHistFolders(i));
        DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
        
        BEGIN
            FOR rec IN (
                SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
                FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
                    credential_name => vCredentialName,
                    location_uri => vHistBucketUri || vHistFolders(i)
                ))
                WHERE object_name LIKE '%.parquet'
                ORDER BY created DESC
                FETCH FIRST 5 ROWS ONLY
            ) LOOP
                vFileCount := vFileCount + 1;
                vTotalFiles := vTotalFiles + 1;
                DBMS_OUTPUT.PUT_LINE('  [' || vFileCount || '] ' || rec.object_name || 
                                   ' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
            END LOOP;
            
            IF vFileCount = 0 THEN
                DBMS_OUTPUT.PUT_LINE('  No Parquet files found (OK - clean folder)');
            ELSE
                DBMS_OUTPUT.PUT_LINE('  >> WARNING: Found Parquet files (showing first 5)');
                DBMS_OUTPUT.PUT_LINE('  >> Total files in folder: Check bucket manually for complete count');
            END IF;
        EXCEPTION
            WHEN OTHERS THEN
                IF SQLCODE = -20404 THEN
                    DBMS_OUTPUT.PUT_LINE('  Folder does not exist (OK - will be created during export)');
                ELSE
                    DBMS_OUTPUT.PUT_LINE('  ERROR checking folder: ' || SQLERRM);
                END IF;
        END;
    END LOOP;
    
    DBMS_OUTPUT.PUT_LINE('');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Pre-Check Summary');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Total files found: ' || vTotalFiles);
    
    IF vTotalFiles > 0 THEN
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('INFO: Existing files detected in buckets');
        DBMS_OUTPUT.PUT_LINE('      Export will ADD new files to existing ones.');
        DBMS_OUTPUT.PUT_LINE('      NOTE: Rollback will NOT delete pre-existing files.');
        DBMS_OUTPUT.PUT_LINE('      Record count verification may show MISMATCH due to pre-existing data.');
    ELSE
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('OK: No existing files found - clean export environment');
    END IF;
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('ERROR during pre-check: ' || SQLERRM);
        DBMS_OUTPUT.PUT_LINE('SQLCODE: ' || SQLCODE);
        RAISE;
END;
/

PROMPT
PROMPT Pre-check completed
PROMPT
