-- =====================================================================================
-- Script: 03_MARS_835_verify_exports.sql
-- Purpose: Verify exported files exist in HIST bucket after export (HIST-only strategy)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Target Locations: mrds_hist_dev/ARCHIVE/CSDB/
-- =====================================================================================

SET SERVEROUTPUT ON SIZE UNLIMITED;
SET FEEDBACK ON;
SET VERIFY OFF;
SET LINESIZE 200;

PROMPT =====================================================================================
PROMPT MARS-835 Verification: Listing exported files in HIST bucket (HIST-only strategy)
PROMPT =====================================================================================

DECLARE
    vHistBucketUri VARCHAR2(500);
    vCredentialName VARCHAR2(100);
    vFileCount NUMBER := 0;
    vTotalHistFiles NUMBER := 0;
    vTotalHistSize NUMBER := 0;
    
    TYPE t_folder_info IS RECORD (
        folder_name VARCHAR2(200),
        table_name VARCHAR2(100),
        expected_format VARCHAR2(20)
    );
    TYPE t_folder_list IS TABLE OF t_folder_info;
    
    vHistFolders t_folder_list;
BEGIN
    -- Get bucket URI and credential from FILE_MANAGER
    vHistBucketUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('ARCHIVE');
    vCredentialName := CT_MRDS.ENV_MANAGER.gvCredentialName;
    
    DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3'));
    DBMS_OUTPUT.PUT_LINE('HIST Bucket URI: ' || vHistBucketUri);
    DBMS_OUTPUT.PUT_LINE('');
    
    -- Initialize folder list (all tables in HIST)
    -- Initialize folder list (all 6 tables in HIST)
    vHistFolders := t_folder_list(
        t_folder_info('ARCHIVE/CSDB/CSDB_DEBT/', 'DEBT', 'Parquet'),
        t_folder_info('ARCHIVE/CSDB/CSDB_DEBT_DAILY/', 'DEBT_DAILY', 'Parquet'),
        t_folder_info('ARCHIVE/CSDB/CSDB_INSTR_RAT_FULL/', 'INSTR_RAT_FULL', 'Parquet'),
        t_folder_info('ARCHIVE/CSDB/CSDB_INSTR_DESC_FULL/', 'INSTR_DESC_FULL', 'Parquet'),
        t_folder_info('ARCHIVE/CSDB/CSDB_ISSUER_RAT_FULL/', 'ISSUER_RAT_FULL', 'Parquet'),
        t_folder_info('ARCHIVE/CSDB/CSDB_ISSUER_DESC_FULL/', 'ISSUER_DESC_FULL', 'Parquet')
    );
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Checking HIST Bucket Exports (Parquet with Hive partitioning - ALL data)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
    -- Check HIST bucket exports
    FOR i IN 1..vHistFolders.COUNT LOOP
        vFileCount := 0;
        
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('Table: ' || vHistFolders(i).table_name || ' (' || vHistFolders(i).expected_format || ')');
        DBMS_OUTPUT.PUT_LINE('Folder: ' || vHistFolders(i).folder_name);
        DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------------');
        
        BEGIN
            FOR rec IN (
                SELECT object_name, bytes, TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS created_date
                FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
                    credential_name => vCredentialName,
                    location_uri => vHistBucketUri || vHistFolders(i).folder_name
                ))
                WHERE object_name LIKE '%.parquet'
                ORDER BY created DESC
                FETCH FIRST 10 ROWS ONLY
            ) LOOP
                vFileCount := vFileCount + 1;
                vTotalHistFiles := vTotalHistFiles + 1;
                vTotalHistSize := vTotalHistSize + rec.bytes;
                
                IF vFileCount <= 5 THEN
                    DBMS_OUTPUT.PUT_LINE('  [' || vFileCount || '] ' || rec.object_name || 
                                       ' (' || ROUND(rec.bytes/1024/1024, 2) || ' MB) - ' || rec.created_date);
                END IF;
            END LOOP;
            
            IF vFileCount = 0 THEN
                DBMS_OUTPUT.PUT_LINE('  [ERROR] No Parquet files found - Export may have failed!');
            ELSIF vFileCount > 5 THEN
                DBMS_OUTPUT.PUT_LINE('  [SUCCESS] Found ' || vFileCount || '+ Parquet files (showing first 5)');
                DBMS_OUTPUT.PUT_LINE('  [INFO] Check Hive partitioning: PARTITION_YEAR=YYYY/PARTITION_MONTH=MM/');
            ELSE
                DBMS_OUTPUT.PUT_LINE('  [SUCCESS] Found ' || vFileCount || ' Parquet file(s)');
            END IF;
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('  [ERROR] Cannot access folder - ' || SQLERRM);
        END;
    END LOOP;
    
    DBMS_OUTPUT.PUT_LINE('');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Export Verification Summary');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('HIST Bucket (Parquet - HIST-only strategy):');
    DBMS_OUTPUT.PUT_LINE('  - Total files: ' || vTotalHistFiles || '+');
    DBMS_OUTPUT.PUT_LINE('  - Total size: ' || ROUND(vTotalHistSize/1024/1024/1024, 2) || '+ GB (sample)');
    DBMS_OUTPUT.PUT_LINE('  - Expected tables: 6 (all CSDB tables exported to HIST)');
    DBMS_OUTPUT.PUT_LINE('');
    
    IF vTotalHistFiles >= 6 THEN
        DBMS_OUTPUT.PUT_LINE('[SUCCESS] OVERALL STATUS: Export appears SUCCESSFUL');
        DBMS_OUTPUT.PUT_LINE('  Files found in HIST bucket for all tables');
        DBMS_OUTPUT.PUT_LINE('  Proceed to record count verification (Step 4)');
    ELSIF vTotalHistFiles = 0 THEN
        DBMS_OUTPUT.PUT_LINE('[FAILED] OVERALL STATUS: Export FAILED');
        DBMS_OUTPUT.PUT_LINE('  No files found in HIST bucket');
        DBMS_OUTPUT.PUT_LINE('  Review export logs for errors');
    ELSE
        DBMS_OUTPUT.PUT_LINE('[WARNING] OVERALL STATUS: Partial export detected');
        DBMS_OUTPUT.PUT_LINE('  Some files missing - review export logs');
    END IF;
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('ERROR during verification: ' || SQLERRM);
        RAISE;
END;
/

PROMPT
PROMPT Export verification completed
PROMPT
