-- =====================================================================================
-- Script: 04_MARS_835_verify_record_counts.sql
-- Purpose: Verify record counts match between source tables and exported data (HIST-only)
-- Author: Grzegorz Michalski
-- Created: 2025-12-17
-- Updated: 2026-02-24 (Changed to HIST-only verification)
-- MARS Issue: MARS-835
-- Verification: Compare OU_CSDB source tables with ODS external tables (HIST only)
-- =====================================================================================

SET SERVEROUTPUT ON SIZE UNLIMITED;
SET FEEDBACK OFF;
SET VERIFY OFF;
SET LINESIZE 200;

PROMPT =====================================================================================
PROMPT MARS-835 Record Count Verification (HIST-only strategy)
PROMPT =====================================================================================
PROMPT Comparing source table counts with HIST external table counts
PROMPT =====================================================================================

DECLARE
    TYPE t_table_info IS RECORD (
        source_schema VARCHAR2(50),
        source_table VARCHAR2(100),
        hist_external_table VARCHAR2(100)
    );
    TYPE t_table_list IS TABLE OF t_table_info;
    
    vTables t_table_list;
    vSourceCount NUMBER;
    vHistCount NUMBER;
    vTotalSourceCount NUMBER := 0;
    vTotalHistCount NUMBER := 0;
    vMismatchCount NUMBER := 0;
    vSql VARCHAR2(4000);
BEGIN
    DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
    DBMS_OUTPUT.PUT_LINE('');
    
    -- Initialize table list (all tables HIST-only)
    vTables := t_table_list(
        t_table_info('OU_CSDB', 'LEGACY_DEBT', 'ODS.CSDB_DEBT_ARCHIVE'),
        t_table_info('OU_CSDB', 'LEGACY_DEBT_DAILY', 'ODS.CSDB_DEBT_DAILY_ARCHIVE'),
        t_table_info('OU_CSDB', 'LEGACY_INSTR_RAT_FULL', 'ODS.CSDB_INSTR_RAT_FULL_ARCHIVE'),
        t_table_info('OU_CSDB', 'LEGACY_INSTR_DESC_FULL', 'ODS.CSDB_INSTR_DESC_FULL_ARCHIVE'),
        t_table_info('OU_CSDB', 'LEGACY_ISSUER_RAT_FULL', 'ODS.CSDB_ISSUER_RAT_FULL_ARCHIVE'),
        t_table_info('OU_CSDB', 'LEGACY_ISSUER_DESC_FULL', 'ODS.CSDB_ISSUER_DESC_FULL_ARCHIVE')
    );
    
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE('Table Name                Source Count      HIST Count   Status');
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    
    FOR i IN 1..vTables.COUNT LOOP
        -- Get source table count
        vSql := 'SELECT COUNT(*) FROM ' || vTables(i).source_schema || '.' || vTables(i).source_table;
        
        BEGIN
            EXECUTE IMMEDIATE vSql INTO vSourceCount;
            vTotalSourceCount := vTotalSourceCount + vSourceCount;
        EXCEPTION
            WHEN OTHERS THEN
                vSourceCount := -1;
                DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table');
                CONTINUE;
        END;
        
        -- Get HIST external table count
        vSql := 'SELECT COUNT(*) FROM ' || vTables(i).hist_external_table;
        BEGIN
            EXECUTE IMMEDIATE vSql INTO vHistCount;
            vTotalHistCount := vTotalHistCount + vHistCount;
        EXCEPTION
            WHEN OTHERS THEN
                -- If source table is empty (0 records), no files were exported
                -- External table returns error, treat as 0
                -- Acceptable error codes:
                --   ORA-29913: error in executing ODCIEXTTABLEOPEN callout
                --   ORA-29400: data cartridge error
                --   KUP-13023: nothing matched wildcard query (no files in bucket)
                -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
                IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
                    vHistCount := 0;
                ELSE
                    vHistCount := -1;
                END IF;
        END;
        
        -- Display results
        DECLARE
            vStatus VARCHAR2(20);
            vHistDisplay VARCHAR2(17);
        BEGIN
            -- Format HIST count display
            IF vHistCount = -1 THEN
                vHistDisplay := 'ERROR';
            ELSE
                vHistDisplay := TO_CHAR(vHistCount, '9,999,999,999');
            END IF;
            
            -- Determine status (HIST only: check HIST = SOURCE)
            IF vHistCount = vSourceCount THEN
                vStatus := 'PASS';
            ELSIF vHistCount = -1 THEN
                vStatus := 'ERROR';
                vMismatchCount := vMismatchCount + 1;
            ELSE
                vStatus := 'MISMATCH';
                vMismatchCount := vMismatchCount + 1;
            END IF;
            
            DBMS_OUTPUT.PUT_LINE(
                RPAD(vTables(i).source_table, 24) ||
                LPAD(TO_CHAR(vSourceCount, '9,999,999,999'), 15) ||
                LPAD(vHistDisplay, 15) || '   ' ||
                vStatus
            );
        END;
    END LOOP;
    
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE(
        RPAD('TOTALS', 24) ||
        LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
        LPAD(TO_CHAR(vTotalHistCount, '9,999,999,999'), 15)
    );
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE('');
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary (HIST-only strategy)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
    DBMS_OUTPUT.PUT_LINE('Total HIST records: ' || TO_CHAR(vTotalHistCount, '9,999,999,999') || ' (all data in HIST)');
    DBMS_OUTPUT.PUT_LINE('');
    
    IF vMismatchCount = 0 THEN
        DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED');
        DBMS_OUTPUT.PUT_LINE('  All record counts match between source and exported data');
        DBMS_OUTPUT.PUT_LINE('  Export completed successfully');
    ELSE
        DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES');
        DBMS_OUTPUT.PUT_LINE('  Found ' || vMismatchCount || ' table(s) with count mismatches');
        DBMS_OUTPUT.PUT_LINE('  NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)');
        DBMS_OUTPUT.PUT_LINE('  Review export logs and pre-check results before re-running exports');
    END IF;
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('');
    DBMS_OUTPUT.PUT_LINE('Legend:');
    DBMS_OUTPUT.PUT_LINE('  PASS       - Record counts match (export successful)');
    DBMS_OUTPUT.PUT_LINE('  MISMATCH   - Record counts differ (may be pre-existing files or export issue)');
    DBMS_OUTPUT.PUT_LINE('               Check pre-check results to identify pre-existing files');
    DBMS_OUTPUT.PUT_LINE('  ERROR      - Cannot access table (may not exist yet)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('ERROR during record count verification: ' || SQLERRM);
        RAISE;
END;
/

SET FEEDBACK ON;

PROMPT
PROMPT Record count verification completed
PROMPT
