-- ===================================================================
-- MARS-956 Verify Data Integrity: Source vs Exported Data Validation
-- ===================================================================
-- Purpose: Verify data integrity between source tables and exported files
-- Author: Grzegorz Michalski
-- Date: 2026-02-12

SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON

PROMPT =========================================================================
PROMPT MARS-956 Data Integrity Verification
PROMPT =========================================================================

-- Check 1: Source table record counts vs expected ETL keys
PROMPT Checking source table record counts...
DECLARE
    vAdminRows NUMBER := 0;
    vContentRows NUMBER := 0;
    vCriterionRows NUMBER := 0;
    vTotalRows NUMBER := 0;
    vExpectedFiles NUMBER := 9;  -- 3 tables x 3 ETL keys average
BEGIN
    EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminRows;
    EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentRows;
    EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionRows;
    
    vTotalRows := vAdminRows + vContentRows + vCriterionRows;
    
    DBMS_OUTPUT.PUT_LINE('Source table record counts:');
    DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN: ' || vAdminRows || ' records');
    DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT: ' || vContentRows || ' records');
    DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION: ' || vCriterionRows || ' records');
    DBMS_OUTPUT.PUT_LINE('- TOTAL: ' || vTotalRows || ' records');
    
    IF vTotalRows > 0 THEN
        DBMS_OUTPUT.PUT_LINE('SUCCESS: All source tables contain data');
    ELSE
        DBMS_OUTPUT.PUT_LINE('ERROR: No data found in source tables');
    END IF;
END;
/

-- Check 2: ETL key distribution analysis
PROMPT Checking ETL key distribution...
DECLARE
    vAdminKeys NUMBER := 0;
    vContentKeys NUMBER := 0;
    vCriterionKeys NUMBER := 0;
    vTotalKeys NUMBER := 0;
BEGIN
    EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
    EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
    EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
    
    SELECT COUNT(DISTINCT etl_key)
    INTO vTotalKeys
    FROM (
        SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
    );
    
    DBMS_OUTPUT.PUT_LINE('ETL key distribution:');
    DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct keys: ' || vAdminKeys);
    DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct keys: ' || vContentKeys);
    DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct keys: ' || vCriterionKeys);
    DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys: ' || vTotalKeys);
    
    IF vTotalKeys > 0 THEN
        DBMS_OUTPUT.PUT_LINE('SUCCESS: ETL key distribution looks normal');
    ELSE
        DBMS_OUTPUT.PUT_LINE('ERROR: No ETL keys found in source data');
    END IF;
END;
/

-- Check 3: Template table compatibility verification
PROMPT Checking template table compatibility...
DECLARE
    vAdminCols NUMBER := 0;
    vContentCols NUMBER := 0;
    vCriterionCols NUMBER := 0;
BEGIN
    -- Check MPEC_ADMIN template compatibility
    SELECT COUNT(*)
    INTO vAdminCols
    FROM all_tab_columns
    WHERE owner = 'CT_ET_TEMPLATES'
      AND table_name = 'C2D_MPEC_ADMIN';

    -- Check MPEC_CONTENT template compatibility
    SELECT COUNT(*)
    INTO vContentCols
    FROM all_tab_columns
    WHERE owner = 'CT_ET_TEMPLATES'
      AND table_name = 'C2D_MPEC_CONTENT';

    -- Check MPEC_CONTENT_CRITERION template compatibility
    SELECT COUNT(*)
    INTO vCriterionCols
    FROM all_tab_columns
    WHERE owner = 'CT_ET_TEMPLATES'
      AND table_name = 'C2D_MPEC_CONTENT_CRITERION';

    DBMS_OUTPUT.PUT_LINE('Template table column counts:');
    DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_ADMIN: ' || vAdminCols || ' columns');    
    DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT: ' || vContentCols || ' columns');
    DBMS_OUTPUT.PUT_LINE('- C2D_MPEC_CONTENT_CRITERION: ' || vCriterionCols || ' columns');

    IF vAdminCols > 0 AND vContentCols > 0 AND vCriterionCols > 0 THEN
        DBMS_OUTPUT.PUT_LINE('SUCCESS: All template tables have defined structure');
    ELSE
        DBMS_OUTPUT.PUT_LINE('ERROR: One or more template tables missing columns');
    END IF;
END;
/

-- Check 4: Verify A_ETL_LOAD_SET_FK values exist in A_LOAD_HISTORY
PROMPT Checking ETL key references in A_LOAD_HISTORY...
DECLARE
    vValidKeys NUMBER := 0;
    vTotalSourceKeys NUMBER := 0;
BEGIN
    -- Count total distinct ETL keys in source tables
    SELECT COUNT(DISTINCT etl_key)
    INTO vTotalSourceKeys
    FROM (
        SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
    );

    -- Count how many exist in A_LOAD_HISTORY
    SELECT COUNT(DISTINCT etl_key)
    INTO vValidKeys
    FROM (
        SELECT A_ETL_LOAD_SET_FK AS etl_key FROM OU_LEGACY_C2D.MPEC_ADMIN
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT
        UNION
        SELECT A_ETL_LOAD_SET_FK FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION
    ) src
    WHERE EXISTS (
        SELECT 1 FROM CT_ODS.A_LOAD_HISTORY h
        WHERE h.A_ETL_LOAD_SET_KEY = src.etl_key
    );

    DBMS_OUTPUT.PUT_LINE('ETL key validation:');
    DBMS_OUTPUT.PUT_LINE('- Total distinct ETL keys in source: ' || vTotalSourceKeys);
    DBMS_OUTPUT.PUT_LINE('- Valid keys (exist in A_LOAD_HISTORY): ' || vValidKeys);

    IF vValidKeys = vTotalSourceKeys THEN
        DBMS_OUTPUT.PUT_LINE('SUCCESS: All source ETL keys are valid');
    ELSE
        DBMS_OUTPUT.PUT_LINE('ERROR: Some ETL keys may be invalid: ' || (vTotalSourceKeys - vValidKeys));
    END IF;
END;
/

PROMPT =====================================================================================
PROMPT MARS-956 Record Count Verification
PROMPT =====================================================================================
PROMPT Comparing source table counts with exported external table counts
PROMPT =====================================================================================

DECLARE
    TYPE t_table_info IS RECORD (
        source_schema VARCHAR2(50),
        source_table VARCHAR2(100),
        external_table VARCHAR2(100),
        description VARCHAR2(200)
    );
    TYPE t_table_list IS TABLE OF t_table_info;
    
    vTables t_table_list;
    vSourceCount NUMBER;
    vTargetCount NUMBER;
    vTotalSourceCount NUMBER := 0;
    vTotalTargetCount NUMBER := 0;
    vMismatchCount NUMBER := 0;
    vSql VARCHAR2(4000);
    vFileCount NUMBER := 0;
    vValidationResult VARCHAR2(100);
BEGIN
    DBMS_OUTPUT.PUT_LINE('VERIFICATION TIME: ' || TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS'));
    DBMS_OUTPUT.PUT_LINE('');
    
    -- Initialize table list with C2D MPEC configuration
    vTables := t_table_list(
        t_table_info('OU_LEGACY_C2D', 'MPEC_ADMIN', 'ODS.C2D_MPEC_ADMIN_ODS', 'MPEC Admin data (ETL keys 2001-2005)'),
        t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT', 'ODS.C2D_MPEC_CONTENT_ODS', 'MPEC Content data (ETL keys 2006-2008)'),
        t_table_info('OU_LEGACY_C2D', 'MPEC_CONTENT_CRITERION', 'ODS.C2D_MPEC_CONTENT_CRITERION_ODS', 'MPEC Criterion data (ETL keys 2009-2010)')
    );
    
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE('Table Name                Source Count   Target Count   Status');
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    
    FOR i IN 1..vTables.COUNT LOOP
        -- Get source table count
        vSql := 'SELECT COUNT(*) FROM ' || vTables(i).source_schema || '.' || vTables(i).source_table;
        
        BEGIN
            EXECUTE IMMEDIATE vSql INTO vSourceCount;
            vTotalSourceCount := vTotalSourceCount + vSourceCount;
        EXCEPTION
            WHEN OTHERS THEN
                vSourceCount := -1;
                DBMS_OUTPUT.PUT_LINE(RPAD(vTables(i).source_table, 24) || 'ERROR: Cannot access source table');
                CONTINUE;
        END;
        
        -- Get target external table count
        vSql := 'SELECT COUNT(*) FROM ' || vTables(i).external_table;
        BEGIN
            EXECUTE IMMEDIATE vSql INTO vTargetCount;
            vTotalTargetCount := vTotalTargetCount + vTargetCount;
        EXCEPTION
            WHEN OTHERS THEN
                -- Handle expected errors for empty external tables
                -- ORA-29913: error in executing ODCIEXTTABLEOPEN callout
                -- ORA-29400: data cartridge error
                -- KUP-13023: nothing matched wildcard query (no files in bucket)
                -- NOTE: ORA-30653 (reject limit) is a real data quality error, not treated as empty
                IF vSourceCount = 0 OR SQLCODE IN (-29913, -29400) OR SQLERRM LIKE '%KUP-13023%' THEN
                    vTargetCount := 0;  -- Treat as empty (no files exported yet)
                ELSE
                    vTargetCount := -1;  -- Real error
                END IF;
        END;
        
        -- Display comparison results with thousands separators
        DECLARE
            vStatus VARCHAR2(20);
            vSourceDisplay VARCHAR2(17);
            vTargetDisplay VARCHAR2(17);
        BEGIN
            -- Format source count display
            IF vSourceCount = -1 THEN
                vSourceDisplay := 'ERROR';
            ELSE
                vSourceDisplay := TO_CHAR(vSourceCount, '9,999,999,999');
            END IF;
            
            -- Format target count display
            IF vTargetCount = -1 THEN
                vTargetDisplay := 'ERROR';
            ELSE
                vTargetDisplay := TO_CHAR(vTargetCount, '9,999,999,999');
            END IF;
            
            -- Determine status
            IF vSourceCount = vTargetCount THEN
                vStatus := 'PASS';
            ELSIF vTargetCount = -1 THEN
                vStatus := 'ERROR';
                vMismatchCount := vMismatchCount + 1;
            ELSIF vSourceCount = -1 THEN
                vStatus := 'ERROR';
                vMismatchCount := vMismatchCount + 1;
            ELSE
                vStatus := 'MISMATCH';
                vMismatchCount := vMismatchCount + 1;
            END IF;
            
            DBMS_OUTPUT.PUT_LINE(
                RPAD(vTables(i).source_table, 24) ||
                LPAD(vSourceDisplay, 15) ||
                LPAD(vTargetDisplay, 15) || '   ' ||
                vStatus
            );
        END;
    END LOOP;
    
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE(
        RPAD('TOTALS', 24) ||
        LPAD(TO_CHAR(vTotalSourceCount, '9,999,999,999'), 15) ||
        LPAD(TO_CHAR(vTotalTargetCount, '9,999,999,999'), 15)
    );
    DBMS_OUTPUT.PUT_LINE('-----------------------------------------------------------------------------------------');
    DBMS_OUTPUT.PUT_LINE('');
    
    -- Count exported files for additional verification
    SELECT COUNT(*)
    INTO vFileCount
    FROM CT_MRDS.A_SOURCE_FILE_RECEIVED
    WHERE RECEPTION_DATE >= SYSDATE - 1/24
      AND (SOURCE_FILE_NAME LIKE '200_%');
    
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Record Count Verification Summary');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('Total source records: ' || TO_CHAR(vTotalSourceCount, '9,999,999,999'));
    DBMS_OUTPUT.PUT_LINE('Total target records: ' || TO_CHAR(vTotalTargetCount, '9,999,999,999') || ' (exported to ODS)');
    DBMS_OUTPUT.PUT_LINE('Export files registered: ' || vFileCount);
    DBMS_OUTPUT.PUT_LINE('');
    
    IF vMismatchCount = 0 AND vFileCount > 0 THEN
        DBMS_OUTPUT.PUT_LINE('[PASS] VERIFICATION PASSED');
        DBMS_OUTPUT.PUT_LINE('  All record counts match between source and exported data');
        DBMS_OUTPUT.PUT_LINE('  Export completed successfully');
    ELSIF vMismatchCount > 0 THEN
        DBMS_OUTPUT.PUT_LINE('[INFO] VERIFICATION COMPLETED WITH MISMATCHES');
        DBMS_OUTPUT.PUT_LINE('  Found ' || vMismatchCount || ' table(s) with count mismatches');
        DBMS_OUTPUT.PUT_LINE('  NOTE: Mismatches may be caused by pre-existing files in buckets (see pre-check)');
        DBMS_OUTPUT.PUT_LINE('  Review export logs and pre-check results before re-running exports');
    ELSE
        DBMS_OUTPUT.PUT_LINE('[WARN] NO EXPORT DETECTED');
        DBMS_OUTPUT.PUT_LINE('  No files found in export registration');
        DBMS_OUTPUT.PUT_LINE('  Verify export execution completed successfully');
    END IF;
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    DBMS_OUTPUT.PUT_LINE('');
    
    DBMS_OUTPUT.PUT_LINE('Legend:');
    DBMS_OUTPUT.PUT_LINE('  PASS       - Record counts match (export successful)');
    DBMS_OUTPUT.PUT_LINE('  MISMATCH   - Record counts differ (may be pre-existing files or export issue)');
    DBMS_OUTPUT.PUT_LINE('               Check pre-check results to identify pre-existing files');
    DBMS_OUTPUT.PUT_LINE('  ERROR      - Cannot access table (verify table exists and permissions)');
    DBMS_OUTPUT.PUT_LINE('=====================================================================================');
    
    -- Additional ETL key analysis for C2D MPEC data
    DBMS_OUTPUT.PUT_LINE('');
    DBMS_OUTPUT.PUT_LINE('ETL Key Analysis:');
    
    DECLARE
        vAdminKeys NUMBER;
        vContentKeys NUMBER;
        vCriterionKeys NUMBER;
    BEGIN
        EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_ADMIN' INTO vAdminKeys;
        EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT' INTO vContentKeys;
        EXECUTE IMMEDIATE 'SELECT COUNT(DISTINCT A_ETL_LOAD_SET_FK) FROM OU_LEGACY_C2D.MPEC_CONTENT_CRITERION' INTO vCriterionKeys;
        
        DBMS_OUTPUT.PUT_LINE('- MPEC_ADMIN distinct ETL keys: ' || vAdminKeys || ' (expected: 3 for keys 2001-2005)');
        DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT distinct ETL keys: ' || vContentKeys || ' (expected: 3 for keys 2006-2008)');
        DBMS_OUTPUT.PUT_LINE('- MPEC_CONTENT_CRITERION distinct ETL keys: ' || vCriterionKeys || ' (expected: 2 for keys 2009-2010)');
        
        -- Expected file count = sum of distinct ETL keys per table
        DBMS_OUTPUT.PUT_LINE('- Expected export files: ' || (vAdminKeys + vContentKeys + vCriterionKeys));
        DBMS_OUTPUT.PUT_LINE('- Actual export files: ' || vFileCount);
    END;
END;
/

PROMPT =========================================================================
PROMPT Data Integrity Verification Completed
PROMPT =========================================================================