-- ============================================================================
-- MARS-1409 Diagnostic: Workflow key status after step 09
-- ============================================================================
-- Purpose: For each INPUT config with an ODS table, report:
--   [A] Files present in ODS bucket but NOT registered in A_SOURCE_FILE_RECEIVED
--   [B] Files registered in A_SOURCE_FILE_RECEIVED but NOT in ODS bucket
--   [C] Files present in both - with A_WORKFLOW_HISTORY_KEY populated
--   [D] Files present in both - A_WORKFLOW_HISTORY_KEY still NULL
--
-- Can be run at any time, read-only (no DML).
-- ============================================================================

SET SERVEROUTPUT ON SIZE UNLIMITED
SET LINESIZE 200

PROMPT
PROMPT ============================================================================
PROMPT Diagnosing workflow key status (ODS bucket vs A_SOURCE_FILE_RECEIVED)
PROMPT ============================================================================
PROMPT

DECLARE
    TYPE tStringList IS TABLE OF VARCHAR2(500);

    vTableName           VARCHAR2(200);
    vTableExists         NUMBER;
    vBucketEmpty         BOOLEAN;
    vRefCursor           SYS_REFCURSOR;
    vFileName            VARCHAR2(500);

    -- Per-config counters
    vOnlyInBucket        NUMBER;
    vOnlyInDB            NUMBER;
    vInBothWithKey       NUMBER;
    vInBothNoKey         NUMBER;

    -- Grand totals
    vConfigsChecked      NUMBER := 0;
    vConfigsWithIssues   NUMBER := 0;
    vTotalOnlyInBucket   NUMBER := 0;
    vTotalOnlyInDB       NUMBER := 0;
    vTotalInBothWithKey  NUMBER := 0;
    vTotalInBothNoKey    NUMBER := 0;

    -- How many individual file names to print per category before summarising
    cMaxPrint CONSTANT NUMBER := 1000;
    vPrinted  NUMBER;

    FUNCTION IS_EXTERNAL_TABLE_EMPTY_ERROR(
        pSqlCode NUMBER,
        pSqlErrm VARCHAR2
    ) RETURN BOOLEAN
    IS
    BEGIN
        RETURN pSqlCode IN (-29913, -29400)
           OR INSTR(pSqlErrm, 'KUP-05002') > 0;
    END;

BEGIN

    FOR config_rec IN (
        SELECT sfc.A_SOURCE_FILE_CONFIG_KEY,
               sfc.A_SOURCE_KEY,
               sfc.SOURCE_FILE_ID,
               sfc.TABLE_ID
        FROM   CT_MRDS.A_SOURCE_FILE_CONFIG sfc
        WHERE  sfc.SOURCE_FILE_TYPE = 'INPUT'
          AND  sfc.TABLE_ID         IS NOT NULL
        ORDER BY sfc.A_SOURCE_KEY, sfc.SOURCE_FILE_ID, sfc.TABLE_ID
    ) LOOP

        vTableName := 'ODS.' || config_rec.TABLE_ID || '_ODS';

        SELECT COUNT(*) INTO vTableExists
        FROM   ALL_TABLES
        WHERE  OWNER      = 'ODS'
          AND  TABLE_NAME = config_rec.TABLE_ID || '_ODS';

        IF vTableExists = 0 THEN
            CONTINUE;
        END IF;

        -- Check if the bucket location has any files at all
        -- (empty bucket raises ORA-29913 instead of returning 0 rows)
        vBucketEmpty := FALSE;
        BEGIN
            EXECUTE IMMEDIATE
                'SELECT COUNT(*) FROM ' || vTableName || ' t WHERE ROWNUM = 1'
            INTO vTableExists;
        EXCEPTION
            WHEN OTHERS THEN
                IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
                    vBucketEmpty := TRUE;
                ELSE
                    RAISE;
                END IF;
        END;

        IF vBucketEmpty THEN
            -- Bucket is empty: nothing in ODS, but registered records are all "not in bucket"
            vOnlyInBucket := 0;
            SELECT COUNT(*) INTO vOnlyInDB
            FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
            WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
              AND  sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
            vInBothWithKey := 0;
            vInBothNoKey   := 0;
        ELSE

        BEGIN
            -- ----------------------------------------------------------------
            -- [A] In ODS bucket but NOT in A_SOURCE_FILE_RECEIVED
            -- ----------------------------------------------------------------
            EXECUTE IMMEDIATE
                'SELECT COUNT(DISTINCT t.file$name) ' ||
                'FROM   ' || vTableName || ' t ' ||
                'WHERE  t.file$name IS NOT NULL ' ||
                '  AND  NOT EXISTS ( ' ||
                '       SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                '       WHERE  sfr.SOURCE_FILE_NAME          = t.file$name ' ||
                '         AND  sfr.A_SOURCE_FILE_CONFIG_KEY  = :1)'
            INTO vOnlyInBucket
            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;

            -- ----------------------------------------------------------------
            -- [B] In A_SOURCE_FILE_RECEIVED (targeted statuses) but NOT in ODS bucket
            -- ----------------------------------------------------------------
            EXECUTE IMMEDIATE
                'SELECT COUNT(*) ' ||
                'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
                '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                '  AND  NOT EXISTS ( ' ||
                '       SELECT 1 FROM ' || vTableName || ' t ' ||
                '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME)'
            INTO vOnlyInDB
            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;

            -- ----------------------------------------------------------------
            -- [C] In both, A_WORKFLOW_HISTORY_KEY IS NOT NULL
            -- ----------------------------------------------------------------
            EXECUTE IMMEDIATE
                'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
                'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY    = :1 ' ||
                '  AND  sfr.A_WORKFLOW_HISTORY_KEY       IS NOT NULL ' ||
                '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                '  AND  EXISTS ( ' ||
                '       SELECT 1 FROM ' || vTableName || ' t ' ||
                '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME)'
            INTO vInBothWithKey
            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;

            -- ----------------------------------------------------------------
            -- [D] In both, A_WORKFLOW_HISTORY_KEY IS NULL
            -- ----------------------------------------------------------------
            EXECUTE IMMEDIATE
                'SELECT COUNT(DISTINCT sfr.A_SOURCE_FILE_RECEIVED_KEY) ' ||
                'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY    = :1 ' ||
                '  AND  sfr.A_WORKFLOW_HISTORY_KEY       IS NULL ' ||
                '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                '  AND  EXISTS ( ' ||
                '       SELECT 1 FROM ' || vTableName || ' t ' ||
                '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME)'
            INTO vInBothNoKey
            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
        EXCEPTION
            WHEN OTHERS THEN
                IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
                    vBucketEmpty := TRUE;
                    vOnlyInBucket := 0;
                    SELECT COUNT(*) INTO vOnlyInDB
                    FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr
                    WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = config_rec.A_SOURCE_FILE_CONFIG_KEY
                      AND  sfr.PROCESSING_STATUS IN ('VALIDATED','READY_FOR_INGESTION','INGESTED','ARCHIVED','ARCHIVED_AND_TRASHED','ARCHIVED_AND_PURGED');
                    vInBothWithKey := 0;
                    vInBothNoKey   := 0;
                    DBMS_OUTPUT.PUT_LINE('  NOTE: ODS bucket became empty/inaccessible during diagnostics for ' || vTableName || '. Falling back to DB-only counts for [B].');
                ELSE
                    RAISE;
                END IF;
        END;

        END IF; -- vBucketEmpty

        -- Skip configs with nothing to report
        IF vOnlyInBucket = 0 AND vOnlyInDB = 0 AND vInBothWithKey = 0 AND vInBothNoKey = 0 THEN
            CONTINUE;
        END IF;

        vConfigsChecked := vConfigsChecked + 1;

        DBMS_OUTPUT.PUT_LINE('Config ' || config_rec.A_SOURCE_FILE_CONFIG_KEY ||
                           ' (' || config_rec.A_SOURCE_KEY ||
                           '/' || config_rec.SOURCE_FILE_ID ||
                           '/' || config_rec.TABLE_ID || ')');
        DBMS_OUTPUT.PUT_LINE('  [A] In bucket, not registered:          ' || vOnlyInBucket);
        DBMS_OUTPUT.PUT_LINE('  [B] Registered, not in bucket:          ' || vOnlyInDB);
        DBMS_OUTPUT.PUT_LINE('  [C] In both, A_WORKFLOW_HISTORY_KEY set: ' || vInBothWithKey);
        DBMS_OUTPUT.PUT_LINE('  [D] In both, A_WORKFLOW_HISTORY_KEY NULL: ' || vInBothNoKey);

        -- Print individual file names for categories with problems
        IF vOnlyInBucket > 0 THEN
            DBMS_OUTPUT.PUT_LINE('  [A] Files in bucket not registered:');
            vPrinted := 0;
            OPEN vRefCursor FOR
                'SELECT DISTINCT t.file$name ' ||
                'FROM   ' || vTableName || ' t ' ||
                'WHERE  t.file$name IS NOT NULL ' ||
                '  AND  NOT EXISTS ( ' ||
                '       SELECT 1 FROM CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                '       WHERE  sfr.SOURCE_FILE_NAME         = t.file$name ' ||
                '         AND  sfr.A_SOURCE_FILE_CONFIG_KEY = :1) ' ||
                'ORDER BY t.file$name'
                USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
            LOOP
                FETCH vRefCursor INTO vFileName;
                EXIT WHEN vRefCursor%NOTFOUND;
                vPrinted := vPrinted + 1;
                IF vPrinted <= cMaxPrint THEN
                    DBMS_OUTPUT.PUT_LINE('      ' || vFileName);
                ELSIF vPrinted = cMaxPrint + 1 THEN
                    DBMS_OUTPUT.PUT_LINE('      ... and ' || (vOnlyInBucket - cMaxPrint) || ' more');
                END IF;
            END LOOP;
            CLOSE vRefCursor;
        END IF;

        IF vOnlyInDB > 0 THEN
            vConfigsWithIssues := vConfigsWithIssues + 1;
            DBMS_OUTPUT.PUT_LINE('  [B] Registered files not found in bucket:');
            vPrinted := 0;
            BEGIN
                IF vBucketEmpty THEN
                    OPEN vRefCursor FOR
                        'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
                        'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                        'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
                        '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                        'ORDER BY sfr.SOURCE_FILE_NAME'
                        USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
                ELSE
                    OPEN vRefCursor FOR
                        'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
                        'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                        'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
                        '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                        '  AND  NOT EXISTS ( ' ||
                        '       SELECT 1 FROM ' || vTableName || ' t ' ||
                        '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME) ' ||
                        'ORDER BY sfr.SOURCE_FILE_NAME'
                        USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
                END IF;
            EXCEPTION
                WHEN OTHERS THEN
                    IF IS_EXTERNAL_TABLE_EMPTY_ERROR(SQLCODE, SQLERRM) THEN
                        vBucketEmpty := TRUE;
                        DBMS_OUTPUT.PUT_LINE('      NOTE: Skipping ODS anti-join details due to empty/inaccessible external table for ' || vTableName || '.');
                        OPEN vRefCursor FOR
                            'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS, sfr.A_WORKFLOW_HISTORY_KEY ' ||
                            'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                            'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY = :1 ' ||
                            '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                            'ORDER BY sfr.SOURCE_FILE_NAME'
                            USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
                    ELSE
                        RAISE;
                    END IF;
            END;
            LOOP
                DECLARE
                    vStatus  VARCHAR2(50);
                    vWfKey   NUMBER;
                BEGIN
                    FETCH vRefCursor INTO vFileName, vStatus, vWfKey;
                    EXIT WHEN vRefCursor%NOTFOUND;
                    vPrinted := vPrinted + 1;
                    IF vPrinted <= cMaxPrint THEN
                        DBMS_OUTPUT.PUT_LINE('      ' || vFileName ||
                                           '  status=' || vStatus ||
                                           '  wf_key=' || NVL(TO_CHAR(vWfKey), 'NULL'));
                    ELSIF vPrinted = cMaxPrint + 1 THEN
                        DBMS_OUTPUT.PUT_LINE('      ... and ' || (vOnlyInDB - cMaxPrint) || ' more');
                    END IF;
                END;
            END LOOP;
            CLOSE vRefCursor;
        END IF;

        IF vInBothNoKey > 0 THEN
            vConfigsWithIssues := vConfigsWithIssues + 1;
            DBMS_OUTPUT.PUT_LINE('  [D] Files in both but A_WORKFLOW_HISTORY_KEY still NULL:');
            vPrinted := 0;
            OPEN vRefCursor FOR
                'SELECT sfr.SOURCE_FILE_NAME, sfr.PROCESSING_STATUS ' ||
                'FROM   CT_MRDS.A_SOURCE_FILE_RECEIVED sfr ' ||
                'WHERE  sfr.A_SOURCE_FILE_CONFIG_KEY   = :1 ' ||
                '  AND  sfr.A_WORKFLOW_HISTORY_KEY      IS NULL ' ||
                '  AND  sfr.PROCESSING_STATUS IN (''VALIDATED'',''READY_FOR_INGESTION'',''INGESTED'',''ARCHIVED'',''ARCHIVED_AND_TRASHED'',''ARCHIVED_AND_PURGED'') ' ||
                '  AND  EXISTS ( ' ||
                '       SELECT 1 FROM ' || vTableName || ' t ' ||
                '       WHERE  t.file$name = sfr.SOURCE_FILE_NAME) ' ||
                'ORDER BY sfr.SOURCE_FILE_NAME'
                USING config_rec.A_SOURCE_FILE_CONFIG_KEY;
            LOOP
                DECLARE
                    vStatus VARCHAR2(50);
                BEGIN
                    FETCH vRefCursor INTO vFileName, vStatus;
                    EXIT WHEN vRefCursor%NOTFOUND;
                    vPrinted := vPrinted + 1;
                    IF vPrinted <= cMaxPrint THEN
                        DBMS_OUTPUT.PUT_LINE('      ' || vFileName || '  status=' || vStatus);
                    ELSIF vPrinted = cMaxPrint + 1 THEN
                        DBMS_OUTPUT.PUT_LINE('      ... and ' || (vInBothNoKey - cMaxPrint) || ' more');
                    END IF;
                END;
            END LOOP;
            CLOSE vRefCursor;
        END IF;

        DBMS_OUTPUT.PUT_LINE('');

        -- Accumulate totals
        vTotalOnlyInBucket  := vTotalOnlyInBucket  + vOnlyInBucket;
        vTotalOnlyInDB      := vTotalOnlyInDB      + vOnlyInDB;
        vTotalInBothWithKey := vTotalInBothWithKey + vInBothWithKey;
        vTotalInBothNoKey   := vTotalInBothNoKey   + vInBothNoKey;

    END LOOP;

    DBMS_OUTPUT.PUT_LINE('============================================================================');
    DBMS_OUTPUT.PUT_LINE('Grand Summary:');
    DBMS_OUTPUT.PUT_LINE('  Configs with data checked:               ' || vConfigsChecked);
    DBMS_OUTPUT.PUT_LINE('  Configs with issues (B or D):            ' || vConfigsWithIssues);
    DBMS_OUTPUT.PUT_LINE('  [A] Files in bucket, not registered:     ' || vTotalOnlyInBucket);
    DBMS_OUTPUT.PUT_LINE('  [B] Registered, not in bucket:           ' || vTotalOnlyInDB);
    DBMS_OUTPUT.PUT_LINE('  [C] In both - A_WORKFLOW_HISTORY_KEY set: ' || vTotalInBothWithKey);
    DBMS_OUTPUT.PUT_LINE('  [D] In both - A_WORKFLOW_HISTORY_KEY NULL: ' || vTotalInBothNoKey);
    DBMS_OUTPUT.PUT_LINE('============================================================================');

    IF vTotalOnlyInDB > 0 THEN
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('WARNING [B]: ' || vTotalOnlyInDB || ' registered file(s) not found in ODS bucket.');
        DBMS_OUTPUT.PUT_LINE('            These may have been moved to ARCHIVE or deleted from ODS.');
    END IF;

    IF vTotalInBothNoKey > 0 THEN
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('WARNING [D]: ' || vTotalInBothNoKey || ' file(s) present in both but A_WORKFLOW_HISTORY_KEY is still NULL.');
        DBMS_OUTPUT.PUT_LINE('            ODS table rows for these files may have A_WORKFLOW_HISTORY_KEY = NULL.');
        DBMS_OUTPUT.PUT_LINE('            Re-run step 09 after the ODS rows are populated by the pipeline.');
    END IF;

    IF vConfigsWithIssues = 0 THEN
        DBMS_OUTPUT.PUT_LINE('');
        DBMS_OUTPUT.PUT_LINE('OK: No issues found. All registered files in ODS have A_WORKFLOW_HISTORY_KEY assigned.');
    END IF;

EXCEPTION
    WHEN OTHERS THEN
        IF vRefCursor%ISOPEN THEN
            CLOSE vRefCursor;
        END IF;
        DBMS_OUTPUT.PUT_LINE('ERROR: ' || SQLERRM);
        RAISE;
END;
/

PROMPT
PROMPT Diagnosis complete.
PROMPT
