--=============================================================================================================================
-- MARS-835: Export Group 1 - Split DATA + HIST (DEBT, DEBT_DAILY)
--=============================================================================================================================
-- Purpose: Export last 6 months to DATA bucket (CSV), older data to HIST bucket (Parquet)
--          Applies column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
--          Excludes legacy columns not required in new structure
--          USES: DATA_EXPORTER v2.4.0 with pTemplateTableName for column order and date formats
-- Author: Grzegorz Michalski
-- Date: 2025-12-17
-- Updated: 2026-01-11 (Updated to DATA_EXPORTER v2.4.0 with pTemplateTableName)
-- Related: MARS-835 - CSDB Data Export
--=============================================================================================================================

SET SERVEROUTPUT ON SIZE UNLIMITED
SET TIMING ON

DEFINE cutoff_date = "TRUNC(ADD_MONTHS(SYSDATE, -6), 'MM')"

PROMPT ========================================================================
PROMPT Exporting CSDB.DEBT - Split DATA + HIST
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: IDIRDEPOSITORY, VA_BONDDURATION
PROMPT ========================================================================

-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
    vFileCount NUMBER := 0;
    vRecordCount NUMBER := 0;
    vLocationUri VARCHAR2(1000);
BEGIN
    -- Get bucket URI for DATA bucket
    vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT/';
    
    -- Count existing files
    SELECT COUNT(*)
    INTO vFileCount
    FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
        credential_name => 'OCI$RESOURCE_PRINCIPAL',
        location_uri => vLocationUri
    ))
    WHERE object_name NOT LIKE '%/';  -- Exclude directories
    
    IF vFileCount > 0 THEN
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
        DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
        DBMS_OUTPUT.PUT_LINE('');
        
        -- List existing files
        DBMS_OUTPUT.PUT_LINE('Existing files:');
        FOR rec IN (
            SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
            FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
                credential_name => 'OCI$RESOURCE_PRINCIPAL',
                location_uri => vLocationUri
            ))
            WHERE object_name NOT LIKE '%/'
            ORDER BY object_name
        ) LOOP
            DBMS_OUTPUT.PUT_LINE('  - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
        END LOOP;
        
        -- Count records in external table
        BEGIN
            EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_ODS' INTO vRecordCount;
            DBMS_OUTPUT.PUT_LINE('');
            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
            DBMS_OUTPUT.PUT_LINE('>>>');
            DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
            DBMS_OUTPUT.PUT_LINE('>>>');
            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('');
                DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
                DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
        END;
        
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('');
    ELSE
        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
        DBMS_OUTPUT.PUT_LINE('');
    END IF;
END;
/

-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to DATA bucket (last 6 months)...');
    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
    
    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
        pSchemaName => 'OU_CSDB',
        pTableName => 'LEGACY_DEBT',
        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
        pBucketArea => 'DATA',
        pFolderName => 'ODS/CSDB/CSDB_DEBT',
        pMinDate => &cutoff_date,
        pMaxDate => DATE '9999-12-31',  -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
        pParallelDegree => 16,
        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT',
        pMaxFileSize => 104857600,  -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
        pRegisterExport => TRUE,  -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
        pProcessName => 'MARS-835'                 -- Process identifier for tracking
    );
    
    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to DATA bucket with template column order');
END;
/

-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT data to HIST bucket (older than 6 months)...');
    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT');
    
    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
        pSchemaName => 'OU_CSDB',
        pTableName => 'LEGACY_DEBT',
        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
        pBucketArea => 'ARCHIVE',
        pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT',
        pMaxDate => &cutoff_date,
        pParallelDegree => 16,
        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT'
    );
    
    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT exported to HIST bucket with template column order');
END;
/

PROMPT ========================================================================
PROMPT Exporting CSDB.LEGACY_DEBT_DAILY - Split DATA + HIST
PROMPT ========================================================================
PROMPT Last 6 months to DATA bucket (CSV format)
PROMPT Older data to HIST bucket (Parquet with partitioning)
PROMPT Column mapping: A_ETL_LOAD_SET_FK to A_WORKFLOW_HISTORY_KEY
PROMPT Excluded columns: STEPID, PROGRAMNAME, PROGRAMCEILING, PROGRAMSTATUS,
PROMPT                   ISSUERNACE21SECTOR, INSTRUMENTQUOTATIONBASIS
PROMPT ========================================================================

-- PRE-EXPORT CHECK: List existing files and count records
DECLARE
    vFileCount NUMBER := 0;
    vRecordCount NUMBER := 0;
    vLocationUri VARCHAR2(1000);
BEGIN
    -- Get bucket URI for DATA bucket
    vLocationUri := CT_MRDS.FILE_MANAGER.GET_BUCKET_URI('DATA') || 'ODS/CSDB/CSDB_DEBT_DAILY/';
    
    -- Count existing files
    SELECT COUNT(*)
    INTO vFileCount
    FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
        credential_name => 'OCI$RESOURCE_PRINCIPAL',
        location_uri => vLocationUri
    ))
    WHERE object_name NOT LIKE '%/';  -- Exclude directories
    
    IF vFileCount > 0 THEN
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: Files already exist in DATA bucket');
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('Location: ' || vLocationUri);
        DBMS_OUTPUT.PUT_LINE('Files found: ' || vFileCount);
        DBMS_OUTPUT.PUT_LINE('');
        
        -- List existing files
        DBMS_OUTPUT.PUT_LINE('Existing files:');
        FOR rec IN (
            SELECT object_name, bytes, TO_CHAR(last_modified, 'YYYY-MM-DD HH24:MI:SS') AS modified
            FROM TABLE(DBMS_CLOUD.LIST_OBJECTS(
                credential_name => 'OCI$RESOURCE_PRINCIPAL',
                location_uri => vLocationUri
            ))
            WHERE object_name NOT LIKE '%/'
            ORDER BY object_name
        ) LOOP
            DBMS_OUTPUT.PUT_LINE('  - ' || rec.object_name || ' (' || rec.bytes || ' bytes, ' || rec.modified || ')');
        END LOOP;
        
        -- Count records in external table
        BEGIN
            EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ODS.CSDB_DEBT_DAILY_ODS' INTO vRecordCount;
            DBMS_OUTPUT.PUT_LINE('');
            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
            DBMS_OUTPUT.PUT_LINE('>>>');
            DBMS_OUTPUT.PUT_LINE('>>> Records currently readable via external table: ' || vRecordCount);
            DBMS_OUTPUT.PUT_LINE('>>>');
            DBMS_OUTPUT.PUT_LINE('-------------------------------------------------------------------------------');
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('');
                DBMS_OUTPUT.PUT_LINE('WARNING: Cannot count records in external table');
                DBMS_OUTPUT.PUT_LINE('Error: ' || SQLERRM);
        END;
        
        DBMS_OUTPUT.PUT_LINE('===============================================================================');
        DBMS_OUTPUT.PUT_LINE('');
    ELSE
        DBMS_OUTPUT.PUT_LINE('PRE-EXPORT CHECK: No existing files found in DATA bucket - bucket is clean');
        DBMS_OUTPUT.PUT_LINE('');
    END IF;
END;
/

-- Export recent data to DATA bucket (CSV)
-- NEW v2.4.0: Per-column date format handling with template table for column order
BEGIN
    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to DATA bucket (last 6 months)...');
    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
    
    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_TO_CSV_BY_DATE(
        pSchemaName => 'OU_CSDB',
        pTableName => 'LEGACY_DEBT_DAILY',
        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
        pBucketArea => 'DATA',
        pFolderName => 'ODS/CSDB/CSDB_DEBT_DAILY',
        pMinDate => &cutoff_date,
        pMaxDate => DATE '9999-12-31',  -- Include future dates (MAX_LOAD_START can be beyond SYSDATE)
        pParallelDegree => 16,
        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY',
        pMaxFileSize => 104857600,  -- 100MB in bytes (safe for parallel execution, avoids ORA-04036)
        pRegisterExport => TRUE,  -- Register exported files in A_SOURCE_FILE_RECEIVED with metadata (CHECKSUM, CREATED, BYTES)
        pProcessName => 'MARS-835'                 -- Process identifier for tracking
    );
    
    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to DATA bucket with template column order');
END;
/

-- Export historical data to HIST bucket (Parquet)
-- NEW v2.4.0: Per-column date format handling with template table
BEGIN
    DBMS_OUTPUT.PUT_LINE('Exporting LEGACY_DEBT_DAILY data to HIST bucket (older than 6 months)...');
    DBMS_OUTPUT.PUT_LINE('Using Template Table: CT_ET_TEMPLATES.CSDB_DEBT_DAILY');
    
    CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE(
        pSchemaName => 'OU_CSDB',
        pTableName => 'LEGACY_DEBT_DAILY',
        pKeyColumnName => 'A_ETL_LOAD_SET_FK',
        pBucketArea => 'ARCHIVE',
        pFolderName => 'ARCHIVE/CSDB/CSDB_DEBT_DAILY',
        pMaxDate => &cutoff_date,
        pParallelDegree => 16,
        pTemplateTableName => 'CT_ET_TEMPLATES.CSDB_DEBT_DAILY'
    );
    
    DBMS_OUTPUT.PUT_LINE('SUCCESS: LEGACY_DEBT_DAILY exported to HIST bucket with template column order');
END;
/

PROMPT ========================================================================
PROMPT Group 1 Export Completed
PROMPT ========================================================================
PROMPT - LEGACY_DEBT: DATA + HIST exported
PROMPT - LEGACY_DEBT_DAILY: DATA + HIST exported
PROMPT ========================================================================

--=============================================================================================================================
-- End of Script
--=============================================================================================================================
