--MARS-757

set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;

column owner format a15
column object_name format a30
column spoolname new_value spoolname 

select 'install_package_MarsDEVOreplicator_MARS757_package_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'


prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;

--set echo on
--=============================================================================================================================
-- start install --MARS-757 create user
--=============================================================================================================================
CREATE OR REPLACE EDITIONABLE PACKAGE "MRDS_LOADER"."DATA_REPLICATOR" IS
   --Error handling function with detailed error information
   FUNCTION get_error_stack(
         p_format  VARCHAR2 DEFAULT 'OUTPUT'
      ) RETURN VARCHAR2;
   PROCEDURE export_data_by_rowid(
    p_service_name    IN VARCHAR2,
    p_table_owner     IN VARCHAR2,
    p_table_name      IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_start_rowid     IN ROWID,
    p_end_rowid       IN ROWID
); 
    PROCEDURE export_table(
        p_service_name IN VARCHAR2,
        p_table_owner IN VARCHAR2,
        p_table_name IN VARCHAR2,
        p_objectstore_uri IN VARCHAR2
    );
    PROCEDURE list_s3_bucket_objects(
        p_service_name     IN VARCHAR2,
        p_owner_name       IN VARCHAR2,
        p_table_name       IN VARCHAR2,
        p_objectstore_uri  IN VARCHAR2
    );
     PROCEDURE clear_s3_bucket(
        p_service_name IN VARCHAR2,
        p_table_owner IN VARCHAR2,
        p_table_name IN VARCHAR2,
        p_objectstore_uri IN VARCHAR2
    );
      PROCEDURE export_data(
        p_service_name    IN VARCHAR2,
        p_table_owner     IN VARCHAR2,
        p_table_name      IN VARCHAR2,
        p_objectstore_uri IN VARCHAR2,
        p_date_column     IN VARCHAR2,
        p_start           IN NUMBER,
        p_end             IN NUMBER
      );
    PROCEDURE start_log_table (
        p_run_id IN VARCHAR2,
        p_service_name IN VARCHAR2,
        p_table_owner IN VARCHAR2,
        p_table_name IN VARCHAR2
    );
    PROCEDURE end_log_table (
        p_service_name IN VARCHAR2,
        p_table_owner IN VARCHAR2,
        p_table_name IN VARCHAR2
    );
    PROCEDURE end_log_table_generator(
        p_service_name IN VARCHAR2,
        p_table_owner IN VARCHAR2,
        p_table_name IN VARCHAR2,
        p_status IN VARCHAR2
    );
END DATA_REPLICATOR;
/

CREATE OR REPLACE EDITIONABLE PACKAGE BODY "MRDS_LOADER"."DATA_REPLICATOR" IS
  -- Credential name to access Object Storage (adjust or parameterize as needed).
  c_credential CONSTANT VARCHAR2(100) := 'DEVO_CRED_ARN';
--  c_credential CONSTANT VARCHAR2(100) := 'OCI$RESOURCE_PRINCIPAL';

  -- Hive partition keys used to organize exported files (adjust as needed).
  c_hive_partition_keys CONSTANT VARCHAR2(100) := 'YEAR_COL,MONTH_COL';

  -- Constants for formatting
  cgBL CONSTANT VARCHAR2(2) := CHR(13) || CHR(10);

   -- Error handling function with detailed error information
   FUNCTION get_error_stack(
         p_format  VARCHAR2 DEFAULT 'OUTPUT'
      ) RETURN VARCHAR2
      IS
         l_full_error_core VARCHAR2(32000);
         l_full_error_msg  VARCHAR2(32000);
         l_timestamp       VARCHAR2(30);
      BEGIN
         -- Get current timestamp
         SELECT TO_CHAR(SYSTIMESTAMP, 'YYYY-MM-DD HH24:MI:SS.FF3') 
         INTO l_timestamp 
         FROM DUAL;

         l_full_error_core := 'Error Timestamp: ' || l_timestamp || cgBL
                        || 'Session User: ' || USER || cgBL
                        || 'SQL Error Code: ' || SQLCODE || cgBL
                        || 'Error Message: ' || SQLERRM || cgBL
                        || '-------------------------------------------------------'
                        || cgBL || 'Error Stack:'
                        || cgBL || DBMS_UTILITY.FORMAT_ERROR_STACK
                        || '-------------------------------------------------------'
                        || cgBL || 'Error Backtrace:'
                        || cgBL || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE;

         IF (p_format = 'OUTPUT') THEN
            l_full_error_msg := cgBL || '------------------------------------------------------+'
                           || cgBL || l_full_error_core
                           || '------------------------------------------------------+';
         ELSIF (p_format = 'TABLE') THEN
            l_full_error_msg := l_full_error_core;
         END IF;

         RETURN l_full_error_msg;
      END get_error_stack;


-------------------------------------------------------------------------------------------
  -------------------------------------------------------------------------
  -- Procedure: export_data
  -- 
  -- Exports data for a single partition (determined by p_start and p_end values)
  -- from the specified table to a file in Object Storage.
  --
  -- p_date_column is used to compute the partition value and filter records in this partition.
    -------------------------------------------------------------------------
    
PROCEDURE export_data_by_rowid(
    p_service_name    IN VARCHAR2,
    p_table_owner IN VARCHAR2,
    p_table_name IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_start_rowid IN ROWID,
    p_end_rowid IN ROWID
) AS
    l_query CLOB; -- Changed to CLOB to handle long queries
    l_file_uri VARCHAR2(2000);
    l_table_alias  VARCHAR2(100);
    l_log_table    VARCHAR2(1000); -- Added log table


BEGIN
    EXECUTE IMMEDIATE 'ALTER SESSION SET NLS_DATE_FORMAT = ''YYYY-MM-DD HH24:MI:SS''';
    -- Generate the file URI for the current chunk
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
    ELSE
        RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
    END IF;

    BEGIN
    EXECUTE IMMEDIATE 'SELECT TABLE_ALIAS FROM ' || l_log_table ||
                          ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
          INTO l_table_alias
          USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No TABLE_ALIAS found for table: ' ||
                                              p_table_owner || '.' || p_table_name);
    END;
    l_file_uri := p_objectstore_uri || UPPER(l_table_alias) || UPPER('/PARTITION_by_ROWID') || '_' || '.parquet';

    -- Start building the dynamic SQL query
    l_query := 'SELECT ';

    -- Iterate through all columns in the table and dynamically build the column list
FOR col IN (
    SELECT column_name, data_type, data_precision
    FROM all_tab_columns
    WHERE table_name = UPPER(p_table_name)
      AND owner = UPPER(p_table_owner)
    ORDER BY column_id
) LOOP
    -- Obsługa różnych typów danych
    IF col.data_type LIKE 'TIMESTAMP%' OR col.data_type LIKE 'DATE%' THEN
        l_query := l_query || 'TO_CHAR(' || col.column_name || ', ''YYYY-MM-DD HH24:MI:SS'') AS ' || col.column_name || ',';
    ELSIF col.data_type = 'FLOAT' THEN
        l_query := l_query || 'TO_CHAR(' || col.column_name || ') AS ' || col.column_name || ',';
    ELSIF col.data_type = 'NUMBER' AND col.data_precision IS NULL THEN
        l_query := l_query || 'TO_CHAR(' || col.column_name || ') AS ' || col.column_name || ',';
    ELSE
        l_query := l_query || col.column_name || ',';
    END IF;
END LOOP;

    -- Remove the trailing comma from the column list
    l_query := SUBSTR(l_query, 1, LENGTH(l_query) - 1);

    -- Append the FROM and WHERE clauses to the query
    l_query := l_query || ' FROM ' || UPPER(p_table_owner) || '.' || UPPER(p_table_name) ||
               ' WHERE ROWID BETWEEN ''' || p_start_rowid || ''' AND ''' || p_end_rowid || '''';

    -- Export the data using DBMS_CLOUD
    DBMS_CLOUD.EXPORT_DATA(
        credential_name => c_credential,
        file_uri_list   => l_file_uri,
        format          => json_object('type' VALUE 'parquet'),
        query           => l_query
    );

    -- Log success message
    DBMS_OUTPUT.PUT_LINE('Data exported successfully for ROWID range: ' || p_start_rowid || ' to ' || p_end_rowid);

EXCEPTION
    WHEN OTHERS THEN
        -- Handle errors
        DBMS_OUTPUT.PUT_LINE('Error in export_data_by_rowid: ' || get_error_stack());
        RAISE;
END export_data_by_rowid;


-------------------------------------------------------------------------------------------
PROCEDURE clear_s3_bucket( 
    p_service_name IN VARCHAR2, 
    p_table_owner IN VARCHAR2, 
    p_table_name IN VARCHAR2, 
    p_objectstore_uri IN VARCHAR2 
) IS
    l_log_table VARCHAR2(1000);
    l_save_mode VARCHAR2(50);
    l_files_to_delete SYS_REFCURSOR;
    l_file_name VARCHAR2(4000);
    l_last_processed_key NUMBER;
    l_partition_column VARCHAR2(100);
    l_partition_column_type VARCHAR2(30);
    l_snapshot_dates SYS_REFCURSOR;
    l_snapshot_date VARCHAR2(6);
    l_load_set_column VARCHAR2(100);
    l_table_alias VARCHAR2(100);
BEGIN
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
        l_load_set_column := 'A_MOPDB_LOAD_SET_FK';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
        l_load_set_column := 'A_DWH_LOAD_SET_FK';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
        l_load_set_column := 'A_WORKFLOW_HISTORY_KEY';
    ELSE
        RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
    END IF;

    BEGIN
        EXECUTE IMMEDIATE 'SELECT TABLE_ALIAS FROM ' || l_log_table || ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
            INTO l_table_alias USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No TABLE_ALIAS found for table: ' || p_table_owner || '.' || p_table_name);
    END;

    BEGIN
        EXECUTE IMMEDIATE 'SELECT SAVE_MODE FROM ' || l_log_table || ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
            INTO l_save_mode USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
    END;

    BEGIN
        EXECUTE IMMEDIATE 'SELECT PARTITION_COLUMN FROM ' || l_log_table || ' WHERE OWNER = :1 AND TABLE_NAME = :2'
            INTO l_partition_column USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
    END;

    IF UPPER(l_partition_column) != 'ROWID' THEN
        BEGIN
            EXECUTE IMMEDIATE 'SELECT DATA_TYPE FROM ALL_TAB_COLUMNS WHERE OWNER = :1 AND TABLE_NAME = :2 AND COLUMN_NAME = :3'
                INTO l_partition_column_type USING UPPER(p_table_owner), UPPER(p_table_name), UPPER(l_partition_column);
            IF UPPER(l_partition_column_type) != 'DATE' THEN
                RAISE_APPLICATION_ERROR(-20004, 'Invalid PARTITION_COLUMN: ' || l_partition_column || ' must be of type DATE.');
            END IF;
        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                RAISE_APPLICATION_ERROR(-20005, 'Invalid PARTITION_COLUMN: ' || l_partition_column || ' does not exist in table ' || p_table_owner || '.' || p_table_name);
        END;
    END IF;

    IF UPPER(l_save_mode) = 'OVERWRITE' THEN
        DBMS_OUTPUT.PUT_LINE('OVERWRITE mode detected. Clearing all files for table: ' || UPPER(p_table_name));
        OPEN l_files_to_delete FOR SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => c_credential, location_uri => p_objectstore_uri ));
        LOOP
            FETCH l_files_to_delete INTO l_file_name;
            EXIT WHEN l_files_to_delete%NOTFOUND;
            IF INSTR(UPPER(l_file_name), UPPER(l_table_alias || '/')) = 1 THEN
                BEGIN
                    DBMS_CLOUD.DELETE_OBJECT( credential_name => c_credential, object_uri => p_objectstore_uri || l_file_name );
                    DBMS_OUTPUT.PUT_LINE('Deleted file: ' || l_file_name);
                EXCEPTION
                    WHEN OTHERS THEN
                        DBMS_OUTPUT.PUT_LINE('Failed to delete file: ' || l_file_name || ' Error: ' || get_error_stack());
                END;
            END IF;
        END LOOP;
        CLOSE l_files_to_delete;

    ELSIF UPPER(l_save_mode) = 'APPEND' THEN
        BEGIN
            EXECUTE IMMEDIATE 'SELECT LAST_PROCESSED_KEY FROM ' || l_log_table || ' WHERE OWNER = :1 AND TABLE_NAME = :2'
                INTO l_last_processed_key USING UPPER(p_table_owner), UPPER(p_table_name);
            IF l_last_processed_key IS NULL THEN
                l_last_processed_key := -100;
            END IF;
        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
        END;

        IF l_last_processed_key = -100 THEN
            DBMS_OUTPUT.PUT_LINE('Initial load detected. Clearing all files for table: ' || UPPER(p_table_name));
            OPEN l_files_to_delete FOR SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => c_credential, location_uri => p_objectstore_uri ));
            LOOP
                FETCH l_files_to_delete INTO l_file_name;
                EXIT WHEN l_files_to_delete%NOTFOUND;
                IF INSTR(UPPER(l_file_name), UPPER(l_table_alias || '/')) = 1 THEN
                    BEGIN
                        DBMS_CLOUD.DELETE_OBJECT( credential_name => c_credential, object_uri => p_objectstore_uri || l_file_name );
                        DBMS_OUTPUT.PUT_LINE('Deleted file: ' || l_file_name);
                    EXCEPTION
                        WHEN OTHERS THEN
                            DBMS_OUTPUT.PUT_LINE('Failed to delete file: ' || l_file_name || ' Error: ' || get_error_stack());
                    END;
                END IF;
            END LOOP;
            CLOSE l_files_to_delete;
        ELSE
            OPEN l_snapshot_dates FOR 'SELECT DISTINCT TO_CHAR(' || l_partition_column || ', ''YYYYMM'') AS PARTITION_DATE ' ||
                                  'FROM ' || p_table_owner || '.' || p_table_name || ' ' ||
                                  'WHERE ' || l_load_set_column || ' > :1' USING l_last_processed_key;
            LOOP
                FETCH l_snapshot_dates INTO l_snapshot_date;
                EXIT WHEN l_snapshot_dates%NOTFOUND;
                OPEN l_files_to_delete FOR SELECT object_name FROM TABLE(DBMS_CLOUD.LIST_OBJECTS( credential_name => c_credential, location_uri => p_objectstore_uri ));
                LOOP
                    FETCH l_files_to_delete INTO l_file_name;
                    EXIT WHEN l_files_to_delete%NOTFOUND;
                    IF INSTR(UPPER(l_file_name), UPPER(l_table_alias || '/PARTITION_' || l_snapshot_date || '_')) = 1 THEN
                        BEGIN
                            DBMS_CLOUD.DELETE_OBJECT( credential_name => c_credential, object_uri => p_objectstore_uri || l_file_name );
                            DBMS_OUTPUT.PUT_LINE('Deleted file: ' || l_file_name);
                        EXCEPTION
                            WHEN OTHERS THEN
                                DBMS_OUTPUT.PUT_LINE('Failed to delete file: ' || l_file_name || ' Error: ' || get_error_stack());
                        END;
                    END IF;
                END LOOP;
                CLOSE l_files_to_delete;
            END LOOP;
            CLOSE l_snapshot_dates;
        END IF;

    ELSE
        DBMS_OUTPUT.PUT_LINE('Unsupported SAVE_MODE: ' || l_save_mode || '. Skipping bucket cleanup.');
        RETURN;
    END IF;

    DBMS_OUTPUT.PUT_LINE('Bucket cleanup completed successfully for table ' || UPPER(p_table_name));
EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('Error in clear_s3_bucket: ' || get_error_stack());
        RAISE;
END clear_s3_bucket;


-------------------------------------------------------------------------------------------
      PROCEDURE export_data(
        p_service_name    IN VARCHAR2,
        p_table_owner     IN VARCHAR2,
        p_table_name      IN VARCHAR2,
        p_objectstore_uri IN VARCHAR2,
        p_date_column     IN VARCHAR2,
        p_start           IN NUMBER,
        p_end             IN NUMBER
      ) AS
        l_query    CLOB;
        l_file_uri VARCHAR2(2000);
        l_table_alias  VARCHAR2(100);
        l_log_table    VARCHAR2(1000); -- Added log table

      BEGIN
    EXECUTE IMMEDIATE 'ALTER SESSION SET NLS_DATE_FORMAT = ''YYYY-MM-DD HH24:MI:SS''';
    -- Generate the file URI for the current chunk
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
    ELSE
        RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
    END IF;

    BEGIN
    EXECUTE IMMEDIATE 'SELECT TABLE_ALIAS FROM ' || l_log_table ||
                          ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
          INTO l_table_alias
          USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No TABLE_ALIAS found for table: ' ||
                                              p_table_owner || '.' || p_table_name);
    END;

        -- Build the fully qualified file URI.
        l_file_uri := p_objectstore_uri || upper(l_table_alias) || upper('/PARTITION_') || p_start || '_' || p_end || '.parquet';

        /* Build dynamic SQL:
             - The query selects all columns from the table.
             - It computes a partition identifier using the supplied p_date_column.
             - It assumes that the partition (year-month) is computed using TO_CHAR with 'YYYYMM'.
               Adjust as needed if the partitioning logic is different.
             - The bind variables :start_id and :end_id are used for filtering.
        */
    -- Initialize the dynamic SQL query.
    l_query := 'SELECT ';

    -- Construct the column list dynamically.
       FOR col IN (
        SELECT column_name, data_type, data_precision
        FROM all_tab_columns
        WHERE table_name = UPPER(p_table_name)
          AND owner = UPPER(p_table_owner)
        ORDER BY column_id
    ) LOOP
        IF col.data_type LIKE 'TIMESTAMP%' OR col.data_type LIKE 'DATE%' THEN
            l_query := l_query || 'TO_CHAR(' || col.column_name || ', ''YYYY-MM-DD HH24:MI:SS'') AS ' || col.column_name || ',';
        ELSIF col.data_type = 'FLOAT' THEN
            l_query := l_query || 'TO_CHAR(' || col.column_name || ') AS ' || col.column_name || ',';
        ELSIF col.data_type = 'NUMBER' AND col.data_precision IS NULL THEN
            l_query := l_query || 'TO_CHAR(' || col.column_name || ') AS ' || col.column_name || ',';
        ELSE
            l_query := l_query || col.column_name || ',';
        END IF;
    END LOOP;

    -- Remove the trailing comma.
    l_query := SUBSTR(l_query, 1, LENGTH(l_query) - 1);

    -- Add the FROM clause and WHERE clause.
    l_query := l_query || ' FROM ' || UPPER(p_table_owner) || '.' || UPPER(p_table_name);
    l_query := l_query || ' WHERE ' || p_date_column || ' BETWEEN TO_DATE(''' || p_end || '01'',''YYYYMMDD'') ';
    l_query := l_query || ' AND ADD_MONTHS(TO_DATE(''' || p_end || '01'',''YYYYMMDD''),1)-1';

        /* Call DBMS_CLOUD.EXPORT_DATA passing the dynamic query.
           Additional format options are provided in JSON (e.g., hive partitioning and partition keys).
        */

        DBMS_CLOUD.EXPORT_DATA(
          credential_name => c_credential,
          file_uri_list   => l_file_uri,
          format          => json_object(
                               'type'              VALUE 'parquet' 
                             ),
          query           => l_query
        );

      EXCEPTION
        WHEN OTHERS THEN
          DBMS_OUTPUT.PUT_LINE('Error in export_data: ' || get_error_stack());
          RAISE;
      END export_data;

-------------------------------------------------------------------------------------------
  -------------------------------------------------------------------------
  -- Procedure: export_table
  --
  -- Finds distinct partitions (using the supplied p_date_column) for the 
  -- specified table and dispatches parallel tasks to export each partition by 
  -- calling export_data. The anonymous block in run_task now uses two bind 
  -- variables, :start_id and :end_id.
  -------------------------------------------------------------------------
    PROCEDURE export_table(
        p_service_name    IN VARCHAR2,
        p_table_owner     IN VARCHAR2,
        p_table_name      IN VARCHAR2,
        p_objectstore_uri IN VARCHAR2
    ) IS
        l_sql              CLOB;
        l_task_name        VARCHAR2(200);
        l_task_state       VARCHAR2(30);
        l_timeout          NUMBER := 3600; -- Timeout in seconds (e.g., 1 hour)
        l_elapsed_time     NUMBER := 0;
        l_poll_interval    NUMBER := 10;   -- Polling interval in seconds
        l_save_mode        VARCHAR2(50);
        l_partition_column VARCHAR2(100);
        l_max_threads      NUMBER;
        l_last_processed_key NUMBER;
        l_log_table          VARCHAR2(1000);
        l_partition_column_type VARCHAR2(30);
        l_chunk_size           NUMBER; 
        l_total_records        NUMBER; 
    BEGIN
        -- Validate p_service_name
        IF UPPER(p_service_name) NOT IN ('MOPDB', 'RAR', 'RQSD') THEN
            RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
        END IF;

        -- Determine the appropriate log table based on p_service_name
IF UPPER(p_service_name) = 'MOPDB' THEN
    l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
ELSIF UPPER(p_service_name) = 'RAR' THEN
    l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
ELSIF UPPER(p_service_name) = 'RQSD' THEN
    l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
ELSE
    RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
END IF;

        -- Fetch SAVE_MODE
        BEGIN
            EXECUTE IMMEDIATE 'SELECT SAVE_MODE FROM ' || l_log_table || ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
                INTO l_save_mode USING UPPER(p_table_owner), UPPER(p_table_name);
        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
        END;

        -- Validate SAVE_MODE
        IF UPPER(l_save_mode) NOT IN ('OVERWRITE', 'APPEND') THEN
            RAISE_APPLICATION_ERROR(-20001, 'Unrecognized SAVE_MODE: ' || l_save_mode);
        END IF;

        -- Fetch PARTITION_COLUMN
        BEGIN
            EXECUTE IMMEDIATE 'SELECT PARTITION_COLUMN FROM ' || l_log_table || ' WHERE OWNER = :1 AND TABLE_NAME = :2'
                INTO l_partition_column USING UPPER(p_table_owner), UPPER(p_table_name);
        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
        END;

             -- Validate PARTITION_COLUMN
        IF UPPER(l_partition_column) != 'ROWID' THEN
            -- Check if column exists and is of type DATE
            BEGIN
                EXECUTE IMMEDIATE 'SELECT DATA_TYPE FROM ALL_TAB_COLUMNS WHERE OWNER = :1 AND TABLE_NAME = :2 AND COLUMN_NAME = :3'
                    INTO l_partition_column_type
                    USING UPPER(p_table_owner), UPPER(p_table_name), UPPER(l_partition_column);

                IF UPPER(l_partition_column_type) != 'DATE' THEN
                    RAISE_APPLICATION_ERROR(-20004, 'Invalid PARTITION_COLUMN: ' || l_partition_column || ' must be of type DATE.');
                END IF;
            EXCEPTION
                WHEN NO_DATA_FOUND THEN
                    RAISE_APPLICATION_ERROR(-20005, 'Invalid PARTITION_COLUMN: ' || l_partition_column || ' does not exist in table ' || p_table_owner || '.' || p_table_name);
            END;
        END IF;


        -- Fetch MAX_THREADS
        BEGIN
            EXECUTE IMMEDIATE 'SELECT MAX_THREADS FROM ' || l_log_table || ' WHERE OWNER = :1 AND TABLE_NAME = :2'
                INTO l_max_threads USING UPPER(p_table_owner), UPPER(p_table_name);
        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
        END;
        
            -- Fetch total record count
    BEGIN
        EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM ' || UPPER(p_table_owner) || '.' || UPPER(p_table_name)
            INTO l_total_records;
    EXCEPTION
        WHEN OTHERS THEN
            RAISE_APPLICATION_ERROR(-20007, 'Error fetching total record count for table: ' || p_table_owner || '.' || p_table_name);
    END;

    -- Calculate chunk size
    IF l_max_threads > 0 THEN
        IF l_max_threads > 1 THEN
        l_chunk_size := CEIL(l_total_records /(l_max_threads-1)); 
        ELSE
        l_chunk_size := CEIL(l_total_records /(l_max_threads));
        END IF;
    ELSE
        RAISE_APPLICATION_ERROR(-20008, 'Invalid MAX_THREADS value: ' || l_max_threads || '. Must be greater than 0.');
    END IF;

            -- *** Clear S3 Bucket ***
    -- Call the clear_s3_bucket procedure before chunking and loading
    clear_s3_bucket(
        p_service_name   => p_service_name,
        p_table_owner    => p_table_owner,
        p_table_name     => p_table_name,
        p_objectstore_uri => p_objectstore_uri
    );

        -- Handle chunking based on PARTITION_COLUMN
        IF UPPER(l_partition_column) != 'ROWID' THEN
            -- Handle SNAPSHOT_DATE chunking
            IF UPPER(l_save_mode) = 'APPEND' THEN
                DECLARE
                    l_load_set_column VARCHAR2(100);
                BEGIN
                    IF UPPER(p_service_name) = 'MOPDB' THEN
                        l_load_set_column := 'A_MOPDB_LOAD_SET_FK';
                    ELSIF UPPER(p_service_name) = 'RAR' THEN
                        l_load_set_column := 'A_DWH_LOAD_SET_FK';
                    ELSIF UPPER(p_service_name) = 'RQSD' THEN
                        l_load_set_column := 'A_WORKFLOW_HISTORY_KEY';
                        --'A_WORKFLOW_HISTORY_KEY';
                    END IF;

-- Fetch LAST_PROCESSED_KEY
BEGIN
    EXECUTE IMMEDIATE 
        'SELECT LAST_PROCESSED_KEY FROM ' || l_log_table || ' WHERE OWNER = :1 AND TABLE_NAME = :2' 
    INTO 
        l_last_processed_key 
    USING 
        UPPER(p_table_owner), UPPER(p_table_name);
    IF l_last_processed_key IS NULL THEN
        l_last_processed_key := -100;
    END IF;

EXCEPTION
    WHEN NO_DATA_FOUND THEN
        RAISE_APPLICATION_ERROR(
            -20006, 
            'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name
        );
END;
                -- Generate SQL for APPEND
                l_sql := 'SELECT DISTINCT TO_CHAR(' || l_partition_column || ', ''YYYYMM'') AS start_id, '
                        || 'TO_CHAR(' || l_partition_column || ', ''YYYYMM'') AS end_id '
                        || 'FROM ' || p_table_owner || '.' || p_table_name || ' '
                        || 'WHERE ' || l_load_set_column || ' > ' || l_last_processed_key;

                -- Check if APPEND query have any data
                DECLARE
                    l_check_count NUMBER;
                BEGIN
                    EXECUTE IMMEDIATE 'SELECT COUNT(*) FROM (' || l_sql || ')' INTO l_check_count;
                    IF l_check_count = 0 THEN
                        DBMS_OUTPUT.PUT_LINE('The table "' || p_table_owner || '.' || p_table_name || '" is empty or all data has already been processed.');
                        DBMS_OUTPUT.PUT_LINE('Last processed key: ' || l_last_processed_key || '. No loading will be performed.');
                        RETURN; -- No data to load
                    END IF;
                END;
                END;
            ELSIF UPPER(l_save_mode) = 'OVERWRITE' THEN
                -- Generate SQL for OVERWRITE
                l_sql := 'SELECT DISTINCT TO_NUMBER(TO_CHAR(' || l_partition_column || ',''YYYYMM'')) AS start_id, ' ||
                         'TO_NUMBER(TO_CHAR(' || l_partition_column || ',''YYYYMM'')) AS end_id ' ||
                         'FROM ' || p_table_owner || '.' || p_table_name;
            END IF;

        ELSIF UPPER(l_partition_column) = 'ROWID' THEN
            -- Handle ROWID chunking (only for SAVE_MODE = OVERWRITE)
            IF UPPER(l_save_mode) = 'APPEND' THEN
                RAISE_APPLICATION_ERROR(-20003, 'ROWID chunking is not supported for SAVE_MODE = APPEND.');
            END IF;

            -- Generate SQL for ROWID chunking
            l_sql := NULL; -- ROWID chunking does not require explicit SQL, handled by DBMS_PARALLEL_EXECUTE
--        ELSIF UPPER(l_partition_column) = 'KEY' THEN
        
        
        END IF;

        -- Create a unique task name
        l_task_name := 'EXPORT_' || UPPER(p_table_name);

        -- Remove an existing task if it exists
        BEGIN
            DBMS_PARALLEL_EXECUTE.DROP_TASK(task_name => l_task_name);
        EXCEPTION
            WHEN OTHERS THEN
                NULL; -- Ignore errors if the task doesn't exist
        END;

        -- Create a new parallel task
        BEGIN
            DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => l_task_name);
            DBMS_OUTPUT.PUT_LINE('Task has been successfully created: ' || l_task_name);
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('An error occurred while creating the task.');
                DBMS_OUTPUT.PUT_LINE(get_error_stack());
                RETURN;
        END;

        -- Create chunks for the task
        BEGIN
            DBMS_OUTPUT.PUT_LINE('USING PARTITION COLUMN LIKE: ' || l_partition_column);
            IF UPPER(l_partition_column) != 'ROWID' THEN
                DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
                    task_name => l_task_name,
                    sql_stmt  => l_sql,
                    by_rowid  => FALSE
                );
            ELSIF UPPER(l_partition_column) = 'ROWID' THEN
                DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_ROWID(
                    task_name    => l_task_name,
                    table_owner  => UPPER(p_table_owner),
                    table_name   => UPPER(p_table_name),
                    by_row       => true,
                    chunk_size   => l_chunk_size
                );
            END IF;
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('An error occurred while creating chunks.');
                DBMS_OUTPUT.PUT_LINE(get_error_stack());
                RETURN;
        END;

        -- Run the task
        BEGIN
IF UPPER(l_partition_column) = 'ROWID' THEN
    -- RUN_TASK for ROWID
    DBMS_PARALLEL_EXECUTE.RUN_TASK(
        task_name     => l_task_name,
        sql_stmt      => 'BEGIN DATA_REPLICATOR.export_data_by_rowid(' ||
                         CHR(39) || p_service_name || CHR(39) || ', ' ||   
                         CHR(39) || p_table_owner || CHR(39) || ', ' ||
                         CHR(39) || p_table_name || CHR(39) || ', ' ||
                         CHR(39) || p_objectstore_uri || CHR(39) || ', ' ||
                         ':start_id, :end_id); END;',
        language_flag => DBMS_SQL.NATIVE,
        parallel_level => l_max_threads
    );
ELSE
    -- RUN_TASK for non ROWID
    DBMS_PARALLEL_EXECUTE.RUN_TASK(
        task_name      => l_task_name,
        sql_stmt       => 'BEGIN DATA_REPLICATOR.export_data(' ||
                         CHR(39) || p_service_name || CHR(39) || ', ' ||   
                         CHR(39) || p_table_owner || CHR(39) || ', ' || CHR(39) || p_table_name || CHR(39) || ', ' || CHR(39) || p_objectstore_uri || CHR(39) || ', ' || CHR(39) || l_partition_column || CHR(39) || ', :start_id, :end_id); END;',
        language_flag  => DBMS_SQL.NATIVE,
        parallel_level => l_max_threads
    );
END IF;
        EXCEPTION
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('An error occurred while running the task.');
                DBMS_OUTPUT.PUT_LINE(get_error_stack());
                RETURN;
        END;

        DBMS_OUTPUT.PUT_LINE('Export completed successfully for table ' || p_table_name);

    END export_table;


-------------------------------------------------------------------------------------------
PROCEDURE start_log_table(
    p_run_id       IN VARCHAR2,
    p_service_name IN VARCHAR2,
    p_table_owner  IN VARCHAR2,
    p_table_name   IN VARCHAR2
) AS 
    l_log_table VARCHAR2(1000);
    l_sql_stmt VARCHAR2(2000);
BEGIN
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
    ELSE
        RAISE_APPLICATION_ERROR(-20000, 'Invalid service name: ' || p_service_name);
    END IF;

    -- Dynamic SQL for updating the appropriate table
    l_sql_stmt := 'UPDATE ' || l_log_table || ' ' ||
                  'SET LAST_RUN_ID = :1, LAST_START_TIME = SYSDATE, LAST_STATUS = ''RUNNING'' ' ||
                  'WHERE OWNER = :2 AND TABLE_NAME = :3';

    EXECUTE IMMEDIATE l_sql_stmt USING p_run_id, UPPER(p_table_owner), UPPER(p_table_name);
END start_log_table;


-------------------------------------------------------------------------------------------
PROCEDURE end_log_table(
    p_service_name IN VARCHAR2,
    p_table_owner IN VARCHAR2,
    p_table_name IN VARCHAR2
) AS
    l_log_table VARCHAR2(1000);
    l_sql_stmt VARCHAR2(2000);
    l_in_status VARCHAR2(4000);
    l_last_processed_key NUMBER;
    l_load_set_column VARCHAR2(100);
    l_save_mode VARCHAR2(50);
BEGIN
    -- Select the log table based on the service name
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
        l_load_set_column := 'A_MOPDB_LOAD_SET_FK';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
        l_load_set_column := 'A_DWH_LOAD_SET_FK';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
        l_load_set_column := 'A_WORKFLOW_HISTORY_KEY';
    ELSE
        RAISE_APPLICATION_ERROR(-20001, 'Invalid service name: ' || p_service_name);
    END IF;

    -- Fetch the status of the parallel task
    BEGIN
        SELECT status
        INTO l_in_status
        FROM sys.user_parallel_execute_tasks
        WHERE task_name = 'EXPORT_' || UPPER(p_table_name);

    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            DBMS_OUTPUT.PUT_LINE('No task found for table: ' || p_table_name);
            l_in_status := 'NOT_FOUND'; -- Default status
        WHEN OTHERS THEN
            DBMS_OUTPUT.PUT_LINE('Error fetching task status: ' || get_error_stack());
            RAISE;
    END;

    -- Debug: Display diagnostic information
    DBMS_OUTPUT.PUT_LINE('Task status: ' || l_in_status);

    -- Fetch the maximum processed key from the table but only for SAVE_MODE = APPEND :
    -- Fetch SAVE_MODE
    BEGIN
        EXECUTE IMMEDIATE 'SELECT SAVE_MODE FROM ' || l_log_table || ' WHERE UPPER(OWNER) = :1 AND UPPER(TABLE_NAME) = :2'
            INTO l_save_mode
            USING UPPER(p_table_owner), UPPER(p_table_name);
    EXCEPTION
        WHEN NO_DATA_FOUND THEN
            RAISE_APPLICATION_ERROR(-20006, 'No configuration found for table: ' || p_table_owner || '.' || p_table_name || ' in service: ' || p_service_name);
    END;

    -- Fetch the maximum processed key
    IF l_save_mode = 'APPEND' THEN
        BEGIN
            EXECUTE IMMEDIATE
                'SELECT MAX(' || l_load_set_column || ') FROM ' || p_table_owner || '.' || p_table_name
            INTO l_last_processed_key;

            DBMS_OUTPUT.PUT_LINE('Last processed key: ' || l_last_processed_key);

        EXCEPTION
            WHEN NO_DATA_FOUND THEN
                DBMS_OUTPUT.PUT_LINE('No data found in table: ' || p_table_owner || '.' || p_table_name);
                l_last_processed_key := NULL; -- No key to process
            WHEN OTHERS THEN
                DBMS_OUTPUT.PUT_LINE('Error fetching last processed key: ' || get_error_stack());
                RAISE;
        END;
    ELSE
        l_last_processed_key := NULL;
    END IF;

    -- Build dynamic query to update the log table
    IF l_in_status = 'FINISHED' THEN
        -- Update LAST_END_TIME and LAST_PROCESSED_KEY if status is 'FINISHED'
        l_sql_stmt := 'UPDATE ' || l_log_table || ' ' ||
                    'SET LAST_END_TIME = SYSDATE, ' ||
                    '    LAST_STATUS = :1, ' ||
                    '    LAST_PROCESSED_KEY = :2 ' ||
                    'WHERE OWNER = :3 AND TABLE_NAME = :4';
        DBMS_OUTPUT.PUT_LINE('Updating log table with LAST_END_TIME and LAST_PROCESSED_KEY.');
    ELSE
        -- Only update LAST_STATUS if status is not 'FINISHED'
        l_sql_stmt := 'UPDATE ' || l_log_table || ' ' ||
                    'SET LAST_STATUS = :1 ' ||
                    'WHERE OWNER = :2 AND TABLE_NAME = :3';
        DBMS_OUTPUT.PUT_LINE('Updating log table without LAST_END_TIME and LAST_PROCESSED_KEY.');
    END IF;

    -- Execute the dynamic query
    BEGIN
        IF l_in_status = 'FINISHED' THEN
            EXECUTE IMMEDIATE l_sql_stmt USING l_in_status, l_last_processed_key, UPPER(p_table_owner), UPPER(p_table_name);
        ELSE
            EXECUTE IMMEDIATE l_sql_stmt USING l_in_status, UPPER(p_table_owner), UPPER(p_table_name);
        END IF;
        DBMS_OUTPUT.PUT_LINE('Log table updated successfully.');

    EXCEPTION
        WHEN OTHERS THEN
            DBMS_OUTPUT.PUT_LINE('Error updating log table: ' || get_error_stack());
            RAISE;
    END;

EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('Error in end_log_table: ' || get_error_stack());
        RAISE;
END end_log_table;

PROCEDURE end_log_table_generator(
    p_service_name IN VARCHAR2,
    p_table_owner  IN VARCHAR2,
    p_table_name   IN VARCHAR2,
    p_status       IN VARCHAR2
) AS
    l_log_table VARCHAR2(1000);
    l_sql_stmt  VARCHAR2(2000);
BEGIN
    -- Determine the log table based on the service name
    IF UPPER(p_service_name) = 'MOPDB' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB';
    ELSIF UPPER(p_service_name) = 'RAR' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RAR';
    ELSIF UPPER(p_service_name) = 'RQSD' THEN
        l_log_table := 'CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD';
    ELSE
        -- Raise an error if the service name is invalid
        RAISE_APPLICATION_ERROR(-20001, 'Invalid service name: ' || p_service_name);
    END IF;

    -- Debug: Display task status
    DBMS_OUTPUT.PUT_LINE('Task status: ' || p_status);

    -- Build the dynamic SQL query to update the log table
    l_sql_stmt := 'UPDATE ' || l_log_table || ' ' ||
                  'SET LAST_END_TIME = SYSDATE, ' ||
                  '    LAST_STATUS = :1 ' ||
                  'WHERE OWNER = :2 AND TABLE_NAME = :3';

    DBMS_OUTPUT.PUT_LINE('Updating log table with LAST_END_TIME and LAST_STATUS.');

    -- Execute the dynamic SQL query
    BEGIN
        EXECUTE IMMEDIATE l_sql_stmt USING p_status, UPPER(p_table_owner), UPPER(p_table_name);
        DBMS_OUTPUT.PUT_LINE('Log table updated successfully.');
    EXCEPTION
        WHEN OTHERS THEN
            DBMS_OUTPUT.PUT_LINE('Error updating log table: ' || DBMS_UTILITY.FORMAT_ERROR_STACK);
            RAISE;
    END;

EXCEPTION
    WHEN OTHERS THEN
        DBMS_OUTPUT.PUT_LINE('Error in end_log_table_generator: ' || DBMS_UTILITY.FORMAT_ERROR_STACK);
        RAISE;
END end_log_table_generator;

-------------------------------------------------------------------------------------------
PROCEDURE list_s3_bucket_objects(
        p_service_name     IN VARCHAR2,
        p_owner_name       IN VARCHAR2,
        p_table_name       IN VARCHAR2,
        p_objectstore_uri  IN VARCHAR2
) IS
        l_files SYS_REFCURSOR;
        l_file_name VARCHAR2(4000);
BEGIN
        -- Diagnostic information
        DBMS_OUTPUT.PUT_LINE('Listing S3 bucket objects...');
        DBMS_OUTPUT.PUT_LINE('Service Name: ' || p_service_name);
        DBMS_OUTPUT.PUT_LINE('Owner Name: ' || p_owner_name);
        DBMS_OUTPUT.PUT_LINE('Table Name: ' || p_table_name);
        DBMS_OUTPUT.PUT_LINE('Object Store URI: ' || p_objectstore_uri);
        DBMS_OUTPUT.PUT_LINE('---------------------------------');

        -- Fetch the list of objects from Object Storage
        OPEN l_files FOR
            SELECT object_name
            FROM TABLE(
                DBMS_CLOUD.LIST_OBJECTS(
                    credential_name => c_credential,
                    location_uri    => p_objectstore_uri
                )
            )
            WHERE UPPER(object_name) LIKE UPPER(p_table_name) || '/%';

        -- Iterate through results and print them
        LOOP
            FETCH l_files INTO l_file_name;
            EXIT WHEN l_files%NOTFOUND;

            -- Displaying the file name
            DBMS_OUTPUT.PUT_LINE('Found object: ' || l_file_name);
        END LOOP;

        -- Close the cursor
        CLOSE l_files;

        DBMS_OUTPUT.PUT_LINE('Listing completed successfully.');

    EXCEPTION
        WHEN OTHERS THEN
            DBMS_OUTPUT.PUT_LINE('Error in list_s3_bucket_objects: ' || get_error_stack());
            RAISE;
    END list_s3_bucket_objects;
END DATA_REPLICATOR;
/
