CREATE OR REPLACE PACKAGE DATA_REPLICATOR IS
  /**********************************************************************
   This package exports partitions (by year and month) of a specified table 
   to Oracle Object Storage in Parquet format using parallel execution and Hive 
   partitioning. This generic version accepts parameters for the table name, the 
   Object Store URI, and the date column used for partitioning.

   Parameters:
     p_table_name      - Name of the table to export.
     p_objectstore_uri - Base Object Store URI where the export files will be written.
                         e.g.: 'objectstore://<container>/exports/'
     p_date_column     - Name of the DATE column used for partitioning (e.g., 'SNAPSHOT_DATE')
  
   Usage:
     Call DATA_REPLICATOR.export_table('HUGE_TABLE', 'objectstore://<your_container>/exports/', 'SNAPSHOT_DATE');
  **********************************************************************/
  PROCEDURE export_table(
    p_table_owner     IN VARCHAR2,
    p_table_name      IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_date_column     IN VARCHAR2
  );
  
  PROCEDURE export_data(
    p_table_owner     IN VARCHAR2,
    p_table_name      IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_date_column     IN VARCHAR2,
    p_start           IN NUMBER,
    p_end             IN NUMBER
  );
END DATA_REPLICATOR;
/

CREATE OR REPLACE PACKAGE BODY DATA_REPLICATOR IS
  -- Credential name to access Object Storage (adjust or parameterize as needed).
  c_credential CONSTANT VARCHAR2(100) := 'DEF_CRED_ARN';
  
  -- Hive partition keys used to organize exported files (adjust as needed).
  c_hive_partition_keys CONSTANT VARCHAR2(100) := 'YEAR_COL,MONTH_COL';

  -------------------------------------------------------------------------
  -- Procedure: export_data
  -- 
  -- Exports data for a single partition (determined by p_start and p_end values)
  -- from the specified table to a file in Object Storage.
  --
  -- p_date_column is used to compute the partition value and filter records in this partition.
  -------------------------------------------------------------------------
  PROCEDURE export_data(
    p_table_owner     IN VARCHAR2,
    p_table_name      IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_date_column     IN VARCHAR2,
    p_start           IN NUMBER,
    p_end             IN NUMBER
  ) IS
    l_query    VARCHAR2(4000);
    l_file_uri VARCHAR2(2000);
  BEGIN
    -- Build the fully qualified file URI.
    l_file_uri := p_objectstore_uri || p_table_owner || '_'|| p_table_name || '/partition_' || p_start || '_' || p_end || '.parquet';
    
    /* Build dynamic SQL:
         - The query selects all columns from the table.
         - It computes a partition identifier using the supplied p_date_column.
         - It assumes that the partition (year-month) is computed using TO_CHAR with 'YYYYMM'.
           Adjust as needed if the partitioning logic is different.
         - The bind variables :start_id and :end_id are used for filtering.
    */
    l_query :=
      'SELECT * FROM ( ' ||
      '  SELECT TO_NUMBER(TO_CHAR(' || p_date_column || ',''YYYYMM'')) AS PARTITION_ID,';
      
    -- Construct the column list dynamically.
    FOR col IN (
      SELECT column_name 
      FROM dba_tab_columns 
      WHERE table_name = UPPER(p_table_name)
        AND OWNER = UPPER(p_table_owner)
      ORDER BY column_id
    ) LOOP
      l_query := l_query || col.column_name || ',';
    END LOOP;
    
    -- Remove the trailing comma.
    l_query := SUBSTR(l_query, 1, LENGTH(l_query) - 1);
    l_query := l_query || ' FROM ' || p_table_owner || '.'|| p_table_name || ') ' ||
              '  WHERE ' || p_date_column || ' BETWEEN TO_DATE(''' || p_end || '01'',''YYYYMMDD'') ' ||
              '  AND ADD_MONTHS(TO_DATE(''' || p_end || '01'',''YYYYMMDD''),1)-1';
    
    /* Call DBMS_CLOUD.EXPORT_DATA passing the dynamic query.
       Additional format options are provided in JSON (e.g., hive partitioning and partition keys).
    */
    DBMS_CLOUD.EXPORT_DATA(
      credential_name => c_credential,
      file_uri_list   => l_file_uri,
      format          => json_object(
                           'type'              VALUE 'parquet' 
                         ),
      query           => l_query
    );
    
  EXCEPTION
    WHEN OTHERS THEN
      DBMS_OUTPUT.PUT_LINE('Error in export_data: ' || SQLERRM);
      RAISE;
  END export_data;

  -------------------------------------------------------------------------
  -- Procedure: export_table
  --
  -- Finds distinct partitions (using the supplied p_date_column) for the 
  -- specified table and dispatches parallel tasks to export each partition by 
  -- calling export_data. The anonymous block in run_task now uses two bind 
  -- variables, :start_id and :end_id.
  -------------------------------------------------------------------------
  PROCEDURE export_table(
    p_table_owner     IN VARCHAR2,
    p_table_name      IN VARCHAR2,
    p_objectstore_uri IN VARCHAR2,
    p_date_column     IN VARCHAR2
  ) IS
    -- Local record and table type for partition values.
    TYPE t_partition_rec IS RECORD (
      partition_val NUMBER
    );
    TYPE t_partition_tab IS TABLE OF t_partition_rec INDEX BY PLS_INTEGER;
    l_partitions t_partition_tab;
    
    l_sql       CLOB;
    l_task_name VARCHAR2(200);
  BEGIN
    DBMS_OUTPUT.PUT_LINE('Starting export of table ' || p_table_name);
    
    -- Create a unique task name (optionally you can include a timestamp).
    l_task_name := 'EXPORT_' || UPPER(p_table_name);
    
    -- Remove an existing task if it exists.
    BEGIN
      DBMS_PARALLEL_EXECUTE.DROP_TASK(task_name => l_task_name);
    EXCEPTION
      WHEN OTHERS THEN
        NULL;
    END;
    
    DBMS_PARALLEL_EXECUTE.CREATE_TASK(task_name => l_task_name);
    
    /* Construct dynamic SQL to produce distinct partition values.
       Here we assume the partition value is computed on the supplied date column.
       The logic uses TO_CHAR(...,'YYYYMM') so that each chunk represents a year-month.
    */
    l_sql :=
      'SELECT DISTINCT TO_NUMBER(TO_CHAR(' || p_date_column || ',''YYYYMM'')) AS start_id, ' ||
                      'TO_NUMBER(TO_CHAR(' || p_date_column || ',''YYYYMM'')) AS end_id ' ||
      'FROM ' || p_table_owner || '.' || p_table_name ;
      
    
    -- Create chunks (one chunk per partition) using the dynamic SQL.
    DBMS_PARALLEL_EXECUTE.CREATE_CHUNKS_BY_SQL(
      task_name  => l_task_name,
      sql_stmt   => l_sql,
      by_rowid   => FALSE
    );
    
    /* Now run a task for each chunk. The anonymous PL/SQL block passed to 
       run_task uses two bind variables :start_id and :end_id for partition value.
       In addition, static bind variables for :table_name, :objectstore_uri, and
       :date_column are passed using additional_bind_vars.
       
       Here, the same partition value is passed for both :start_id and :end_id.
       If needed, adjust logic so that these can be different.
    */
   
    l_sql := 'BEGIN DATA_REPLICATOR.export_data(' || CHR(39) || p_table_owner || CHR(39) ||',
                                                 ' || CHR(39) || p_table_name || CHR(39) ||',
                                                 ' || CHR(39) || p_objectstore_uri || CHR(39) ||',
                                                 ' || CHR(39) || p_date_column || CHR(39) ||',
                                                  :start_id,
                                                  :end_id); END;';
    dbms_output.put_line(l_sql);
    DBMS_PARALLEL_EXECUTE.RUN_TASK(
      task_name       => l_task_name,
      sql_stmt        => l_sql,
      language_flag   => DBMS_SQL.NATIVE,
      parallel_level  => 20
    );
    
    /* Optionally, if you want synchronous execution, you can wait for all tasks:
       DBMS_PARALLEL_EXECUTE.WAIT_FOR_TASKS(task_name => l_task_name);
    */
    
  EXCEPTION
    WHEN OTHERS THEN
      DBMS_OUTPUT.PUT_LINE('Error in export_table: ' || SQLERRM);
      RAISE;
  END export_table;
  
END DATA_REPLICATOR;
/
show errors
set serveroutput on
set timing on
--exec DATA_REPLICATOR.export_data('c2d','t_collateral_all_sources','https://oci-test-sani.bucket.vpce-0b3a5f000733397b0-kxlyoh5z.s3.eu-central-1.vpce.amazonaws.com/','SNAPSHOT_DATE',202401,202401);
exec DATA_REPLICATOR.export_table('c2d','t_collateral_all_sources','https://oci-test-sani.bucket.vpce-0b3a5f000733397b0-kxlyoh5z.s3.eu-central-1.vpce.amazonaws.com/','SNAPSHOT_DATE');
