from airflow import DAG from airflow.operators.python import PythonOperator from airflow.providers.oracle.hooks.oracle import OracleHook from airflow.utils.dates import days_ago from datetime import timedelta import logging p_run_id = 1234 p_service_name = 'MyService' p_table_owner = 'MY_SCHEMA' p_table_name = 'MY_TABLE' p_objectstore_uri = 's3://bucket/uri' # subject to change appropriate for RAR/MOPDB def start_log_table_task(**context): proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.start_log_table(:1, :2, :3, :4); END;" try: oracle_hook = OracleHook(oracle_conn_id='oracle_default') conn = oracle_hook.get_conn() cursor = conn.cursor() cursor.execute(proc_call, [p_run_id, p_service_name, p_table_owner, p_table_name]) conn.commit() cursor.close() conn.close() logging.info("start_log_table executed successfully.") except Exception as e: logging.error("Failed to execute start_log_table: %s", e, exc_info=True) raise def export_table_task(**context): proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.export_table(:1, :2, :3, :4); END;" try: oracle_hook = OracleHook(oracle_conn_id='oracle_default') conn = oracle_hook.get_conn() cursor = conn.cursor() cursor.execute(proc_call, [p_service_name, p_table_owner, p_table_name, p_objectstore_uri]) conn.commit() cursor.close() conn.close() logging.info("export_table executed successfully.") except Exception as e: logging.error("Failed to execute export_table: %s", e, exc_info=True) raise def devo_impyla_task(**context): # Placeholder for Impyla (Devo) code # Example for future: # from impala.dbapi import connect # conn = connect(host="...", port=21050) # cursor = conn.cursor() # cursor.execute("...") logging.info("Impyla (Devo) task placeholder ran. Please implement.") def end_log_table_task(**context): proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.end_log_table(:1, :2, :3); END;" try: oracle_hook = OracleHook(oracle_conn_id='oracle_default') conn = oracle_hook.get_conn() cursor = conn.cursor() cursor.execute(proc_call, [p_service_name, p_table_owner, p_table_name]) conn.commit() cursor.close() conn.close() logging.info("end_log_table executed successfully.") except Exception as e: logging.error("Failed to execute end_log_table: %s", e, exc_info=True) raise default_args = { 'owner': 'airflow', 'depends_on_past': False, 'start_date': days_ago(1), 'email_on_failure': False, 'email_on_retry': False, 'retries': 2, 'retry_delay': timedelta(minutes=5), } with DAG( dag_id='rqsd_devo_replicator_test_old', default_args=default_args, description='Run Devo replicator workflow', schedule_interval=None, catchup=False, tags=['Devo', 'RQSD', 'Replicator'], ) as dag: t1 = PythonOperator( task_id='start_log_table', python_callable=start_log_table_task, ) t2 = PythonOperator( task_id='export_table', python_callable=export_table_task, ) t3 = PythonOperator( task_id='devo_impyla', python_callable=devo_impyla_task, ) t4 = PythonOperator( task_id='end_log_table', python_callable=end_log_table_task, ) t1 >> t2 >> t3 >> t4