from jinja2 import Environment, FileSystemLoader import csv environment = Environment(loader=FileSystemLoader("templates/")) template_sq = environment.get_template("m_Template_sq.sql") template_tgt = environment.get_template("m_Template_target.sql") template_db = environment.get_template("db_Template.sql") template_dag = environment.get_template("dag_Template.py") csvFile = csv.reader(open("ods_mopdb_plain.txt", "r")) header = next(csvFile) # skip header for row in csvFile: (schema, table) = row # ou_tms,ACTIVITYLOGDUE,TMS,T_ACTIVITYLOGDUE,m_MOPDB_TMS_T_ACTIVITYLOGDUE_OU_TMS_ACTIVITYLOGDUE,w_ODS_TMS_ACTIVITYLOGDUE target_table = f"T_{table}" source_schema = f"OU_{schema}" source_schema_lower = f"{source_schema}".lower() source_table = table mapping_name = f"m_MOPDB_{schema}_{target_table}_{source_schema}_{source_table}" workflow_name = f"w_MOPDB_{schema}_{target_table}" content = template_sq.render( source_schema = source_schema_lower, source_table = table, target_schema = schema, target_table = target_table, mapping_name = mapping_name, workflow_name = workflow_name ) filename=f"source_qualifiers/{mapping_name}_SQ.sql" with open(filename, mode="w", encoding="utf-8") as message: message.write(content) print(f"... wrote {filename}") content = template_tgt.render( source_schema = source_schema_lower, source_table = table, target_schema = schema, target_table = target_table, mapping_name = mapping_name, workflow_name = workflow_name ) filename=f"targets/{mapping_name}.sql" with open(filename, mode="w", encoding="utf-8") as message: message.write(content) print(f"... wrote {filename}") content = template_dag.render( table = table ) filename=f"dags/{workflow_name}.py" with open(filename, mode="w", encoding="utf-8") as message: message.write(content) print(f"... wrote {filename}") content = template_db.render( table = table, schema = schema ) filename=f"db/{source_schema}_{source_table}.sql" with open(filename, mode="w", encoding="utf-8") as message: message.write(content) print(f"... wrote {filename}") filename=f"db/table_changes.sql" with open(filename, mode="a", encoding="utf-8") as message: message.write(f"{content}\n") print(f"... wrote {filename}")