74 lines
2.4 KiB
Python
74 lines
2.4 KiB
Python
from jinja2 import Environment, FileSystemLoader
|
|
import csv
|
|
|
|
environment = Environment(loader=FileSystemLoader("templates/"))
|
|
template_sq = environment.get_template("m_Template_sq.sql")
|
|
template_tgt = environment.get_template("m_Template_target.sql")
|
|
template_db = environment.get_template("db_Template.sql")
|
|
template_dag = environment.get_template("dag_Template.py")
|
|
|
|
csvFile = csv.reader(open("ods_mopdb_plain.txt", "r"))
|
|
header = next(csvFile) # skip header
|
|
|
|
for row in csvFile:
|
|
|
|
(schema, table) = row
|
|
|
|
# ou_tms,ACTIVITYLOGDUE,TMS,T_ACTIVITYLOGDUE,m_MOPDB_TMS_T_ACTIVITYLOGDUE_OU_TMS_ACTIVITYLOGDUE,w_ODS_TMS_ACTIVITYLOGDUE
|
|
|
|
target_table = f"T_{table}"
|
|
source_schema = f"OU_{schema}"
|
|
source_schema_lower = f"{source_schema}".lower()
|
|
source_table = table
|
|
mapping_name = f"m_MOPDB_{schema}_{target_table}_{source_schema}_{source_table}"
|
|
workflow_name = f"w_MOPDB_{schema}_{target_table}"
|
|
|
|
|
|
content = template_sq.render(
|
|
source_schema = source_schema_lower,
|
|
source_table = table,
|
|
target_schema = schema,
|
|
target_table = target_table,
|
|
mapping_name = mapping_name,
|
|
workflow_name = workflow_name
|
|
)
|
|
filename=f"source_qualifiers/{mapping_name}_SQ.sql"
|
|
with open(filename, mode="w", encoding="utf-8") as message:
|
|
message.write(content)
|
|
print(f"... wrote {filename}")
|
|
|
|
content = template_tgt.render(
|
|
source_schema = source_schema_lower,
|
|
source_table = table,
|
|
target_schema = schema,
|
|
target_table = target_table,
|
|
mapping_name = mapping_name,
|
|
workflow_name = workflow_name
|
|
)
|
|
filename=f"targets/{mapping_name}.sql"
|
|
with open(filename, mode="w", encoding="utf-8") as message:
|
|
message.write(content)
|
|
print(f"... wrote {filename}")
|
|
|
|
content = template_dag.render(
|
|
table = table
|
|
)
|
|
filename=f"dags/{workflow_name}.py"
|
|
with open(filename, mode="w", encoding="utf-8") as message:
|
|
message.write(content)
|
|
print(f"... wrote {filename}")
|
|
|
|
content = template_db.render(
|
|
table = table,
|
|
schema = schema
|
|
)
|
|
filename=f"db/{source_schema}_{source_table}.sql"
|
|
with open(filename, mode="w", encoding="utf-8") as message:
|
|
message.write(content)
|
|
print(f"... wrote {filename}")
|
|
filename=f"db/table_changes.sql"
|
|
with open(filename, mode="a", encoding="utf-8") as message:
|
|
message.write(f"{content}\n")
|
|
print(f"... wrote {filename}")
|
|
|