178 lines
4.1 KiB
Python
178 lines
4.1 KiB
Python
from . import oraconn
|
|
from . import sql_statements
|
|
from . import utils
|
|
|
|
# Get the next load id from the sequence
|
|
|
|
#
|
|
# Workflows
|
|
#
|
|
|
|
|
|
def process_source_file_from_event(resource_id: str):
|
|
#
|
|
# expects object uri in the form /n/<namespace>/b/<bucket>/o/<object>
|
|
# eg /n/frcnomajoc7v/b/dmarsdb1/o/sqlnet.log
|
|
# and calls process_source_file with prefix and file_name extracted from that uri
|
|
#
|
|
|
|
_, _, prefix, file_name = utils.parse_uri_with_regex(resource_id)
|
|
process_source_file(prefix, file_name)
|
|
|
|
|
|
def process_source_file(prefix: str, filename: str):
|
|
|
|
sourcefile = f"{prefix.rstrip('/')}/{filename}" # rstrip to cater for cases where the prefix is passed with a trailing slash
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
oraconn.run_proc(conn, "CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE", [sourcefile])
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def execute_query(query, query_parameters=None, account_alias="MRDS_LOADER"):
|
|
query_result = None
|
|
try:
|
|
conn = oraconn.connect(account_alias)
|
|
curs = conn.cursor()
|
|
if query_parameters != None:
|
|
curs.execute(query, query_parameters)
|
|
else:
|
|
curs.execute(query)
|
|
query_result = curs.fetchall()
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return [t[0] for t in query_result]
|
|
|
|
|
|
def get_file_prefix(source_key, source_file_id, table_id):
|
|
query_result = None
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
curs = conn.cursor()
|
|
|
|
curs.execute(
|
|
sql_statements.get_sql("get_file_prefix"),
|
|
[source_key, source_file_id, table_id],
|
|
)
|
|
query_result = curs.fetchone()
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return query_result[0]
|
|
|
|
|
|
def get_inbox_bucket():
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_INBOX_BUCKET", str, [])
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|
|
|
|
|
|
def get_data_bucket():
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_DATA_BUCKET", str, [])
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|
|
|
|
|
|
def add_source_file_config(
|
|
source_key,
|
|
source_file_type,
|
|
source_file_id,
|
|
source_file_desc,
|
|
source_file_name_pattern,
|
|
table_id,
|
|
template_table_name,
|
|
):
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
ret = oraconn.run_proc(
|
|
conn,
|
|
"CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG",
|
|
[
|
|
source_key,
|
|
source_file_type,
|
|
source_file_id,
|
|
source_file_desc,
|
|
source_file_name_pattern,
|
|
table_id,
|
|
template_table_name,
|
|
],
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|
|
|
|
|
|
def add_column_date_format(template_table_name, column_name, date_format):
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
ret = oraconn.run_proc(
|
|
conn,
|
|
"CT_MRDS.FILE_MANAGER.ADD_column_date_format",
|
|
[template_table_name, column_name, date_format],
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|
|
|
|
|
|
def execute(stmt):
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
curs = conn.cursor()
|
|
curs.execute(stmt)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def create_external_table(table_name, template_table_name, prefix):
|
|
try:
|
|
conn = oraconn.connect("ODS_LOADER")
|
|
|
|
ret = oraconn.run_proc(
|
|
conn,
|
|
"CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE",
|
|
[table_name, template_table_name, prefix, get_bucket("ODS")],
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|
|
|
|
|
|
def get_bucket(bucket):
|
|
try:
|
|
conn = oraconn.connect("MRDS_LOADER")
|
|
|
|
ret = oraconn.run_func(
|
|
conn, "CT_MRDS.FILE_MANAGER.GET_BUCKET_URI", str, [bucket]
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
return ret
|