from . import oraconn from . import sql_statements from . import utils # Get the next load id from the sequence # # Workflows # def process_source_file_from_event(resource_id: str): # # expects object uri in the form /n//b//o/ # eg /n/frcnomajoc7v/b/dmarsdb1/o/sqlnet.log # and calls process_source_file with prefix and file_name extracted from that uri # _, _, prefix, file_name = utils.parse_uri_with_regex(resource_id) process_source_file(prefix, file_name) def process_source_file(prefix: str, filename: str): sourcefile = f"{prefix.rstrip('/')}/{filename}" # rstrip to cater for cases where the prefix is passed with a trailing slash try: conn = oraconn.connect("MRDS_LOADER") oraconn.run_proc(conn, "CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE", [sourcefile]) conn.commit() finally: conn.close() def execute_query(query, query_parameters=None, account_alias="MRDS_LOADER"): query_result = None try: conn = oraconn.connect(account_alias) curs = conn.cursor() if query_parameters != None: curs.execute(query, query_parameters) else: curs.execute(query) query_result = curs.fetchall() conn.commit() finally: conn.close() return [t[0] for t in query_result] def get_file_prefix(source_key, source_file_id, table_id): query_result = None try: conn = oraconn.connect("MRDS_LOADER") curs = conn.cursor() curs.execute( sql_statements.get_sql("get_file_prefix"), [source_key, source_file_id, table_id], ) query_result = curs.fetchone() conn.commit() finally: conn.close() return query_result[0] def get_inbox_bucket(): try: conn = oraconn.connect("MRDS_LOADER") ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_INBOX_BUCKET", str, []) conn.commit() finally: conn.close() return ret def get_data_bucket(): try: conn = oraconn.connect("MRDS_LOADER") ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_DATA_BUCKET", str, []) conn.commit() finally: conn.close() return ret def add_source_file_config( source_key, source_file_type, source_file_id, source_file_desc, source_file_name_pattern, table_id, template_table_name, ): try: conn = oraconn.connect("MRDS_LOADER") ret = oraconn.run_proc( conn, "CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG", [ source_key, source_file_type, source_file_id, source_file_desc, source_file_name_pattern, table_id, template_table_name, ], ) conn.commit() finally: conn.close() return ret def add_column_date_format(template_table_name, column_name, date_format): try: conn = oraconn.connect("MRDS_LOADER") ret = oraconn.run_proc( conn, "CT_MRDS.FILE_MANAGER.ADD_column_date_format", [template_table_name, column_name, date_format], ) conn.commit() finally: conn.close() return ret def execute(stmt): try: conn = oraconn.connect("MRDS_LOADER") curs = conn.cursor() curs.execute(stmt) conn.commit() finally: conn.close() def create_external_table(table_name, template_table_name, prefix): try: conn = oraconn.connect("ODS_LOADER") ret = oraconn.run_proc( conn, "CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE", [table_name, template_table_name, prefix, get_bucket("ODS")], ) conn.commit() finally: conn.close() return ret def get_bucket(bucket): try: conn = oraconn.connect("MRDS_LOADER") ret = oraconn.run_func( conn, "CT_MRDS.FILE_MANAGER.GET_BUCKET_URI", str, [bucket] ) conn.commit() finally: conn.close() return ret