admin/mars-elt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-02 09:47:35 +01:00
commit 2c225d68ac
715 changed files with 130067 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
dbt/macros/.gitkeep Normal file
View File

19
dbt/macros/check_data_exists.sql Normal file
View File

@@ -0,0 +1,19 @@
{% macro check_data_exists(table_name) %}
{% if execute %}
{% set run_sql %}
select count(*)
from (
select 1 res
from {{ table_name }}
where rownum = 1
)
{% endset %}
{% set res = run_query(run_sql) %}
{% set row_ct = res.columns[0].values()[0] %}
{{ return(row_ct) }}
{% endif %}
{% endmacro %}

16
dbt/macros/common/check_model_exists.sql Normal file
View File

@@ -0,0 +1,16 @@
{% macro check_model_exists(model_name) %}
{% if execute %}
{% if manifest %}
{% set model_exists = false %}
{% for node in manifest.nodes.values() %}
{% if node.resource_type == 'model' and node.name == model_name %}
{% set model_exists = true %}
{% break %}
{% endif %}
{% endfor %}
{{ return(model_exists) }}
{% else %}
{{ exceptions.raise_compiler_error("'manifest' object is not available in this context.") }}
{% endif %}
{% endif %}
{% endmacro %}

15
dbt/macros/common/get_child_models.sql Normal file
View File

@@ -0,0 +1,15 @@
{% macro get_child_models(model_name) %}
{% if execute %}
{% set graph = graph.nodes %}
{% set children = [] %}
{% for node, details in graph.items() %}
{% for item in details.depends_on.nodes %}
{% if model_name == item.split('.')[2] %}
{% do children.append(node.split('.')[2]) %}
{% break %}
{% endif %}
{% endfor %}
{% endfor %}
{{ return(children) }}
{% endif %}
{% endmacro %}

12
dbt/macros/common/get_parent_models.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro get_parent_models(model_name) %}
{% if execute %}
{% set model = ( graph.nodes.values() | selectattr('name', 'equalto', model_name) | list ).pop() %}
{% set depends_on = model.depends_on['nodes'] %}
{% set new_depends_on = [] %}
{% for item in depends_on %}
{% do new_depends_on.append(item.split('.')[2]) %}
{% endfor %}
{{ return(new_depends_on) }}
{% endif %}
{% endmacro %}

12
dbt/macros/common/get_table_qualified_name.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro get_table_qualified_name(model_name) %}
{% if execute %}
{% set models = graph.nodes.values() %}
{% set model = (models | selectattr('name', 'equalto', model_name) | list).pop() %}
{% if model is not none %}
{% set table_qualified_name = model.schema ~ '.' ~ model.alias if model.alias is not none and model.alias != '' else model.schema ~ '.' ~ model.name %}
{{ return(table_qualified_name) }}
{% else %}
{{ exceptions.raise("Model not found: " ~ model_name) }}
{% endif %}
{% endif %}
{% endmacro %}

22
dbt/macros/control_tables/control_external_run_end.sql Normal file
View File

@@ -0,0 +1,22 @@
{% macro control_external_run_end() %}
{{ elog("START macro -> control_external_run_end", "INFO", 0)}}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %}
{{ elog("Get workflow status", "INFO", 1)}}
{% set status = get_workflow_status() %}
{{ check_null("status", status) }}
{{ elog("Run macro update_A_WORKFLOW_HISTORY", "INFO", 1)}}
{{ update_A_WORKFLOW_HISTORY(convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), status) }}
{# {{ elog("Run macro insert_A_MOPDB_LOAD_HISTORY", "INFO", 1)}}
{{ insert_A_MOPDB_LOAD_HISTORY() }}#}
{% do run_query('COMMIT') %}
{% else %}
{{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}}
{% endif %}
{{ elog("END macro -> control_external_run_end", "INFO", 0)}}
{% endmacro %}

13
dbt/macros/control_tables/control_external_run_start.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro control_external_run_start() %}
{{ elog("START macro -> control_external_run_start", "INFO", 0)}}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %}
{{ elog("Run macro insert_A_WORKFLOW_HISTORY", "INFO", 1)}}
{{ insert_A_WORKFLOW_HISTORY() }}
{% do run_query('COMMIT') %}
{% else %}
{{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}}
{% endif %}
{{ elog("END macro -> control_external_run_start", "INFO", 0)}}
{% endmacro %}

66
dbt/macros/control_tables/control_model_end.sql Normal file
View File

@@ -0,0 +1,66 @@
{% macro control_model_end(model_type) %}
{% set model_name = model.name %}
{{ elog("START macro -> control_model_end, model " ~ model_name, "INFO", 0)}}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{{ check_null("a_task_history_key", a_task_history_key) }}
{% if model_type == 'SQ' %}
{% set table_name = this.schema ~ '.' ~ this.name %}
{{ elog("Check if data exists in SQ, model " ~ model_name, "INFO", 1)}}
{% set sq_data_exists = check_data_exists(table_name) %}
{% if 'm_DWH' in model_name and 'm_DWH_PRELOAD' not in model_name %}
{% set model_sources =get_parent_models(model_name) %}
{% if model_sources | length > 0 %}
{% for model_source in model_sources %}
{% set source_name = get_table_qualified_name(model_source) %}
{{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, source_name,'', '') }}
{% endfor %}
{% endif %}
{% else %}
{% set model_sources =model.sources %}
{% for model_source in model_sources %}
{% if model_source[0] != 'control_tables' %}
{% set source_name = source(model_source[0], model_source[1]) %}
{{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, source_name,'', '') }}
{% endif %}
{% endfor %}
{% endif %}
{% elif model_type == 'TRG' %}
{# {{ elog('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', "INFO", 1) }} #}
{% set parent_models = get_parent_models(model_name) %}
{% if parent_models | length > 0 %}
{% for model in parent_models %}
{% if 'm_DWH' in model or 'm_MOPDB' in model %}
{% set table_name = get_table_qualified_name(model) %}
{# {{ elog('--------------->> Table: ', "INFO", 1) }} #}
{# {{ elog(table_name, "INFO", 1) }} #}
{{ elog("Check if data exists in SQ table: " ~ table_name ~ " for model: " ~ model_name, "INFO", 1)}}
{% set trg_data_exists = check_data_exists(table_name) %}
{{ elog("Run control_source_count for source " ~ table_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, table_name,'A_WORKFLOW_HISTORY_KEY', trg_data_exists) }}
{% endif %}
{% endfor %}
{% else %}
{{ elog("There are no parent SQ for model: " ~ model_name, "INFO", 1)}}
{% endif %}
{# {{ log(parent_models, info=True) }} #}
{% endif %}
{{ elog("Run macro update_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), 'Y') }}
{% if sq_data_exists == 0 %}
{{ exceptions.warn("Model " ~ model_name ~ " has no rows.") }}
{% endif %}
{% do run_query('COMMIT') %}
{{ elog("END macro -> control_model_end, model " ~ model_name, "INFO", 0)}}
{% endmacro %}

16
dbt/macros/control_tables/control_model_start.sql Normal file
View File

@@ -0,0 +1,16 @@
{% macro control_model_start() %}
{% set model_name = model.name %}
{{ elog("START macro -> control_model_start, model " ~ model_name, "INFO", 0)}}
{{ elog("Get process_key for model " ~ model_name, "INFO", 1)}}
{% set a_workflow_history_key = get_workflow_history_key() %}
{{ check_null("a_workflow_history_key", a_workflow_history_key) }}
{{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}}
{{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }}
{% do run_query('COMMIT') %}
{{ elog("END macro -> control_model_start, model " ~ model_name, "INFO", 0)}}
{% endmacro %}

49
dbt/macros/control_tables/control_run_end.sql Normal file
View File

@@ -0,0 +1,49 @@
{% macro control_run_end(results) %}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %}
{{ elog("START macro -> control_run_end", "INFO", 0)}}
{% for result in results %}
{% if result.status != 'skipped' %}
{% set model_name = result.node.name %}
{% set rows_affected = result.adapter_response.get('rows_affected', 0) %}
{% if result.status == 'success' %} {% if rows_affected == 0 %}{% set status = 'W' %}{% else %}{% set status = 'Y' %}{% endif %} {% else %} {% set status = 'N' %} {% endif %}
{% set completed_at = result.timing[-1].completed_at.strftime("%Y-%m-%d %H:%M:%S") %}
{% set table_name = result.node.relation_name %}
{%set full_model_key = result.node.unique_id%}
{%set model = graph.nodes.get(full_model_key)%}
{%if model is not none and (result.node.path.startswith('mopdb') or result.node.path.startswith('rar')) %}
{%set model_tags = model.config.get('tags',[])%}
{%set input_service_name_var = var('input_service_name') %}
{%if 'A_TASK_HISTORY_TARGET_insert' not in model_tags and input_service_name_var == 'RAR'%}
{# Skip the following lines and go straight to the end #}
{{elog("RAR model without a A_TASK_HISTORY_TARGET_insert tag. Skip inserting into A_TASK_HISTORY_TARGET for" ~ model_name, "INFO", 1)}}
{%elif model.config.materialized == 'table_if_rows_exist' %}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{%if a_task_history_key != null and rows_affected > 0%}
{{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }}
{% do run_query('COMMIT') %}
{% endif %}
{%else%}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{{ check_null("a_task_history_key", a_task_history_key) }}
{{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }}
{% do run_query('COMMIT') %}
{% endif %}
{%endif%}
{#
{% if status == 'N' %}
{{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(completed_at,'YYYY-MM-DD HH24:MI:SS','UTC'), 'N') }}
{% endif %}
#}
{% endif %}
{% endfor %}
{{ elog("END macro -> control_run_end", "INFO", 0)}}
{% endif %}
{% endmacro %}

12
dbt/macros/control_tables/control_run_start.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro control_run_start() %}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %}
{{ elog("START macro -> control_run_start", "INFO", 0)}}
{{ elog("Checking required variables...", "INFO", 1)}}
{{ check_variable("orchestration_run_id") }}
{{ check_variable("input_service_name") }}
{{ check_variable("workflow_name") }}
{{ elog("END macro -> control_run_start", "INFO", 0)}}
{% endif %}
{% endmacro %}

7
dbt/macros/control_tables/control_source_count.sql Normal file
View File

@@ -0,0 +1,7 @@
{% macro control_source_count(a_task_history_key, table_name, column_name, sq_data_exists) %}
{% set model_name = model.name %}
{{ elog("Run macro insert_A_TASK_HISTORY_SOURCE for model " ~ model_name, "INFO", 2)}}
{{ insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) }}
{% endmacro %}

20
dbt/macros/control_tables/get_task_history_key.sql Normal file
View File

@@ -0,0 +1,20 @@
{% macro get_task_history_key(model_name) %}
{% if execute %}
{% set fetch_fk %}
SELECT A_TASK_HISTORY_KEY
FROM {{ source('control_tables', 'A_TASK_HISTORY') }}
WHERE TASK_RUN_ID = '{{ invocation_id }}'
AND TASK_NAME = '{{ model_name }}'
AND SERVICE_NAME = '{{ var("input_service_name") }}'
{% endset %}
{% set a_task_history_key_result = run_query(fetch_fk) %}
{% set a_task_history_key = a_task_history_key_result.columns[0].values()[0] %}
{{ return(a_task_history_key) }}
{% endif %}
{% endmacro %}

17
dbt/macros/control_tables/get_workflow_history_key.sql Normal file
View File

@@ -0,0 +1,17 @@
{% macro get_workflow_history_key() %}
{% if execute %}
{% set fetch_fk %}
SELECT A_WORKFLOW_HISTORY_KEY
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }}
WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id")}}'
and SERVICE_NAME = '{{ var("input_service_name")}}'
{% endset %}
{% set a_workflow_history_key_result = run_query(fetch_fk) %}
{% set a_workflow_history_key = a_workflow_history_key_result.columns[0].values()[0] %}
{{ return(a_workflow_history_key) }}
{% endif %}
{% endmacro %}

20
dbt/macros/control_tables/get_workflow_status.sql Normal file
View File

@@ -0,0 +1,20 @@
{% macro get_workflow_status() %}
{% if execute %}
{% set run_sql %}
select case when sum(case when tht.load_successful in ('W','Y') then 0 else 1 end) = 0 then 'Y' else 'N' end status
from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh
join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name
join {{ source('control_tables','A_TASK_HISTORY_TARGET') }} tht on th.a_task_history_key = tht.a_task_history_key and th.service_name = tht.service_name
where wh.service_name = '{{ var("input_service_name") }}'
and wh.workflow_name = '{{ var("workflow_name") }}'
and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}'
{% endset %}
{% set res = run_query(run_sql) %}
{% set status = res.columns[0].values()[0] %}
{{ return(status) }}
{% endif %}
{% endmacro %}

25
dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql Normal file
View File

@@ -0,0 +1,25 @@
{% macro insert_A_MOPDB_LOAD_HISTORY() %}
{% if execute %}
{% set insert_A_MOPDB_LOAD_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_MOPDB_LOAD_HISTORY') }} (A_WORKFLOW_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, TASK_NAME, TASK_START, TASK_END, WORKFLOW_STATUS)
select distinct wh.a_workflow_history_key
,ths.a_workflow_history_source_key
,th.a_task_history_key
,th.task_name
,th.task_start
,th.task_end
,wh.workflow_status
from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh
join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name
join {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} ths on th.a_task_history_key = ths.a_task_history_key and th.service_name = ths.service_name
where wh.service_name = '{{ var("input_service_name") }}'
and wh.workflow_name = '{{ var("workflow_name") }}'
and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}'
and ths.a_workflow_history_source_key is not null
{% endset %}
{% do run_query(insert_A_MOPDB_LOAD_HISTORY) %}
{% endif %}
{% endmacro %}

12
dbt/macros/control_tables/insert_A_TASK_HISTORY.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro insert_A_TASK_HISTORY(a_workflow_history_key, model_name) %}
{% if execute %}
{% set insert_A_TASK_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_TASK_HISTORY') }} (A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_KEY, TASK_RUN_ID, TASK_NAME, TASK_START, TASK_SUCCESSFUL, SERVICE_NAME)
VALUES ({{ source('control_sequences','A_TASK_HISTORY_SEQ') }}.NEXTVAL, {{ a_workflow_history_key }}, '{{ invocation_id}}' ,'{{ model_name }}', cast(current_timestamp as timestamp(0)), 'N', '{{ var("input_service_name") }}')
{% endset %}
{% do run_query(insert_A_TASK_HISTORY) %}
{% endif %}
{% endmacro %}

33
dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql Normal file
View File

@@ -0,0 +1,33 @@
{% macro insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) %}
{% if execute %}
{% set insert_A_TASK_HISTORY_SOURCE %}
{% if column_name | length %}
{% if sq_data_exists == 0 %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', 0, '{{ var("input_service_name") }}' FROM dual
{% else %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, col_name, '{{ table_name }}', ct, '{{ var("input_service_name") }}'
FROM (
SELECT {{ column_name }} col_name, count(*) ct
FROM {{ table_name }}
GROUP BY {{ column_name }}
)
{% endif %}
{% else %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', ct, '{{ var("input_service_name") }}'
FROM (
SELECT count(*) ct
FROM {{ table_name }}
)
{% endif %}
{% endset %}
{% do run_query(insert_A_TASK_HISTORY_SOURCE) %}
{% endif %}
{% endmacro %}

13
dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro insert_A_TASK_HISTORY_TARGET(a_task_history_key, model_name, status, rows_affected) %}
{% if execute %}
{% set insert_A_TASK_HISTORY_TARGET %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_TARGET') }} (A_TASK_HISTORY_TARGET_KEY, A_TASK_HISTORY_KEY, TARGET_NAME, ROW_COUNT_APPLIED, LOAD_SUCCESSFUL, SERVICE_NAME)
VALUES ({{ source('control_sequences','A_TASK_HISTORY_TARGET_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ model_name }}', {{ rows_affected }}, '{{ status }}', '{{ var("input_service_name") }}')
{% endset %}
{% do run_query(insert_A_TASK_HISTORY_TARGET) %}
{% endif %}
{% endmacro %}

13
dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro insert_A_WORKFLOW_HISTORY() %}
{% if execute %}
{% set insert_A_WORKFLOW_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} (SERVICE_NAME, A_WORKFLOW_HISTORY_KEY, ORCHESTRATION_RUN_ID, WORKFLOW_NAME, WORKFLOW_START, WORKFLOW_SUCCESSFUL, DBT_INVOCATION_ID)
VALUES ('{{ var("input_service_name") }}', {{ source('control_sequences','A_WORKFLOW_HISTORY_SEQ') }}.NEXTVAL, '{{ var("orchestration_run_id") }}', '{{ var("workflow_name") }}', cast(current_timestamp as timestamp(0)), 'R', '{{ invocation_id }}')
{% endset %}
{% do run_query(insert_A_WORKFLOW_HISTORY) %}
{% endif %}
{% endmacro %}

16
dbt/macros/control_tables/update_A_TASK_HISTORY.sql Normal file
View File

@@ -0,0 +1,16 @@
{% macro update_A_TASK_HISTORY(a_task_history_key, process_end, status) %}
{% if execute %}
{% set update_table %}
UPDATE {{ source('control_tables', 'A_TASK_HISTORY') }}
SET TASK_END = {{ process_end }},
TASK_SUCCESSFUL = '{{ status }}'
WHERE A_TASK_HISTORY_KEY = {{ a_task_history_key }}
{% endset %}
{% do run_query(update_table) %}
{% endif %}
{% endmacro %}

14
dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,14 @@
{% macro update_A_WORKFLOW_HISTORY(process_end, status) %}
{% if execute %}
{% set update_table %}
UPDATE {{ source('control_tables', 'A_WORKFLOW_HISTORY') }}
SET WORKFLOW_END = {{ process_end }},
WORKFLOW_SUCCESSFUL = '{{ status }}'
WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id") }}'
{% endset %}
{% do run_query(update_table) %}
{% endif %}
{% endmacro %}

12
dbt/macros/convert_timezone.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro convert_timezone(value, format, timezone) %}
{% if timezone | length %}
{% set expression %}
cast (FROM_TZ(to_timestamp('{{ value }}', '{{ format }}'), '{{ timezone }}') at time zone '{{ var("dbt_timezone") }}' as timestamp)
{% endset %}
{% else %}
{% set expression %}
to_timestamp('{{ value }}', '{{ format }}')
{% endset %}
{% endif %}
{{ return(expression) }}
{% endmacro %}

8
dbt/macros/data_transform/clean_boolean_fxcd.sql Normal file
View File

@@ -0,0 +1,8 @@
{% macro clean_boolean_fxcd(column_name) %}
{# Replaces Informatica UDF: udf_clean_BOOLEAN_FIELD_STR_FXCD #}
CASE
WHEN {{ column_name }} IS NULL THEN NULL
WHEN UPPER({{ column_name }}) IN ('N', 'NO', '0', ' ') THEN 'N'
ELSE 'Y'
END
{% endmacro %}

21
dbt/macros/data_transform/create_table_from_source.sql Normal file
View File

@@ -0,0 +1,21 @@
{% macro create_table_from_source (schema, table, wf_name, task_name, except) %}
{#
Creates a table with a structure identical to the source table.
The content of the created table are all rows from the source table that belong
to the hightest WORKFLOW_HISTORY_KEY in the source table
#}
{% set a_workflow_history_key = get_workflow_history_key() %}
--{{ check_null("a_workflow_history_key", a_workflow_history_key) }}
{{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}}
{{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }}
select
{{ a_workflow_history_key }} AS A_WORKFLOW_HISTORY_KEY,
source_table.*
from {{ source(schema, table) }} source_table
where source_table.A_WORKFLOW_HISTORY_KEY IN
{{
filter_workflow_history_key(wf_name, task_name)
}}
{% endmacro %}

9
dbt/macros/data_transform/create_table_target.sql Normal file
View File

@@ -0,0 +1,9 @@
{% macro create_table_target (model) %}
{#
Creates a target table with the same structure and content as the source table.
Usually used to move data from SQ tables to the target table.
#}
select
*
from {{ ref(model) }} source_table
{% endmacro %}

52
dbt/macros/data_transform/filter_workflow_history_key.sql Normal file
View File

@@ -0,0 +1,52 @@
{% macro filter_workflow_history_key(wf_name, tasks_names,custom_string='') %}
{#
The query in this macro determines if data in a target table is up to date or if there is additional data in ODS
Paramters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name: The name of the task that executes the target model or an expression that returns it
Logic:
1) The query selects all A_WORKFLOW_HISTORY_KEYs that exist in ODS for the workflow passed as parameter wf_name from the workflow history
2) The subquery on LOAD_HISTORY joins the A_WORKFLOW_HISTORY_KEYs from 1) agains the LOAD_HISTORY for the target table and returns a 1 for all
A_WORKFLOW_HISTORY_KEYs that are found in LOAD_HISTORY
3) The NOT IN operator excludes the A_WORKFLOW_HISTORY_KEYs matched in 2) from the result of query 1, returning only the
A_WORKFLOW_HISTORY_KEYs that were not yet loaded into the target table
Example:
For model m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_LOAD_SQ this macro is called with
wf_name = "w_ODS_TMS_TRANSACTION"
task_name = "substr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', 1, instr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', '_', -1) -1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT DISTINCT A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
{{ custom_string }}
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} )
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %}
AND NOT EXISTS
(SELECT 1
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} )
)
)
{% endmacro %}

52
dbt/macros/data_transform/filter_workflow_history_max_key.sql Normal file
View File

@@ -0,0 +1,52 @@
{% macro filter_workflow_history_max_key(wf_name, task_name) %}
{#
The query in this macro determines if data in a target table is up to date or if there is newer data in ODS
Paramters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name: The name of the task that executes the target model or an expression that returns it
Logic:
1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY for the ODS workflow passed in as wf_name,
assuming that this is the latest data available to be loaded into the target
2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded into the target table.
If it has already been loaded, the query returns a NULL value, indicating that there is no data to be loaded.
If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is returned to the calling model (and likely used there
to select the data from the ODS table)
Example:
For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is called with
wf_name = "w_ODS_CSDB_RATINGS_DEVO"
task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', 1, INSTR(
'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT CASE WHEN (SELECT COUNT(*)
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY IN (WH.A_WORKFLOW_HISTORY_KEY) AND LOAD_HISTORY.TASK_NAME = {{ task_name }}) > 0
THEN NULL
ELSE WH.A_WORKFLOW_HISTORY_KEY END AS A_WORKFLOW_HISTORY_KEY
FROM (SELECT MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) AS A_WORKFLOW_HISTORY_KEY
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} )
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %} {% endif %}) WH)
{% endmacro %}

65
dbt/macros/data_transform/get_ODS_wf_start_max_key.sql Normal file
View File

@@ -0,0 +1,65 @@
{% macro get_ODS_wf_start_max_key(wf_name, tasks_names) %}
{#
The query in this macro determines if data in a target table is up to date or if there is newer data in ODS
Parameters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name : The name of the task that executes the target model or an expression that returns it
Logic:
1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY
for the ODS workflow passed in as wf_name, assuming that this is the latest data available to be loaded
into the target.
2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded
into the target table. If it has already been loaded, the query returns a NULL value, indicating that there
is no data to be loaded. If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is
returned to the calling model (and likely used there to select the data from the ODS table)
Example:
For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is
called with wf_name = "w_ODS_CSDB_RATINGS_DEVO"
task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ',
1, INSTR( 'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{# Set the lookup table based on the input service name #}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT
MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) OVER (PARTITION BY TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) ) as MAX_ODS_WORKFLOW_KEY_DAY,
A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY,
TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) as ODS_WORKFLOW_START
FROM
{{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE
A_WORKFLOW_HISTORY.WORKFLOW_NAME = ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }}
)
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %}
AND NOT EXISTS
(SELECT 1
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} )
)
)
{% endmacro %}

33
dbt/macros/data_transform/sequence_next_value.sql Normal file
View File

@@ -0,0 +1,33 @@
{% macro sequence_next_value(sequence_name, check_disable=false) %}
{# macro checks if a function for this sequence exists,
if it does, the function is used
if not, the standard command is used
check can be disabled by passing true in the 2nd paramater
#}
{% if execute %}
{% if not check_disable %}
{% set get_func %}
select count(*)
from all_objects
where object_type = 'FUNCTION'
and status = 'VALID'
and owner || '.' || object_name = upper('{{ sequence_name }}_fnc')
{% endset %}
{% set result = run_query(get_func) %}
{% set fnc_count = result.columns[0].values()[0] %}
{% else %}
{% set fnc_count = 0 %}
{% endif %}
{% if fnc_count > 0 %}
{% set nextvalue = sequence_name ~ '_fnc()' %}
{% else %}
{% set nextvalue = sequence_name ~ '.nextval'%}
{% endif %}
{{ return(nextvalue) }}
{% endif %}
{% endmacro %}

10
dbt/macros/dbt_overrides/conditional_truncate.sql Normal file
View File

@@ -0,0 +1,10 @@
{% macro conditional_truncate(target_relation, source_relation) %}
DECLARE
row_count NUMBER;
BEGIN
SELECT COUNT(*) INTO row_count FROM {{ source_relation }};
IF row_count > 0 THEN
EXECUTE IMMEDIATE 'TRUNCATE TABLE ' || '{{ target_relation }}';
END IF;
END;
{% endmacro %}

14
dbt/macros/dbt_overrides/generate_schema_name.sql Normal file
View File

@@ -0,0 +1,14 @@
{% macro generate_schema_name(custom_schema_name, node) -%}
{%- set default_schema = target.schema -%}
{%- if custom_schema_name is none -%}
{{ default_schema }}
{%- else -%}
{{ custom_schema_name | trim }}
{%- endif -%}
{%- endmacro %}

9
dbt/macros/dbt_overrides/oracle__snapshot_hash_arguments.sql Normal file
View File

@@ -0,0 +1,9 @@
{% macro oracle__snapshot_hash_arguments(args) -%}
{# Oracle-specific implementation with NULL-safe SCD_ID generation #}
LOWER(RAWTOHEX(STANDARD_HASH(
{%- for arg in args -%}
coalesce(cast({{ arg }} as varchar(4000)), '__DBT_NULL__')
{%- if not loop.last %} || '|' || {%- endif -%}
{%- endfor -%}
, 'SHA256')))
{%- endmacro %}

214
dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql Normal file
View File

@@ -0,0 +1,214 @@
{% macro oracle__snapshot_staging_table(strategy, source_sql, target_relation) -%}
{#--
Custom Oracle snapshot implementation that ensures unified timestamps across all snapshot operations.
SOLUTION: Using strategy.updated_at throughout
- strategy.updated_at generates a single TO_TIMESTAMP literal at Jinja compile time
- This literal is reused everywhere, replacing multiple snapshot_get_time() calls
- Ensures identical timestamps for all records created in the same snapshot run
- Eliminates microsecond differences that occur when unique_key changes (DELETE+INSERT pattern)
Changes from original dbt-oracle v1.9.4 snapshot.sql:
- Line 12: new_scd_id uses strategy.updated_at instead of snapshot_get_time()
- Lines 127-129: deletes CTE uses strategy.updated_at instead of snapshot_get_time() (3 places)
- Lines 174-175: deletion_records CTE uses strategy.updated_at instead of snapshot_get_time() (2 places)
--#}
{% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
{% if strategy.hard_deletes == 'new_record' %}
{% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, strategy.updated_at]) %}
{% endif %}
with snapshot_query as (
{{ source_sql }}
),
snapshotted_data as (
select {{ target_relation }}.*,
{{ unique_key_fields(strategy.unique_key) }}
from {{ target_relation }}
where
{% if config.get('dbt_valid_to_current') %}
{% set source_unique_key = columns.dbt_valid_to | trim %}
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
{% else %}
{{ columns.dbt_valid_to }} is null
{% endif %}
),
insertions_source_data as (
select
snapshot_query.*,
{{ unique_key_fields(strategy.unique_key) }},
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
{{ oracle__get_dbt_valid_to_current(strategy, columns) }},
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }}
from snapshot_query
),
updates_source_data as (
select
snapshot_query.*,
{{ unique_key_fields(strategy.unique_key) }},
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_to }}
from snapshot_query
),
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
deletes_source_data as (
select
snapshot_query.*,
{{ unique_key_fields(strategy.unique_key) }}
from snapshot_query
),
{% endif %}
insertions as (
select
'insert' as dbt_change_type,
source_data.*
{%- if strategy.hard_deletes == 'new_record' -%}
,'False' as {{ columns.dbt_is_deleted }}
{%- endif %}
from insertions_source_data source_data
left outer join snapshotted_data
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }}
or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }})
{%- if strategy.hard_deletes == 'new_record' -%}
or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and snapshotted_data.{{ columns.dbt_is_deleted }} = 'True')
{%- endif %}
)
),
updates as (
select
'update' as dbt_change_type,
source_data.*,
snapshotted_data.{{ columns.dbt_scd_id }}
{%- if strategy.hard_deletes == 'new_record' -%}
, snapshotted_data.{{ columns.dbt_is_deleted }}
{%- endif %}
from updates_source_data source_data
join snapshotted_data
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
where (
{{ strategy.row_changed }}
)
{%- if strategy.hard_deletes == 'new_record' -%}
or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
{%- endif %}
)
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' -%}
,
deletes as (
select
'delete' as dbt_change_type,
source_data.*,
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
{{ strategy.updated_at }} as {{ columns.dbt_valid_to }},
snapshotted_data.{{ columns.dbt_scd_id }}
{%- if strategy.hard_deletes == 'new_record' -%}
, snapshotted_data.{{ columns.dbt_is_deleted }}
{%- endif %}
from snapshotted_data
left join deletes_source_data source_data
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
{%- if strategy.hard_deletes == 'new_record' %}
and not (
--avoid updating the record's valid_to if the latest entry is marked as deleted
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
and
{% if config.get('dbt_valid_to_current') %}
{% set source_unique_key = columns.dbt_valid_to | trim %}
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
{% else %}
{{ columns.dbt_valid_to }} is null
{% endif %}
)
{%- endif %}
)
{%- endif %}
{%- if strategy.hard_deletes == 'new_record' %}
{% set source_sql_cols = get_column_schema_from_query(source_sql) %}
,
deletion_records as (
select
'insert' as dbt_change_type,
{%- for col in source_sql_cols -%}
snapshotted_data.{{ adapter.quote(col.column) }},
{% endfor -%}
{%- if strategy.unique_key | is_list -%}
{%- for key in strategy.unique_key -%}
snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},
{% endfor -%}
{%- else -%}
snapshotted_data.dbt_unique_key as dbt_unique_key,
{% endif -%}
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},
{{ new_scd_id }} as {{ columns.dbt_scd_id }},
'True' as {{ columns.dbt_is_deleted }}
from snapshotted_data
left join deletes_source_data source_data
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
and not (
--avoid inserting a new record if the latest one is marked as deleted
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
and
{% if config.get('dbt_valid_to_current') %}
{% set source_unique_key = columns.dbt_valid_to | trim %}
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
{% else %}
{{ columns.dbt_valid_to }} is null
{% endif %}
)
)
{%- endif %}
select * from insertions
union all
select * from updates
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
union all
select * from deletes
{%- endif %}
{%- if strategy.hard_deletes == 'new_record' %}
union all
select * from deletion_records
{%- endif %}
{%- endmacro %}

14
dbt/macros/dbt_overrides/unique_key_join_on.sql Normal file
View File

@@ -0,0 +1,14 @@
{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}
{% if unique_key | is_list %}
{% for key in unique_key %}
{% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
{% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
{# Use Oracle-specific NULL-safe equality matching oracle__equals macro #}
({{ source_unique_key }} = {{ target_unique_key }} OR ({{ source_unique_key }} IS NULL AND {{ target_unique_key }} IS NULL))
{%- if not loop.last %} and {%- endif %}
{% endfor %}
{% else %}
{# Use Oracle-specific NULL-safe equality for single unique_key as well #}
({{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key OR ({{ identifier }}.dbt_unique_key IS NULL AND {{ from_identifier }}.dbt_unique_key IS NULL))
{% endif %}
{% endmacro %}

7
dbt/macros/error_handlers/check_null.sql Normal file
View File

@@ -0,0 +1,7 @@
{% macro check_null(var_name, var_value) %}
{% if var_value == None or var_value | string == '' %}
{{ elog("Macro local variable " ~ var_name ~ " is empty!", "ERR", 0)}}
{{ exceptions.raise_compiler_error("Variable " ~ var_name ~ " is empty!") }}
{% endif %}
{% endmacro %}

7
dbt/macros/error_handlers/check_variable.sql Normal file
View File

@@ -0,0 +1,7 @@
{% macro check_variable(var_name) %}
{% if var(var_name, none) is none %}
{{ exceptions.raise_compiler_error("The required global variable " ~ var_name ~ " hasn't been provided!") }}
{% endif %}
{% endmacro %}

5
dbt/macros/get_main_task_name.sql Normal file
View File

@@ -0,0 +1,5 @@
{% macro get_main_task_name(source_model_name) %}
{# return({{ source_model_name | split('__')[-1] }}) #}
{% set ret = "substr('" ~ source_model_name ~ "', 1, instr('" ~ source_model_name ~ "', '_', -1) -1)" %}
{{ return(ret) }}
{% endmacro %}

7
dbt/macros/has_tag.sql Normal file
View File

@@ -0,0 +1,7 @@
{% macro has_tag(tag_name, tag_to_run) %}
{% if tag_name == tag_to_run %}
{{ return(true) }}
{% else %}
{{ return(false) }}
{% endif %}
{% endmacro %}

23
dbt/macros/logging/elog.sql Normal file
View File

@@ -0,0 +1,23 @@
{% macro elog(message, level, depth) %}
{% if execute %}
{% set valid_levels = ["INFO", "WARN", "ERR"] %}
{% if level not in valid_levels %}
{{ exceptions.raise_compiler_error("Invalid log level: " ~ level ~ ". Allowed values: " ~ valid_levels | join(", ")) }}
{% endif %}
{% if not depth is number or depth < 0 %}
{{ exception.raise_compiler_error("Depth must be a non-negatibe integer!") }}
{% endif %}
{% if level == "INFO" %}
{% set start_msg = "\033[0m [INFO]: " ~ "*" * depth * 2 %}
{% elif level == "WARN" %}
{% set start_msg = "\033[33m [WARN]: " ~ "*" * depth * 2 %}
{% elif level == "ERR" %}
{% set start_msg = "\033[31m [ERROR]: " ~ "*" * depth * 2 %}
{% endif %}
{{ log(start_msg | trim ~ " " ~ message, info=True)}}
{% endif %}
{% endmacro %}

22
dbt/macros/m_current_timestamp.sql Normal file
View File

@@ -0,0 +1,22 @@
{% macro m_current_timestamp(type) %}
{% if type | lower == 'db' %}
{% if execute %}
{% set get_timestamp %}
SELECT to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS') FROM DUAL
{% endset %}
{% set result = run_query(get_timestamp) %}
{% set current_timestamp = result.columns[0].values()[0] %}
{% endif %}
{% else %}
{% set current_timestamp = modules.datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") %}
{% endif %}
{{ return(current_timestamp) }}
{% endmacro %}

64
dbt/macros/materialization/table_if_rows_exist.sql Normal file
View File

@@ -0,0 +1,64 @@
{% materialization table_if_rows_exist, adapter='oracle' %}
{%- set target_relation = this -%}
-- Count the number of rows in the source table
{#-- Use the compiled SQL of the model to count rows --#}
{%- set count_query %}
select count(*) as row_count from (
{{ model.compiled_sql }}
)
{%- endset %}
{#-- Execute the count query --#}
{%- set result = run_query(count_query) -%}
{%- set num_rows = result.columns[0].values()[0]|int -%}
{#-- Proceed only if there are rows --#}
{%- if num_rows > 0 -%}
{{ log(num_rows ~ ' rows found in source data. Executing model.', info=True) }}
{#-- collect hooks --#}
{% set pre_hooks = control_model_start() %}
{% set post_hooks = control_model_end('TRG') %}
{#-- run pre_hooks --#}
{% do run_hooks(pre_hooks, inside_transaction=True) %}
{#-- Define a temporary relation with a unique name --#}
{%- set tmp_relation = api.Relation.create(
schema=target_relation.schema,
identifier=target_relation.identifier + '_tmp'
) %}
{#-- Create a temporary table with the model's SQL --#}
{% set ctas_sql = 'CREATE TABLE ' ~ tmp_relation ~ ' AS ' ~ model.compiled_sql %}
{% call statement('main') %}
{{ ctas_sql }}
{% endcall %}
{#-- Drop the existing table if it exists --#}
{%- if adapter.get_relation(target_relation.database, target_relation.schema, target_relation.identifier) is not none -%}
{{ log('Dropping target table ' ~ target_relation.identifier, info=True) }}
{% do run_query('DROP TABLE ' ~ target_relation.schema ~ '.' ~ target_relation.identifier) %}
{%- endif -%}
{#-- Rename the temporary table to the target name --#}
{{ log('Renaming temporary table to target table', info=True) }}
{% do run_query('ALTER TABLE ' ~ tmp_relation ~ ' RENAME TO ' ~ target_relation.identifier) %}
{#-- run post_hooks --#}
{% do run_hooks(post_hooks, inside_transaction=True) %}
{%- else -%}
{#-- Do nothing; leave the target table untouched --#}
{{ log('No rows in source data. Skipping model execution.', info=True) }}
{% call statement('main') %}
SELECT 1 FROM DUAL
{% endcall %}
{%- endif -%}
{#-- Return the target relation to dbt --#}
{{ return({'relations': [target_relation]}) }}
{% endmaterialization %}

268
dbt/macros/mopdb/get_sql_t_collateral_default.sql Normal file
View File

@@ -0,0 +1,268 @@
{% macro get_sql_t_collateral_default() %}
-- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }}
select 'MARKET_ASSET' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, -- to find
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, -- to find
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
ISSUER_CAS AS ISSUER_CRED_ASSESS_SOURCE,
ISSUER_RATING_ENUM_VALUE,
ISSUER_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(18 CHAR)) AS NONMKT_ASSET_TYPE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR,
CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP,
CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME,
CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
CAST(NULL AS DATE) AS MATURITY_DATE, -- to find
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS,
ISSUER_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE,
CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY,
CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER,
ISSUER_CRED_PROVIDER AS ISSUER_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED,
CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
TRIPARTY_AGENT,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE,
CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP,
CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE,
CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT,
CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR,
CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID,
CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID,
CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID,
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }}
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
UNION
select 'NON_MARKET_ASSET' AS DATASET,
CAST (NULL AS DATE) AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS,
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS,
ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR,
DEBTOR_NAME,
DEBTOR_ID,
DEBTOR_TYPE AS DEBTOR_GROUP,
DEBTOR_CLASS,
DEBTOR_RESIDENCE,
DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE,
DEBTOR_RATING_ENUM_VALUE,
DEBTOR_RATING_NUMBER_VALUE,
GUAR_NAME AS GUARANTOR_NAME,
GUAR_ID AS GUARANTOR_ID,
GUAR_RESIDENCE AS GUARANTOR_RESIDENCE,
GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE,
GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE,
GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE,
NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
MATURITY_DATE,
GUAR_CLASS AS GUARANTOR_CLASS,
CAST (NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
INTEREST_PAYMENT_TYPE,
RESIDUAL_MATURITY,
BUCKET_SIZE,
DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER,
GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
SECURED_FLAG AS SECURED,
DENOMINATION AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT,
DEBTOR_ID_TYPE AS DEBTOR_IDTYPE,
CAP,
REFERENCE_RATE,
REFERENCE_RATE_COMMENT,
RESET_PERIOD_MORE_ONE_YEAR,
OA_ID,
CONTRACT_ID,
INSTRMNT_ID,
'C2D_ACC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }}
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
UNION
select 'NON_MARKET_DECC' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
ISIN_CODE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
NOM_AMT_SUBMITTED AS COLL_VALUE_BEFORE_HAIRCUTS,
NOM_AMT_SUBMITTED AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR,
CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP,
CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME,
CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
CAST(NULL AS DATE) AS MATURITY_DATE, -- to find
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE,
CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY,
CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED,
CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
TRIPARTY_AGENT,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, -- check
CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, -- check
CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, -- to find
CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, --check
CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, -- check
CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, -- check
CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, -- check
CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, -- check
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }}
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
UNION
select 'NON_MARKET_ASSET' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS,
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS,
ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR,
DEBTOR_NAME,
DEBTOR_ID,
DEBTOR_TYPE AS DEBTOR_GROUP,
DEBTOR_CLASS,
DEBTOR_RESIDENCE,
DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE,
DEBTOR_RATING_ENUM_VALUE,
DEBTOR_RATING_NUMBER_VALUE,
GUAR_NAME AS GUARANTOR_NAME,
GUAR_ID AS GUARANTOR_ID,
GUAR_RESIDENCE AS GUARANTOR_RESIDENCE,
GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE,
GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE,
GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE,
NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
MATURITY_DATE,
GUAR_CLASS AS GUARANTOR_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
INTEREST_PAYMENT_TYPE,
RESIDUAL_MATURITY,
BUCKET_SIZE,
DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER,
GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
SECURED_FLAG AS SECURED,
DENOMINATION AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT,
DEBTOR_ID_TYPE AS DEBTOR_IDTYPE,
CAP,
REFERENCE_RATE,
REFERENCE_RATE_COMMENT,
RESET_PERIOD_MORE_ONE_YEAR,
OA_ID,
CONTRACT_ID,
INSTRMNT_ID,
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }}
WHERE A_WORKFLOW_HISTORY_KEY IS NULL
{% endmacro %}

100
dbt/macros/mopdb/get_sql_t_collateral_ma_dissem.sql Normal file
View File

@@ -0,0 +1,100 @@
{% macro get_sql_t_collateral_ma_dissem() %}
-- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }}
select 'MARKET_ASSET' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, -- to find
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, -- to find
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
ISSUER_CAS AS ISSUER_CRED_ASSESS_SOURCE,
ISSUER_RATING_ENUM_VALUE,
ISSUER_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(18 CHAR)) AS NONMKT_ASSET_TYPE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR,
CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP,
CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME,
CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
CAST(NULL AS DATE) AS MATURITY_DATE, -- to find
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS,
ISSUER_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE,
CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY,
CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER,
ISSUER_CRED_PROVIDER AS ISSUER_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED,
CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
TRIPARTY_AGENT,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE,
CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP,
CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE,
CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT,
CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR,
CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID,
CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID,
CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID,
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }}
WHERE (REPORTING_NCB, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ this }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }}
)
AND DATASET = 'MARKET_ASSET'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY
{% endmacro %}

71
dbt/macros/mopdb/get_sql_t_collateral_nma_acc.sql Normal file
View File

@@ -0,0 +1,71 @@
{% macro get_sql_t_collateral_nma_acc() %}
-- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }}
select 'NON_MARKET_ASSET' AS DATASET,
CAST (NULL AS DATE) AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS,
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS,
ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR,
DEBTOR_NAME,
DEBTOR_ID,
DEBTOR_TYPE AS DEBTOR_GROUP,
DEBTOR_CLASS,
DEBTOR_RESIDENCE,
DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE,
DEBTOR_RATING_ENUM_VALUE,
DEBTOR_RATING_NUMBER_VALUE,
GUAR_NAME AS GUARANTOR_NAME,
GUAR_ID AS GUARANTOR_ID,
GUAR_RESIDENCE AS GUARANTOR_RESIDENCE,
GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE,
GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE,
GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE,
NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
MATURITY_DATE,
GUAR_CLASS AS GUARANTOR_CLASS,
CAST (NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
INTEREST_PAYMENT_TYPE,
RESIDUAL_MATURITY,
BUCKET_SIZE,
DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER,
GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
SECURED_FLAG AS SECURED,
DENOMINATION AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT,
DEBTOR_ID_TYPE AS DEBTOR_IDTYPE,
CAP,
REFERENCE_RATE,
REFERENCE_RATE_COMMENT,
RESET_PERIOD_MORE_ONE_YEAR,
OA_ID,
CONTRACT_ID,
INSTRMNT_ID,
'C2D_ACC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }}
WHERE (reporting_ncb, SNAPSHOT_DATE) IN (
SELECT DISTINCT reporting_ncb, snapshot_date
FROM {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }})
AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY
{% endmacro %}

101
dbt/macros/mopdb/get_sql_t_collateral_nma_decc_dissem.sql Normal file
View File

@@ -0,0 +1,101 @@
{% macro get_sql_t_collateral_nma_decc_dissem() %}
-- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }}
select 'NON_MARKET_DECC' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
ISIN_CODE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
NOM_AMT_SUBMITTED AS COLL_VALUE_BEFORE_HAIRCUTS,
NOM_AMT_SUBMITTED AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR,
CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP,
CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS,
CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE,
CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME,
CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE,
CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE,
CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
CAST(NULL AS DATE) AS MATURITY_DATE, -- to find
CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE,
CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY,
CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE,
CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED,
CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
TRIPARTY_AGENT,
CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, -- check
CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, -- check
CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, -- to find
CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, --check
CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, -- check
CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, -- check
CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, -- check
CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, -- check
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }}
WHERE (reporting_ncb, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ this }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }}
)
AND DATASET = 'NON_MARKET_DECC'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY
{% endmacro %}

106
dbt/macros/mopdb/get_sql_t_collateral_nma_dissem.sql Normal file
View File

@@ -0,0 +1,106 @@
{% macro get_sql_t_collateral_nma_dissem() %}
-- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }}
select 'NON_MARKET_ASSET' AS DATASET,
FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP,
SNAPSHOT_DATE,
REPORTING_NCB AS NCB,
MFI_ID,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE,
OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER,
NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED,
COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS,
COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS,
TYPE_OF_SYSTEM AS SYSTEM_TYPE,
TYPE_OF_OPERATION,
DOM_OR_XBORDER AS DOM_OR_CB,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE,
CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE,
CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE,
NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE,
NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS,
ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR,
DEBTOR_NAME,
DEBTOR_ID,
DEBTOR_TYPE AS DEBTOR_GROUP,
DEBTOR_CLASS,
DEBTOR_RESIDENCE,
DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE,
DEBTOR_RATING_ENUM_VALUE,
DEBTOR_RATING_NUMBER_VALUE,
GUAR_NAME AS GUARANTOR_NAME,
GUAR_ID AS GUARANTOR_ID,
GUAR_RESIDENCE AS GUARANTOR_RESIDENCE,
GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE,
GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE,
GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE,
NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS,
NCB_COMMENT AS NCB_COMMENTS,
MATURITY_DATE,
GUAR_CLASS AS GUARANTOR_CLASS,
CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS,
INTEREST_PAYMENT_TYPE,
RESIDUAL_MATURITY,
BUCKET_SIZE,
DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER,
GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER,
CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER,
SECURED_FLAG AS SECURED,
DENOMINATION AS NON_MARKET_DENOMINATION,
MOBILISATION_CHANNEL,
CCB,
INVESTOR_SSS,
INTERMEDIARY_SSS,
ISSUER_SSS,
CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT,
DEBTOR_ID_TYPE AS DEBTOR_IDTYPE,
CAP,
REFERENCE_RATE,
REFERENCE_RATE_COMMENT,
RESET_PERIOD_MORE_ONE_YEAR,
OA_ID,
CONTRACT_ID,
INSTRMNT_ID,
'C2D_UC' AS SOURCE,
{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY
from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }}
WHERE (reporting_ncb, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ this }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }}
)
AND DATASET = 'NON_MARKET_ASSET'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY
AND (reporting_ncb, SNAPSHOT_DATE) NOT IN (
SELECT DISTINCT REPORTING_NCB, SNAPSHOT_DATE
FROM {{ this }}
WHERE SOURCE = 'C2D_ACC'
)
{% endmacro %}

126
dbt/macros/mopdb/get_sql_t_collateral_prehook.sql Normal file
View File

@@ -0,0 +1,126 @@
{% macro get_sql_t_collateral_prehook( model, source_model ) %}
{% if var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM' %}
DELETE FROM {{ model }}
WHERE DATASET = 'MARKET_ASSET'
AND SOURCE = 'C2D_UC'
AND (NCB, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ source_model }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ model }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ source_model }}
)
AND DATASET = 'MARKET_ASSET'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
{% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC' %}
DELETE FROM {{ model }}
WHERE DATASET = 'NON_MARKET_ASSET'
AND (NCB, SNAPSHOT_DATE) IN (
SELECT DISTINCT reporting_ncb, snapshot_date
FROM {{ source_model }}
{% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM' %}
DELETE FROM {{ model }}
WHERE DATASET = 'NON_MARKET_DECC'
AND SOURCE = 'C2D_UC'
AND (NCB, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ source_model }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ model }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ source_model }}
)
AND DATASET = 'NON_MARKET_DECC'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
{% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM' %}
DELETE FROM {{ model }}
WHERE DATASET = 'NON_MARKET_ASSET'
AND SOURCE = 'C2D_UC'
AND (NCB, SNAPSHOT_DATE) in (
SELECT DISTINCT
src.reporting_ncb,
src.snapshot_date
FROM
{{ source_model }} src
full outer join (SELECT DISTINCT
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
FROM
{{ model }}
WHERE
( ncb,
snapshot_date ) IN (
SELECT DISTINCT
reporting_ncb,
snapshot_date
FROM
{{ source_model }}
)
AND DATASET = 'NON_MARKET_ASSET'
AND SOURCE = 'C2D_UC'
GROUP BY
ncb,
snapshot_date,
DATABASE_ENTRY_TIMESTAMP
) trg on src.reporting_ncb = trg.ncb
and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE)
WHERE
src.max_a_workflow_history_key = src.a_workflow_history_key
and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1)
)
{% endif %}
{% endmacro %}

46
dbt/macros/oracle/list_relations_without_caching.sql Normal file
View File

@@ -0,0 +1,46 @@
#
# This macro overwrites the macro with the same name in the dbt-oracle adapter, which was found in
# /lib/python3.11/site-packages/dbt/include/oracle/macros/adapters.sql
# Reason: The SQL query in the original macro had severe performance issues when accessing sys.all_views
#
{% macro oracle__list_relations_without_caching(schema_relation) %}
{% call statement('list_relations_without_caching', fetch_result=True) -%}
with tables as
(select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog,
owner table_schema,
table_name,
case
when iot_type = 'Y'
then 'IOT'
when temporary = 'Y'
then 'TEMP'
else 'BASE TABLE'
end table_type
from sys.all_tables
where upper(table_name) not in (
select upper(mview_name)
from sys.all_mviews
where upper(owner) = upper('{{ schema_relation.schema }}')
)
union all
select SYS_CONTEXT('userenv', 'DB_NAME'),
owner,
object_name,
object_type
from sys.all_objects
WHERE object_type in ('VIEW', 'MATERIALIZED VIEW')
)
select table_catalog as "database_name"
,table_name as "name"
,table_schema as "schema_name"
,case table_type
when 'BASE TABLE' then 'table'
when 'VIEW' then 'view'
when 'MATERIALIZED VIEW' then 'materialized_view'
end as "kind"
from tables
where table_type in ('BASE TABLE', 'VIEW', 'MATERIALIZED VIEW')
and upper(table_schema) = upper('{{ schema_relation.schema }}')
{% endcall %}
{{ return(load_result('list_relations_without_caching').table) }}
{% endmacro %}