admin/mars-elt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-02 09:47:35 +01:00
commit 2c225d68ac
715 changed files with 130067 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
dbt/macros/control_tables/control_external_run_end.sql Normal file
View File

@@ -0,0 +1,22 @@
{% macro control_external_run_end() %}
{{ elog("START macro -> control_external_run_end", "INFO", 0)}}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %}
{{ elog("Get workflow status", "INFO", 1)}}
{% set status = get_workflow_status() %}
{{ check_null("status", status) }}
{{ elog("Run macro update_A_WORKFLOW_HISTORY", "INFO", 1)}}
{{ update_A_WORKFLOW_HISTORY(convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), status) }}
{# {{ elog("Run macro insert_A_MOPDB_LOAD_HISTORY", "INFO", 1)}}
{{ insert_A_MOPDB_LOAD_HISTORY() }}#}
{% do run_query('COMMIT') %}
{% else %}
{{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}}
{% endif %}
{{ elog("END macro -> control_external_run_end", "INFO", 0)}}
{% endmacro %}

13
dbt/macros/control_tables/control_external_run_start.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro control_external_run_start() %}
{{ elog("START macro -> control_external_run_start", "INFO", 0)}}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %}
{{ elog("Run macro insert_A_WORKFLOW_HISTORY", "INFO", 1)}}
{{ insert_A_WORKFLOW_HISTORY() }}
{% do run_query('COMMIT') %}
{% else %}
{{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}}
{% endif %}
{{ elog("END macro -> control_external_run_start", "INFO", 0)}}
{% endmacro %}

66
dbt/macros/control_tables/control_model_end.sql Normal file
View File

@@ -0,0 +1,66 @@
{% macro control_model_end(model_type) %}
{% set model_name = model.name %}
{{ elog("START macro -> control_model_end, model " ~ model_name, "INFO", 0)}}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{{ check_null("a_task_history_key", a_task_history_key) }}
{% if model_type == 'SQ' %}
{% set table_name = this.schema ~ '.' ~ this.name %}
{{ elog("Check if data exists in SQ, model " ~ model_name, "INFO", 1)}}
{% set sq_data_exists = check_data_exists(table_name) %}
{% if 'm_DWH' in model_name and 'm_DWH_PRELOAD' not in model_name %}
{% set model_sources =get_parent_models(model_name) %}
{% if model_sources | length > 0 %}
{% for model_source in model_sources %}
{% set source_name = get_table_qualified_name(model_source) %}
{{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, source_name,'', '') }}
{% endfor %}
{% endif %}
{% else %}
{% set model_sources =model.sources %}
{% for model_source in model_sources %}
{% if model_source[0] != 'control_tables' %}
{% set source_name = source(model_source[0], model_source[1]) %}
{{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, source_name,'', '') }}
{% endif %}
{% endfor %}
{% endif %}
{% elif model_type == 'TRG' %}
{# {{ elog('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', "INFO", 1) }} #}
{% set parent_models = get_parent_models(model_name) %}
{% if parent_models | length > 0 %}
{% for model in parent_models %}
{% if 'm_DWH' in model or 'm_MOPDB' in model %}
{% set table_name = get_table_qualified_name(model) %}
{# {{ elog('--------------->> Table: ', "INFO", 1) }} #}
{# {{ elog(table_name, "INFO", 1) }} #}
{{ elog("Check if data exists in SQ table: " ~ table_name ~ " for model: " ~ model_name, "INFO", 1)}}
{% set trg_data_exists = check_data_exists(table_name) %}
{{ elog("Run control_source_count for source " ~ table_name ~ ", model " ~ model_name, "INFO", 1)}}
{{ control_source_count(a_task_history_key, table_name,'A_WORKFLOW_HISTORY_KEY', trg_data_exists) }}
{% endif %}
{% endfor %}
{% else %}
{{ elog("There are no parent SQ for model: " ~ model_name, "INFO", 1)}}
{% endif %}
{# {{ log(parent_models, info=True) }} #}
{% endif %}
{{ elog("Run macro update_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), 'Y') }}
{% if sq_data_exists == 0 %}
{{ exceptions.warn("Model " ~ model_name ~ " has no rows.") }}
{% endif %}
{% do run_query('COMMIT') %}
{{ elog("END macro -> control_model_end, model " ~ model_name, "INFO", 0)}}
{% endmacro %}

16
dbt/macros/control_tables/control_model_start.sql Normal file
View File

@@ -0,0 +1,16 @@
{% macro control_model_start() %}
{% set model_name = model.name %}
{{ elog("START macro -> control_model_start, model " ~ model_name, "INFO", 0)}}
{{ elog("Get process_key for model " ~ model_name, "INFO", 1)}}
{% set a_workflow_history_key = get_workflow_history_key() %}
{{ check_null("a_workflow_history_key", a_workflow_history_key) }}
{{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}}
{{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }}
{% do run_query('COMMIT') %}
{{ elog("END macro -> control_model_start, model " ~ model_name, "INFO", 0)}}
{% endmacro %}

49
dbt/macros/control_tables/control_run_end.sql Normal file
View File

@@ -0,0 +1,49 @@
{% macro control_run_end(results) %}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %}
{{ elog("START macro -> control_run_end", "INFO", 0)}}
{% for result in results %}
{% if result.status != 'skipped' %}
{% set model_name = result.node.name %}
{% set rows_affected = result.adapter_response.get('rows_affected', 0) %}
{% if result.status == 'success' %} {% if rows_affected == 0 %}{% set status = 'W' %}{% else %}{% set status = 'Y' %}{% endif %} {% else %} {% set status = 'N' %} {% endif %}
{% set completed_at = result.timing[-1].completed_at.strftime("%Y-%m-%d %H:%M:%S") %}
{% set table_name = result.node.relation_name %}
{%set full_model_key = result.node.unique_id%}
{%set model = graph.nodes.get(full_model_key)%}
{%if model is not none and (result.node.path.startswith('mopdb') or result.node.path.startswith('rar')) %}
{%set model_tags = model.config.get('tags',[])%}
{%set input_service_name_var = var('input_service_name') %}
{%if 'A_TASK_HISTORY_TARGET_insert' not in model_tags and input_service_name_var == 'RAR'%}
{# Skip the following lines and go straight to the end #}
{{elog("RAR model without a A_TASK_HISTORY_TARGET_insert tag. Skip inserting into A_TASK_HISTORY_TARGET for" ~ model_name, "INFO", 1)}}
{%elif model.config.materialized == 'table_if_rows_exist' %}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{%if a_task_history_key != null and rows_affected > 0%}
{{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }}
{% do run_query('COMMIT') %}
{% endif %}
{%else%}
{{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}}
{% set a_task_history_key = get_task_history_key(model_name) %}
{{ check_null("a_task_history_key", a_task_history_key) }}
{{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }}
{% do run_query('COMMIT') %}
{% endif %}
{%endif%}
{#
{% if status == 'N' %}
{{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(completed_at,'YYYY-MM-DD HH24:MI:SS','UTC'), 'N') }}
{% endif %}
#}
{% endif %}
{% endfor %}
{{ elog("END macro -> control_run_end", "INFO", 0)}}
{% endif %}
{% endmacro %}

12
dbt/macros/control_tables/control_run_start.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro control_run_start() %}
{% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %}
{{ elog("START macro -> control_run_start", "INFO", 0)}}
{{ elog("Checking required variables...", "INFO", 1)}}
{{ check_variable("orchestration_run_id") }}
{{ check_variable("input_service_name") }}
{{ check_variable("workflow_name") }}
{{ elog("END macro -> control_run_start", "INFO", 0)}}
{% endif %}
{% endmacro %}

7
dbt/macros/control_tables/control_source_count.sql Normal file
View File

@@ -0,0 +1,7 @@
{% macro control_source_count(a_task_history_key, table_name, column_name, sq_data_exists) %}
{% set model_name = model.name %}
{{ elog("Run macro insert_A_TASK_HISTORY_SOURCE for model " ~ model_name, "INFO", 2)}}
{{ insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) }}
{% endmacro %}

20
dbt/macros/control_tables/get_task_history_key.sql Normal file
View File

@@ -0,0 +1,20 @@
{% macro get_task_history_key(model_name) %}
{% if execute %}
{% set fetch_fk %}
SELECT A_TASK_HISTORY_KEY
FROM {{ source('control_tables', 'A_TASK_HISTORY') }}
WHERE TASK_RUN_ID = '{{ invocation_id }}'
AND TASK_NAME = '{{ model_name }}'
AND SERVICE_NAME = '{{ var("input_service_name") }}'
{% endset %}
{% set a_task_history_key_result = run_query(fetch_fk) %}
{% set a_task_history_key = a_task_history_key_result.columns[0].values()[0] %}
{{ return(a_task_history_key) }}
{% endif %}
{% endmacro %}

17
dbt/macros/control_tables/get_workflow_history_key.sql Normal file
View File

@@ -0,0 +1,17 @@
{% macro get_workflow_history_key() %}
{% if execute %}
{% set fetch_fk %}
SELECT A_WORKFLOW_HISTORY_KEY
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }}
WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id")}}'
and SERVICE_NAME = '{{ var("input_service_name")}}'
{% endset %}
{% set a_workflow_history_key_result = run_query(fetch_fk) %}
{% set a_workflow_history_key = a_workflow_history_key_result.columns[0].values()[0] %}
{{ return(a_workflow_history_key) }}
{% endif %}
{% endmacro %}

20
dbt/macros/control_tables/get_workflow_status.sql Normal file
View File

@@ -0,0 +1,20 @@
{% macro get_workflow_status() %}
{% if execute %}
{% set run_sql %}
select case when sum(case when tht.load_successful in ('W','Y') then 0 else 1 end) = 0 then 'Y' else 'N' end status
from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh
join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name
join {{ source('control_tables','A_TASK_HISTORY_TARGET') }} tht on th.a_task_history_key = tht.a_task_history_key and th.service_name = tht.service_name
where wh.service_name = '{{ var("input_service_name") }}'
and wh.workflow_name = '{{ var("workflow_name") }}'
and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}'
{% endset %}
{% set res = run_query(run_sql) %}
{% set status = res.columns[0].values()[0] %}
{{ return(status) }}
{% endif %}
{% endmacro %}

25
dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql Normal file
View File

@@ -0,0 +1,25 @@
{% macro insert_A_MOPDB_LOAD_HISTORY() %}
{% if execute %}
{% set insert_A_MOPDB_LOAD_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_MOPDB_LOAD_HISTORY') }} (A_WORKFLOW_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, TASK_NAME, TASK_START, TASK_END, WORKFLOW_STATUS)
select distinct wh.a_workflow_history_key
,ths.a_workflow_history_source_key
,th.a_task_history_key
,th.task_name
,th.task_start
,th.task_end
,wh.workflow_status
from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh
join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name
join {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} ths on th.a_task_history_key = ths.a_task_history_key and th.service_name = ths.service_name
where wh.service_name = '{{ var("input_service_name") }}'
and wh.workflow_name = '{{ var("workflow_name") }}'
and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}'
and ths.a_workflow_history_source_key is not null
{% endset %}
{% do run_query(insert_A_MOPDB_LOAD_HISTORY) %}
{% endif %}
{% endmacro %}

12
dbt/macros/control_tables/insert_A_TASK_HISTORY.sql Normal file
View File

@@ -0,0 +1,12 @@
{% macro insert_A_TASK_HISTORY(a_workflow_history_key, model_name) %}
{% if execute %}
{% set insert_A_TASK_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_TASK_HISTORY') }} (A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_KEY, TASK_RUN_ID, TASK_NAME, TASK_START, TASK_SUCCESSFUL, SERVICE_NAME)
VALUES ({{ source('control_sequences','A_TASK_HISTORY_SEQ') }}.NEXTVAL, {{ a_workflow_history_key }}, '{{ invocation_id}}' ,'{{ model_name }}', cast(current_timestamp as timestamp(0)), 'N', '{{ var("input_service_name") }}')
{% endset %}
{% do run_query(insert_A_TASK_HISTORY) %}
{% endif %}
{% endmacro %}

33
dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql Normal file
View File

@@ -0,0 +1,33 @@
{% macro insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) %}
{% if execute %}
{% set insert_A_TASK_HISTORY_SOURCE %}
{% if column_name | length %}
{% if sq_data_exists == 0 %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', 0, '{{ var("input_service_name") }}' FROM dual
{% else %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, col_name, '{{ table_name }}', ct, '{{ var("input_service_name") }}'
FROM (
SELECT {{ column_name }} col_name, count(*) ct
FROM {{ table_name }}
GROUP BY {{ column_name }}
)
{% endif %}
{% else %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME)
SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', ct, '{{ var("input_service_name") }}'
FROM (
SELECT count(*) ct
FROM {{ table_name }}
)
{% endif %}
{% endset %}
{% do run_query(insert_A_TASK_HISTORY_SOURCE) %}
{% endif %}
{% endmacro %}

13
dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro insert_A_TASK_HISTORY_TARGET(a_task_history_key, model_name, status, rows_affected) %}
{% if execute %}
{% set insert_A_TASK_HISTORY_TARGET %}
INSERT INTO {{ source('control_tables','A_TASK_HISTORY_TARGET') }} (A_TASK_HISTORY_TARGET_KEY, A_TASK_HISTORY_KEY, TARGET_NAME, ROW_COUNT_APPLIED, LOAD_SUCCESSFUL, SERVICE_NAME)
VALUES ({{ source('control_sequences','A_TASK_HISTORY_TARGET_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ model_name }}', {{ rows_affected }}, '{{ status }}', '{{ var("input_service_name") }}')
{% endset %}
{% do run_query(insert_A_TASK_HISTORY_TARGET) %}
{% endif %}
{% endmacro %}

13
dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,13 @@
{% macro insert_A_WORKFLOW_HISTORY() %}
{% if execute %}
{% set insert_A_WORKFLOW_HISTORY %}
INSERT INTO {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} (SERVICE_NAME, A_WORKFLOW_HISTORY_KEY, ORCHESTRATION_RUN_ID, WORKFLOW_NAME, WORKFLOW_START, WORKFLOW_SUCCESSFUL, DBT_INVOCATION_ID)
VALUES ('{{ var("input_service_name") }}', {{ source('control_sequences','A_WORKFLOW_HISTORY_SEQ') }}.NEXTVAL, '{{ var("orchestration_run_id") }}', '{{ var("workflow_name") }}', cast(current_timestamp as timestamp(0)), 'R', '{{ invocation_id }}')
{% endset %}
{% do run_query(insert_A_WORKFLOW_HISTORY) %}
{% endif %}
{% endmacro %}

16
dbt/macros/control_tables/update_A_TASK_HISTORY.sql Normal file
View File

@@ -0,0 +1,16 @@
{% macro update_A_TASK_HISTORY(a_task_history_key, process_end, status) %}
{% if execute %}
{% set update_table %}
UPDATE {{ source('control_tables', 'A_TASK_HISTORY') }}
SET TASK_END = {{ process_end }},
TASK_SUCCESSFUL = '{{ status }}'
WHERE A_TASK_HISTORY_KEY = {{ a_task_history_key }}
{% endset %}
{% do run_query(update_table) %}
{% endif %}
{% endmacro %}

14
dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql Normal file
View File

@@ -0,0 +1,14 @@
{% macro update_A_WORKFLOW_HISTORY(process_end, status) %}
{% if execute %}
{% set update_table %}
UPDATE {{ source('control_tables', 'A_WORKFLOW_HISTORY') }}
SET WORKFLOW_END = {{ process_end }},
WORKFLOW_SUCCESSFUL = '{{ status }}'
WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id") }}'
{% endset %}
{% do run_query(update_table) %}
{% endif %}
{% endmacro %}