admin/mars-elt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-02 09:47:35 +01:00
commit 2c225d68ac
715 changed files with 130067 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dbt/macros/data_transform/clean_boolean_fxcd.sql Normal file
View File

@@ -0,0 +1,8 @@
{% macro clean_boolean_fxcd(column_name) %}
{# Replaces Informatica UDF: udf_clean_BOOLEAN_FIELD_STR_FXCD #}
CASE
WHEN {{ column_name }} IS NULL THEN NULL
WHEN UPPER({{ column_name }}) IN ('N', 'NO', '0', ' ') THEN 'N'
ELSE 'Y'
END
{% endmacro %}

21
dbt/macros/data_transform/create_table_from_source.sql Normal file
View File

@@ -0,0 +1,21 @@
{% macro create_table_from_source (schema, table, wf_name, task_name, except) %}
{#
Creates a table with a structure identical to the source table.
The content of the created table are all rows from the source table that belong
to the hightest WORKFLOW_HISTORY_KEY in the source table
#}
{% set a_workflow_history_key = get_workflow_history_key() %}
--{{ check_null("a_workflow_history_key", a_workflow_history_key) }}
{{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}}
{{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}}
{{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }}
select
{{ a_workflow_history_key }} AS A_WORKFLOW_HISTORY_KEY,
source_table.*
from {{ source(schema, table) }} source_table
where source_table.A_WORKFLOW_HISTORY_KEY IN
{{
filter_workflow_history_key(wf_name, task_name)
}}
{% endmacro %}

9
dbt/macros/data_transform/create_table_target.sql Normal file
View File

@@ -0,0 +1,9 @@
{% macro create_table_target (model) %}
{#
Creates a target table with the same structure and content as the source table.
Usually used to move data from SQ tables to the target table.
#}
select
*
from {{ ref(model) }} source_table
{% endmacro %}

52
dbt/macros/data_transform/filter_workflow_history_key.sql Normal file
View File

@@ -0,0 +1,52 @@
{% macro filter_workflow_history_key(wf_name, tasks_names,custom_string='') %}
{#
The query in this macro determines if data in a target table is up to date or if there is additional data in ODS
Paramters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name: The name of the task that executes the target model or an expression that returns it
Logic:
1) The query selects all A_WORKFLOW_HISTORY_KEYs that exist in ODS for the workflow passed as parameter wf_name from the workflow history
2) The subquery on LOAD_HISTORY joins the A_WORKFLOW_HISTORY_KEYs from 1) agains the LOAD_HISTORY for the target table and returns a 1 for all
A_WORKFLOW_HISTORY_KEYs that are found in LOAD_HISTORY
3) The NOT IN operator excludes the A_WORKFLOW_HISTORY_KEYs matched in 2) from the result of query 1, returning only the
A_WORKFLOW_HISTORY_KEYs that were not yet loaded into the target table
Example:
For model m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_LOAD_SQ this macro is called with
wf_name = "w_ODS_TMS_TRANSACTION"
task_name = "substr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', 1, instr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', '_', -1) -1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT DISTINCT A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
{{ custom_string }}
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} )
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %}
AND NOT EXISTS
(SELECT 1
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} )
)
)
{% endmacro %}

52
dbt/macros/data_transform/filter_workflow_history_max_key.sql Normal file
View File

@@ -0,0 +1,52 @@
{% macro filter_workflow_history_max_key(wf_name, task_name) %}
{#
The query in this macro determines if data in a target table is up to date or if there is newer data in ODS
Paramters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name: The name of the task that executes the target model or an expression that returns it
Logic:
1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY for the ODS workflow passed in as wf_name,
assuming that this is the latest data available to be loaded into the target
2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded into the target table.
If it has already been loaded, the query returns a NULL value, indicating that there is no data to be loaded.
If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is returned to the calling model (and likely used there
to select the data from the ODS table)
Example:
For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is called with
wf_name = "w_ODS_CSDB_RATINGS_DEVO"
task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', 1, INSTR(
'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT CASE WHEN (SELECT COUNT(*)
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY IN (WH.A_WORKFLOW_HISTORY_KEY) AND LOAD_HISTORY.TASK_NAME = {{ task_name }}) > 0
THEN NULL
ELSE WH.A_WORKFLOW_HISTORY_KEY END AS A_WORKFLOW_HISTORY_KEY
FROM (SELECT MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) AS A_WORKFLOW_HISTORY_KEY
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} )
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %} {% endif %}) WH)
{% endmacro %}

65
dbt/macros/data_transform/get_ODS_wf_start_max_key.sql Normal file
View File

@@ -0,0 +1,65 @@
{% macro get_ODS_wf_start_max_key(wf_name, tasks_names) %}
{#
The query in this macro determines if data in a target table is up to date or if there is newer data in ODS
Parameters:
wf_name : The name of the ODS workflow that populates the ODS source table
task_name : The name of the task that executes the target model or an expression that returns it
Logic:
1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY
for the ODS workflow passed in as wf_name, assuming that this is the latest data available to be loaded
into the target.
2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded
into the target table. If it has already been loaded, the query returns a NULL value, indicating that there
is no data to be loaded. If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is
returned to the calling model (and likely used there to select the data from the ODS table)
Example:
For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is
called with wf_name = "w_ODS_CSDB_RATINGS_DEVO"
task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ',
1, INSTR( 'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)"
This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed)
#}
{# Set the lookup table based on the input service name #}
{% if var("input_service_name") == 'MOPDB' %}
{% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %}
{% elif var("input_service_name") == 'RAR' %}
{% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %}
{% endif %}
{# Set the service name filter used in the WORKFLOW_HISTORY subquery.
Here we change the filter based on whether the input service name is MOPDB or RAR #}
{% if var("input_service_name") == 'MOPDB' %}
{% set service_name_filter = 'MOPDB' %}
{% else %}
{% set service_name_filter = 'RAR' %}
{% endif %}
(SELECT
MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) OVER (PARTITION BY TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) ) as MAX_ODS_WORKFLOW_KEY_DAY,
A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY,
TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) as ODS_WORKFLOW_START
FROM
{{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE
A_WORKFLOW_HISTORY.WORKFLOW_NAME = ('{{ wf_name }}')
AND A_WORKFLOW_HISTORY.WORKFLOW_END < (
SELECT WORKFLOW_START
FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY
WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}'
AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }}
)
AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y'
AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS'
{% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %}
AND NOT EXISTS
(SELECT 1
FROM {{ load_history_tab }} LOAD_HISTORY
WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY
AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} )
)
)
{% endmacro %}

33
dbt/macros/data_transform/sequence_next_value.sql Normal file
View File

@@ -0,0 +1,33 @@
{% macro sequence_next_value(sequence_name, check_disable=false) %}
{# macro checks if a function for this sequence exists,
if it does, the function is used
if not, the standard command is used
check can be disabled by passing true in the 2nd paramater
#}
{% if execute %}
{% if not check_disable %}
{% set get_func %}
select count(*)
from all_objects
where object_type = 'FUNCTION'
and status = 'VALID'
and owner || '.' || object_name = upper('{{ sequence_name }}_fnc')
{% endset %}
{% set result = run_query(get_func) %}
{% set fnc_count = result.columns[0].values()[0] %}
{% else %}
{% set fnc_count = 0 %}
{% endif %}
{% if fnc_count > 0 %}
{% set nextvalue = sequence_name ~ '_fnc()' %}
{% else %}
{% set nextvalue = sequence_name ~ '.nextval'%}
{% endif %}
{{ return(nextvalue) }}
{% endif %}
{% endmacro %}