init
This commit is contained in:
Grzegorz Michalski
10
dbt/macros/dbt_overrides/conditional_truncate.sql
Normal file
10
dbt/macros/dbt_overrides/conditional_truncate.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
{% macro conditional_truncate(target_relation, source_relation) %}
|
||||
DECLARE
|
||||
row_count NUMBER;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO row_count FROM {{ source_relation }};
|
||||
IF row_count > 0 THEN
|
||||
EXECUTE IMMEDIATE 'TRUNCATE TABLE ' || '{{ target_relation }}';
|
||||
END IF;
|
||||
END;
|
||||
{% endmacro %}
|
||||
14
dbt/macros/dbt_overrides/generate_schema_name.sql
Normal file
14
dbt/macros/dbt_overrides/generate_schema_name.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
{% macro generate_schema_name(custom_schema_name, node) -%}
|
||||
|
||||
{%- set default_schema = target.schema -%}
|
||||
{%- if custom_schema_name is none -%}
|
||||
|
||||
{{ default_schema }}
|
||||
|
||||
{%- else -%}
|
||||
|
||||
{{ custom_schema_name | trim }}
|
||||
|
||||
{%- endif -%}
|
||||
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,9 @@
|
||||
{% macro oracle__snapshot_hash_arguments(args) -%}
|
||||
{# Oracle-specific implementation with NULL-safe SCD_ID generation #}
|
||||
LOWER(RAWTOHEX(STANDARD_HASH(
|
||||
{%- for arg in args -%}
|
||||
coalesce(cast({{ arg }} as varchar(4000)), '__DBT_NULL__')
|
||||
{%- if not loop.last %} || '|' || {%- endif -%}
|
||||
{%- endfor -%}
|
||||
, 'SHA256')))
|
||||
{%- endmacro %}
|
||||
214
dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql
Normal file
214
dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql
Normal file
@@ -0,0 +1,214 @@
|
||||
{% macro oracle__snapshot_staging_table(strategy, source_sql, target_relation) -%}
|
||||
{#--
|
||||
Custom Oracle snapshot implementation that ensures unified timestamps across all snapshot operations.
|
||||
|
||||
SOLUTION: Using strategy.updated_at throughout
|
||||
- strategy.updated_at generates a single TO_TIMESTAMP literal at Jinja compile time
|
||||
- This literal is reused everywhere, replacing multiple snapshot_get_time() calls
|
||||
- Ensures identical timestamps for all records created in the same snapshot run
|
||||
- Eliminates microsecond differences that occur when unique_key changes (DELETE+INSERT pattern)
|
||||
|
||||
Changes from original dbt-oracle v1.9.4 snapshot.sql:
|
||||
- Line 12: new_scd_id uses strategy.updated_at instead of snapshot_get_time()
|
||||
- Lines 127-129: deletes CTE uses strategy.updated_at instead of snapshot_get_time() (3 places)
|
||||
- Lines 174-175: deletion_records CTE uses strategy.updated_at instead of snapshot_get_time() (2 places)
|
||||
--#}
|
||||
|
||||
{% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
|
||||
{% if strategy.hard_deletes == 'new_record' %}
|
||||
{% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, strategy.updated_at]) %}
|
||||
{% endif %}
|
||||
|
||||
with snapshot_query as (
|
||||
|
||||
{{ source_sql }}
|
||||
|
||||
),
|
||||
|
||||
snapshotted_data as (
|
||||
|
||||
select {{ target_relation }}.*,
|
||||
{{ unique_key_fields(strategy.unique_key) }}
|
||||
from {{ target_relation }}
|
||||
where
|
||||
{% if config.get('dbt_valid_to_current') %}
|
||||
{% set source_unique_key = columns.dbt_valid_to | trim %}
|
||||
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
|
||||
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
|
||||
{% else %}
|
||||
{{ columns.dbt_valid_to }} is null
|
||||
{% endif %}
|
||||
),
|
||||
|
||||
insertions_source_data as (
|
||||
|
||||
select
|
||||
snapshot_query.*,
|
||||
{{ unique_key_fields(strategy.unique_key) }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
||||
{{ oracle__get_dbt_valid_to_current(strategy, columns) }},
|
||||
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }}
|
||||
|
||||
from snapshot_query
|
||||
),
|
||||
|
||||
updates_source_data as (
|
||||
|
||||
select
|
||||
snapshot_query.*,
|
||||
{{ unique_key_fields(strategy.unique_key) }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_to }}
|
||||
|
||||
from snapshot_query
|
||||
),
|
||||
|
||||
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
|
||||
|
||||
deletes_source_data as (
|
||||
|
||||
select
|
||||
snapshot_query.*,
|
||||
{{ unique_key_fields(strategy.unique_key) }}
|
||||
from snapshot_query
|
||||
),
|
||||
{% endif %}
|
||||
|
||||
insertions as (
|
||||
|
||||
select
|
||||
'insert' as dbt_change_type,
|
||||
source_data.*
|
||||
{%- if strategy.hard_deletes == 'new_record' -%}
|
||||
,'False' as {{ columns.dbt_is_deleted }}
|
||||
{%- endif %}
|
||||
|
||||
from insertions_source_data source_data
|
||||
left outer join snapshotted_data
|
||||
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
||||
where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }}
|
||||
or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }})
|
||||
{%- if strategy.hard_deletes == 'new_record' -%}
|
||||
or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and snapshotted_data.{{ columns.dbt_is_deleted }} = 'True')
|
||||
{%- endif %}
|
||||
)
|
||||
|
||||
),
|
||||
|
||||
updates as (
|
||||
|
||||
select
|
||||
'update' as dbt_change_type,
|
||||
source_data.*,
|
||||
snapshotted_data.{{ columns.dbt_scd_id }}
|
||||
{%- if strategy.hard_deletes == 'new_record' -%}
|
||||
, snapshotted_data.{{ columns.dbt_is_deleted }}
|
||||
{%- endif %}
|
||||
|
||||
from updates_source_data source_data
|
||||
join snapshotted_data
|
||||
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
||||
where (
|
||||
{{ strategy.row_changed }}
|
||||
)
|
||||
{%- if strategy.hard_deletes == 'new_record' -%}
|
||||
or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
|
||||
{%- endif %}
|
||||
)
|
||||
|
||||
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' -%}
|
||||
,
|
||||
|
||||
deletes as (
|
||||
|
||||
select
|
||||
'delete' as dbt_change_type,
|
||||
source_data.*,
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_to }},
|
||||
snapshotted_data.{{ columns.dbt_scd_id }}
|
||||
{%- if strategy.hard_deletes == 'new_record' -%}
|
||||
, snapshotted_data.{{ columns.dbt_is_deleted }}
|
||||
{%- endif %}
|
||||
|
||||
from snapshotted_data
|
||||
left join deletes_source_data source_data
|
||||
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
||||
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
|
||||
{%- if strategy.hard_deletes == 'new_record' %}
|
||||
and not (
|
||||
--avoid updating the record's valid_to if the latest entry is marked as deleted
|
||||
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
|
||||
and
|
||||
{% if config.get('dbt_valid_to_current') %}
|
||||
{% set source_unique_key = columns.dbt_valid_to | trim %}
|
||||
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
|
||||
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
|
||||
{% else %}
|
||||
{{ columns.dbt_valid_to }} is null
|
||||
{% endif %}
|
||||
)
|
||||
{%- endif %}
|
||||
|
||||
)
|
||||
{%- endif %}
|
||||
|
||||
{%- if strategy.hard_deletes == 'new_record' %}
|
||||
{% set source_sql_cols = get_column_schema_from_query(source_sql) %}
|
||||
,
|
||||
deletion_records as (
|
||||
|
||||
select
|
||||
'insert' as dbt_change_type,
|
||||
{%- for col in source_sql_cols -%}
|
||||
snapshotted_data.{{ adapter.quote(col.column) }},
|
||||
{% endfor -%}
|
||||
{%- if strategy.unique_key | is_list -%}
|
||||
{%- for key in strategy.unique_key -%}
|
||||
snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},
|
||||
{% endfor -%}
|
||||
{%- else -%}
|
||||
snapshotted_data.dbt_unique_key as dbt_unique_key,
|
||||
{% endif -%}
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
||||
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
||||
snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},
|
||||
{{ new_scd_id }} as {{ columns.dbt_scd_id }},
|
||||
'True' as {{ columns.dbt_is_deleted }}
|
||||
from snapshotted_data
|
||||
left join deletes_source_data source_data
|
||||
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
||||
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
|
||||
and not (
|
||||
--avoid inserting a new record if the latest one is marked as deleted
|
||||
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
|
||||
and
|
||||
{% if config.get('dbt_valid_to_current') %}
|
||||
{% set source_unique_key = columns.dbt_valid_to | trim %}
|
||||
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
|
||||
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
|
||||
{% else %}
|
||||
{{ columns.dbt_valid_to }} is null
|
||||
{% endif %}
|
||||
)
|
||||
)
|
||||
{%- endif %}
|
||||
|
||||
select * from insertions
|
||||
union all
|
||||
select * from updates
|
||||
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
|
||||
union all
|
||||
select * from deletes
|
||||
{%- endif %}
|
||||
{%- if strategy.hard_deletes == 'new_record' %}
|
||||
union all
|
||||
select * from deletion_records
|
||||
{%- endif %}
|
||||
|
||||
{%- endmacro %}
|
||||
|
||||
|
||||
14
dbt/macros/dbt_overrides/unique_key_join_on.sql
Normal file
14
dbt/macros/dbt_overrides/unique_key_join_on.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}
|
||||
{% if unique_key | is_list %}
|
||||
{% for key in unique_key %}
|
||||
{% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
|
||||
{% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
|
||||
{# Use Oracle-specific NULL-safe equality matching oracle__equals macro #}
|
||||
({{ source_unique_key }} = {{ target_unique_key }} OR ({{ source_unique_key }} IS NULL AND {{ target_unique_key }} IS NULL))
|
||||
{%- if not loop.last %} and {%- endif %}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{# Use Oracle-specific NULL-safe equality for single unique_key as well #}
|
||||
({{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key OR ({{ identifier }}.dbt_unique_key IS NULL AND {{ from_identifier }}.dbt_unique_key IS NULL))
|
||||
{% endif %}
|
||||
{% endmacro %}
|
||||
Reference in New Issue
Block a user