admin/mars-elt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-02 09:47:35 +01:00
commit 2c225d68ac
715 changed files with 130067 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml Normal file
View File

@@ -0,0 +1,6 @@
# Flow Configuration
CONNECTOR_TYPE: "casper_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS/RQSD_ANNEX_"
TASK_NAME: "m_ODS_RQSD_CASPER"
OUTPUT_TABLE: "RQSD_ANNEX_"
COLLECTION_ID: "1537"

19
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml Normal file
View File

@@ -0,0 +1,19 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_FX"
OUTPUT_TABLE: "RQSD_FX"
DEVO_QUERY: "select * from
(select tec_ingestion_date,
obs_value,
pow(10, cast(unit_mult as int)) as divisor,
series_key,
time_period,
ROW_NUMBER() OVER (PARTITION BY series_key, time_period ORDER BY tec_ingestion_date DESC) AS rn
from crp_other_pub.fx
where freq = 'Q'
and data_type_fm = 'HSTE'
and provider_fm = 'BL'
and instrument_fm = 'FX'
and currency != 'EUR') t
where t.rn = 1"

34
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml Normal file
View File

@@ -0,0 +1,34 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_FX_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_FX
output_table: RQSD_FX
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'obs_value'
column_header: 'obs_value'
- type: 'csv_header'
value: 'divisor'
column_header: 'divisor'
- type: 'csv_header'
value: 'series_key'
column_header: 'series_key'
- type: 'csv_header'
value: 'time_period'
column_header: 'time_period'
- type: 'csv_header'
value: 'rn'
column_header: 'rn'

31
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml Normal file
View File

@@ -0,0 +1,31 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_OBSERVATIONS"
OUTPUT_TABLE: "RQSD_OBSERVATIONS"
DEVO_QUERY: "with latest_dates as (
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_fin_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_fin_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_2_all_mrds),
latest_update as (
SELECT max(max_date) as max_date FROM latest_dates),
latest as (
select case when count(max_date) = 0 then '1999-01-01 10:28:22' else max(max_date) end max_date
from latest_update)
SELECT a.*,
CAST(NULL AS STRING) AS tec_source_system,
CAST(NULL AS STRING) AS tec_dataset,
CAST(NULL AS STRING) AS tec_surrogate_key,
CAST(NULL AS STRING) AS tec_crc,
CAST(NULL AS STRING) AS tec_ingestion_date,
CAST(NULL AS STRING) AS tec_version_id,
CAST(NULL AS STRING) AS tec_execution_date,
CAST(NULL AS STRING) AS tec_run_id,
CAST(NULL AS STRING) AS tec_business_date
FROM crp_rqsd.tcrqsd_observations a where receivedfilereceiveddate > (select max(max_date) from latest)"

181
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml Normal file
View File

@@ -0,0 +1,181 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_OBSERVATIONS_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_OBSERVATIONS
output_table: RQSD_OBSERVATIONS
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'datacollectioncode'
column_header: 'datacollectioncode'
- type: 'csv_header'
value: 'datacollectionname'
column_header: 'datacollectionname'
- type: 'csv_header'
value: 'datacollectionowner'
column_header: 'datacollectionowner'
- type: 'csv_header'
value: 'reportingcyclename'
column_header: 'reportingcyclename'
- type: 'csv_header'
value: 'reportingcyclestatus'
column_header: 'reportingcyclestatus'
- type: 'csv_header'
value: 'modulecode'
column_header: 'modulecode'
- type: 'csv_header'
value: 'modulename'
column_header: 'modulename'
- type: 'csv_header'
value: 'moduleversionnumber'
column_header: 'moduleversionnumber'
- type: 'csv_header'
value: 'reportingentitycollectionuniqueid'
column_header: 'reportingentitycollectionuniqueid'
- type: 'csv_header'
value: 'entityattributereportingcode'
column_header: 'entityattributereportingcode'
- type: 'csv_header'
value: 'reportingentityname'
column_header: 'reportingentityname'
- type: 'csv_header'
value: 'reportingentityentitytype'
column_header: 'reportingentityentitytype'
- type: 'csv_header'
value: 'entityattributecountry'
column_header: 'entityattributecountry'
- type: 'csv_header'
value: 'entitygroupentityname'
column_header: 'entitygroupentityname'
- type: 'csv_header'
value: 'obligationmodulereferencedate'
column_header: 'obligationmodulereferencedate'
- type: 'csv_header'
value: 'obligationmoduleremittancedate'
column_header: 'obligationmoduleremittancedate'
- type: 'csv_header'
value: 'receivedfilereceiveddate'
column_header: 'receivedfilereceiveddate'
- type: 'csv_header'
value: 'obligationmoduleexpected'
column_header: 'obligationmoduleexpected'
- type: 'csv_header'
value: 'receivedfileversionnumber'
column_header: 'receivedfileversionnumber'
- type: 'csv_header'
value: 'revalidationversionnumber'
column_header: 'revalidationversionnumber'
- type: 'csv_header'
value: 'revalidationdate'
column_header: 'revalidationdate'
- type: 'csv_header'
value: 'receivedfilesystemfilename'
column_header: 'receivedfilesystemfilename'
- type: 'csv_header'
value: 'obligationstatusstatus'
column_header: 'obligationstatusstatus'
- type: 'csv_header'
value: 'filestatussetsubmissionstatus'
column_header: 'filestatussetsubmissionstatus'
- type: 'csv_header'
value: 'filestatussetvalidationstatus'
column_header: 'filestatussetvalidationstatus'
- type: 'csv_header'
value: 'filestatussetexternalvalidationstatus'
column_header: 'filestatussetexternalvalidationstatus'
- type: 'csv_header'
value: 'numberoferrors'
column_header: 'numberoferrors'
- type: 'csv_header'
value: 'numberofwarnings'
column_header: 'numberofwarnings'
- type: 'csv_header'
value: 'delayindays'
column_header: 'delayindays'
- type: 'csv_header'
value: 'failedattempts'
column_header: 'failedattempts'
- type: 'csv_header'
value: 'observationvalue'
column_header: 'observationvalue'
- type: 'csv_header'
value: 'observationtextvalue'
column_header: 'observationtextvalue'
- type: 'csv_header'
value: 'observationdatevalue'
column_header: 'observationdatevalue'
- type: 'csv_header'
value: 'datapointsetdatapointidentifier'
column_header: 'datapointsetdatapointidentifier'
- type: 'csv_header'
value: 'datapointsetlabel'
column_header: 'datapointsetlabel'
- type: 'csv_header'
value: 'obsrvdescdatatype'
column_header: 'obsrvdescdatatype'
- type: 'csv_header'
value: 'ordinatecode'
column_header: 'ordinatecode'
- type: 'csv_header'
value: 'ordinateposition'
column_header: 'ordinateposition'
- type: 'csv_header'
value: 'tablename'
column_header: 'tablename'
- type: 'csv_header'
value: 'isstock'
column_header: 'isstock'
- type: 'csv_header'
value: 'scale'
column_header: 'scale'
- type: 'csv_header'
value: 'currency'
column_header: 'currency'
- type: 'csv_header'
value: 'numbertype'
column_header: 'numbertype'
- type: 'csv_header'
value: 'ismandatory'
column_header: 'ismandatory'
- type: 'csv_header'
value: 'decimalplaces'
column_header: 'decimalplaces'
- type: 'csv_header'
value: 'serieskey'
column_header: 'serieskey'
- type: 'csv_header'
value: 'tec_source_system'
column_header: 'tec_source_system'
- type: 'csv_header'
value: 'tec_dataset'
column_header: 'tec_dataset'
- type: 'csv_header'
value: 'tec_surrogate_key'
column_header: 'tec_surrogate_key'
- type: 'csv_header'
value: 'tec_crc'
column_header: 'tec_crc'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'tec_version_id'
column_header: 'tec_version_id'
- type: 'csv_header'
value: 'tec_execution_date'
column_header: 'tec_execution_date'
- type: 'csv_header'
value: 'tec_run_id'
column_header: 'tec_run_id'
- type: 'csv_header'
value: 'tec_business_date'
column_header: 'tec_business_date'

736
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml Normal file
View File

@@ -0,0 +1,736 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_SUBA_DEVO"
OUTPUT_TABLE: "RQSD_SUBA_DEVO"
DEVO_QUERY: "with suba_reporting_requirements_temp as (
SELECT sender,
riad_code,
name,
tid,
entity_id,
module_name,
CASE
WHEN cons_level = 'IND' THEN 'SOLO'
WHEN cons_level = 'CON' THEN 'CONS'
ELSE cons_level
END AS cons_level,
accounting_standard,
template_id,
reference_date,
expected,
module_id,
dsd_id,
changed_date,
tec_source_system,
tec_dataset,
tec_surrogate_key,
tec_crc,
tec_ingestion_date,
tec_version_id,
tec_execution_date,
tec_run_id,
tec_business_date
FROM crp_suba06.suba_reporting_requirements),
observations_1_2 as
(
select OBLIGATIONMODULEREFERENCEDATE,
RECEIVEDFILERECEIVEDDATE,
SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS ETWDR_COMP_KEY,
REPORTINGENTITYCOLLECTIONUNIQUEID,
DATAPOINTSETDATAPOINTIDENTIFIER,
NVL(NVL(CAST(OBSERVATIONVALUE AS STRING), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBS_VALUE
FROM crp_rqsd.rqsd_rqsd01_observations
WHERE MODULECODE = 'SCOPF'
AND TABLENAME = 'etwdr'
AND obligationmodulereferencedate not like '%2019-12-06%'
AND obligationmodulereferencedate not like '%2017%'),
max_version_1_2_fin_all as
(
SELECT max(RECEIVEDFILERECEIVEDDATE) as max_version, reportingentitycollectionuniqueid, obligationmodulereferencedate
FROM observations_1_2
group by reportingentitycollectionuniqueid, obligationmodulereferencedate),
annex_1_2_fin_all_pivoted as
(SELECT
OBLIGATIONMODULEREFERENCEDATE,
ETWDR_COMP_KEY,
RECEIVEDFILERECEIVEDDATE,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'ann_1_2_ref_date' THEN OBS_VALUE END) AS ANN_1_2_REF_DATE,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_mfi_id' THEN OBS_VALUE END) AS MFI_ID,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_lei' THEN OBS_VALUE END) AS LEGAL_ENTITY_ID,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_name' THEN OBS_VALUE END) AS INST_NAME,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_submitter' THEN OBS_VALUE END) AS ETWDR_SUBMITTER
FROM
observations_1_2
WHERE RECEIVEDFILERECEIVEDDATE in (select max_version from max_version_1_2_fin_all)
GROUP BY
OBLIGATIONMODULEREFERENCEDATE, ETWDR_COMP_KEY, RECEIVEDFILERECEIVEDDATE),
scopf as
(
select mfi_id, inst_name, legal_entity_id, obligationmodulereferencedate, ann_1_2_ref_date from annex_1_2_fin_all_pivoted where etwdr_submitter = 'SUP'
),
EXP_COREP_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_COREP_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_03.00'
and MODULE_NAME = 'COREP_OF'
and CONS_LEVEL = 'CONS'
),
EXP_COREP_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_COREP_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_03.00'
and MODULE_NAME = 'COREP_OF'
and CONS_LEVEL = 'SOLO'
),
EXP_FINREP_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_FINREP_CONS
from suba_reporting_requirements_temp
where template_id = 'tgF_00.01'
and MODULE_NAME in ('FINREP9', 'FINREP9_DP')
and CONS_LEVEL = 'CONS'
),
EXP_FINREP_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_FINREP_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgF_00.01'
and MODULE_NAME in ('FINREP9', 'FINREP9_DP')
and CONS_LEVEL = 'SOLO'
),
EXP_LEV_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'full&trans'
when EXPECTED = 'O' then 'none'
else 'none'
end as EXP_LEV_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_47.00'
and MODULE_NAME in ('COREP_LR')
and CONS_LEVEL = 'CONS'
),
EXP_LEV_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'full&trans'
when EXPECTED = 'O' then 'none'
else 'none'
end as EXP_LEV_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_47.00'
and MODULE_NAME in ('COREP_LR')
and CONS_LEVEL = 'SOLO'
),
EXP_LCR_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_LCR_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_76.00'
and MODULE_NAME = 'COREP_LCR_DA'
and CONS_LEVEL = 'CONS'
),
EXP_LCR_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_LCR_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_76.00'
and MODULE_NAME = 'COREP_LCR_DA'
and CONS_LEVEL = 'SOLO'
),
EXP_NSFR_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_NSFR_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_84.00'
and MODULE_NAME = 'COREP_NSFR'
and CONS_LEVEL = 'CONS'
),
EXP_NSFR_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_NSFR_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_84.00'
and MODULE_NAME = 'COREP_NSFR'
and CONS_LEVEL = 'SOLO'
),
corep_cons_c100 as (
select riad_code,
reported_period,
cons_level,
name,
lei,
c_0100_r0010_c0010,
c_0100_r0020_c0010,
c_0100_r0015_c0010
from crp_suba06.suba_c_0100
where cons_level = 'CONS'
),
corep_solo_c100 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0100_r0010_c0010,
c_0100_r0020_c0010,
c_0100_r0015_c0010
from crp_suba06.suba_c_0100
where cons_level = 'SOLO'
),
corep_cons_c300 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0300_r0010_c0010,
c_0300_r0030_c0010,
c_0300_r0050_c0010
from crp_suba06.suba_c_0300
where cons_level = 'CONS'
),
corep_solo_c300 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0300_r0010_c0010,
c_0300_r0030_c0010,
c_0300_r0050_c0010
from crp_suba06.suba_c_0300
where cons_level = 'SOLO'
),
corep_cons_c200 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0200_r0010_c0010
from crp_suba06.suba_c_0200
where cons_level = 'CONS' and reported_period<'2025-03-31 00:00:00'
union all
select riad_code,
reported_period,
cons_level,
lei,
c_0200a_r0010_c0010
from crp_suba06.suba_c_0200a as c_0200_r0010_c0010
where cons_level = 'CONS' and reported_period>='2025-03-31 00:00:00'
),
corep_solo_c200 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0200_r0010_c0010
from crp_suba06.suba_c_0200
where cons_level = 'SOLO' and reported_period<'2025-03-31 00:00:00'
union all
select riad_code,
reported_period,
cons_level,
lei,
c_0200a_r0010_c0010
from crp_suba06.suba_c_0200a as c_0200_r0010_c0010
where cons_level = 'SOLO' and reported_period>='2025-03-31 00:00:00'
),
finrep_cons as (
select riad_code,
reported_period,
cons_level,
lei,
f_0101_r0380_c0010
from crp_suba06.suba_f_0101
where cons_level = 'CONS'
),
finrep_solo as (
select riad_code,
reported_period,
cons_level,
lei,
f_0101_r0380_c0010
from crp_suba06.suba_f_0101
where cons_level = 'SOLO'
),
lev_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_4700_r0330_c0010,
c_4700_r0340_c0010,
c_4700_r0410_c0010
from crp_suba06.suba_c_4700
where cons_level = 'CONS'
),
lev_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_4700_r0330_c0010,
c_4700_r0340_c0010,
c_4700_r0410_c0010
from crp_suba06.suba_c_4700
where cons_level = 'SOLO'
),
lcr_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_7600a_r0030_c0010
from crp_suba06.suba_c_7600a
where cons_level = 'CONS'
),
lcr_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_7600a_r0030_c0010
from crp_suba06.suba_c_7600a
where cons_level = 'SOLO'
),
nsfr_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_8400a_r0220_c0040
from crp_suba06.suba_c_8400a
where cons_level = 'CONS'
),
nsfr_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_8400a_r0220_c0040
from crp_suba06.suba_c_8400a
where cons_level = 'SOLO'
),
liq_subgroups as (
SELECT entity_id,
substr(entity_id, 0, 20) as short_id,
name,
start_date,
end_date
from crp_suba06.suba_entity_master_data_stage2
where substr(entity_id, -12) = 'CRDLIQSUBGRP'
),
liq_subgroup_lcr as (
select liq_subgroups.entity_id,
short_id,
liq_subgroups.name,
start_date,
end_date,
suba_c_7600a.reported_period,
c_7600a_r0030_c0010
from liq_subgroups
left join crp_suba06.suba_c_7600a on liq_subgroups.entity_id = suba_c_7600a.entity_id
where start_date <= reported_period
and end_date >= reported_period
),
liq_subgroup_nsfr as (
select liq_subgroups.entity_id,
short_id,
liq_subgroups.name,
start_date,
end_date,
suba_c_8400a.reported_period,
c_8400a_r0220_c0040
from liq_subgroups
left join crp_suba06.suba_c_8400a on liq_subgroups.entity_id = suba_c_8400a.entity_id
where start_date <= reported_period
and end_date >= reported_period
),
liq_subgroup_data_riad as (
select liq_subgroup_lcr.entity_id,
liq_subgroup_lcr.short_id,
liq_subgroup_lcr.name,
liq_subgroup_lcr.start_date,
liq_subgroup_lcr.end_date,
liq_subgroup_lcr.reported_period,
c_7600a_r0030_c0010 as c_7600a_r0030_c0010_subgr,
c_8400a_r0220_c0040 as c_8400a_r0220_c0040_subgr,
riad_code
from liq_subgroup_lcr
left join crp_suba06.suba_entity_master_data_stage2 on liq_subgroup_lcr.short_id = suba_entity_master_data_stage2.entity_id
and suba_entity_master_data_stage2.start_date <= liq_subgroup_lcr.reported_period
and suba_entity_master_data_stage2.end_date >= liq_subgroup_lcr.reported_period
left join liq_subgroup_nsfr on liq_subgroup_lcr.entity_id = liq_subgroup_nsfr.entity_id
and liq_subgroup_lcr.reported_period = liq_subgroup_nsfr.reported_period
),
all_dates as (
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0100
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0300
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0200
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0200a
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_f_0101
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_4700
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_7600a
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_8400a
union
select riad_code,
reference_date,
cast(changed_date as STRING) as reception_date
from suba_reporting_requirements_temp
where template_id in (
'tgC_03.00',
'tgF_00.01',
'tgC_47.00',
'tgC_76.00',
'tgC_84.00'
)
and reference_date in (
select distinct obligationmodulereferencedate
from SCOPF
)
),
max_dates as (
select reported_period,
riad_code,
max(reception_date) as receivedfilereceiveddate
from all_dates
group by reported_period,
riad_code
)
select SCOPF.obligationmodulereferencedate,
cast(null as String) as reportingentitycollectionuniqueid,
cast(NULL as DECIMAL(38, 10)) as receivedfileversionnumber,
max_dates.receivedfilereceiveddate,
cast(NULL as STRING) as revalidationdate,
SCOPF.ann_1_2_ref_date as ref_date,
concat(SCOPF.mfi_id, SCOPF.legal_entity_id) as inst_comp_key,
SCOPF.mfi_id,
SCOPF.legal_entity_id,
SCOPF.inst_name,
'EUR' as currency,
cast(NULL as STRING) as reported_by_supervisor,
cast(NULL as STRING) as confirmed_by_supervisor,
coalesce(EXP_COREP_CONS.EXP_COREP_CONS, 'N') as EXP_COREP_CONS,
coalesce (EXP_COREP_SOLO.EXP_COREP_SOLO, 'N') as EXP_COREP_SOLO,
coalesce (EXP_FINREP_CONS.EXP_FINREP_CONS, 'N') as EXP_FINREP_CONS,
coalesce (EXP_FINREP_SOLO.EXP_FINREP_SOLO, 'N') as EXP_FINREP_SOLO,
coalesce(EXP_LEV_CONS.EXP_LEV_CONS, 'none') as EXP_LEV_CONS,
coalesce (EXP_LEV_SOLO.EXP_LEV_SOLO, 'none') as EXP_LEV_SOLO,
coalesce (EXP_LCR_CONS.EXP_LCR_CONS, 'N') as EXP_LCR_CONS,
coalesce (EXP_LCR_SOLO.EXP_LCR_SOLO, 'N') as EXP_LCR_SOLO,
coalesce (EXP_NSFR_CONS.EXP_NSFR_CONS, 'N') as EXP_NSFR_CONS,
coalesce (EXP_NSFR_SOLO.EXP_NSFR_SOLO, 'N') as EXP_NSFR_SOLO,
cast(
corep_cons_c100.c_0100_r0020_c0010 as DECIMAL(38, 10)
) as cons_cet1_amt,
cast(
corep_cons_c100.c_0100_r0015_c0010 as DECIMAL(38, 10)
) as cons_tier1_amt,
cast(
corep_cons_c100.c_0100_r0010_c0010 as DECIMAL(38, 10)
) as cons_tot_cap_amt,
cast(
corep_cons_c300.c_0300_r0010_c0010 as DECIMAL(38, 10)
) as cons_cet1_ratio,
cast(
corep_cons_c300.c_0300_r0030_c0010 as DECIMAL(38, 10)
) as cons_tier1_ratio,
cast(
corep_cons_c300.c_0300_r0050_c0010 as DECIMAL(38, 10)
) as cons_tot_cap_ratio,
cast(
corep_cons_c200.c_0200_r0010_c0010 as DECIMAL(38, 10)
) as cons_risk_wght_assets,
cast(
corep_solo_c100.c_0100_r0020_c0010 as DECIMAL(38, 10)
) as solo_cet1_amt,
cast(
corep_solo_c100.c_0100_r0015_c0010 as DECIMAL(38, 10)
) as solo_tier1_amt,
cast(
corep_solo_c100.c_0100_r0010_c0010 as DECIMAL(38, 10)
) as solo_tot_cap_amt,
cast(
corep_solo_c300.c_0300_r0010_c0010 as DECIMAL(38, 10)
) as solo_cet1_ratio,
cast(
corep_solo_c300.c_0300_r0030_c0010 as DECIMAL(38, 10)
) as solo_tier1_ratio,
cast(
corep_solo_c300.c_0300_r0050_c0010 as DECIMAL(38, 10)
) as solo_tot_cap_ratio,
cast(
corep_solo_c200.c_0200_r0010_c0010 as DECIMAL(38, 10)
) as solo_risk_wght_assets,
cast(finrep_cons.f_0101_r0380_c0010 as DECIMAL(38, 10)) as cons_tot_assets,
cast(finrep_solo.f_0101_r0380_c0010 as DECIMAL(38, 10)) as solo_tot_assets,
cast(lev_cons.c_4700_r0330_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_full,
cast(lev_cons.c_4700_r0340_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_trans,
cast(lev_cons.c_4700_r0410_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_req,
CASE
WHEN lev_cons.c_4700_r0410_c0010 IS NULL THEN NULL
WHEN lev_cons.c_4700_r0410_c0010 = 0.0300000000 THEN 'N'
ELSE 'Y'
END AS cons_lev_ratio_adj,
cast(lev_solo.c_4700_r0330_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_full,
cast(lev_solo.c_4700_r0340_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_trans,
cast(lev_solo.c_4700_r0410_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_req,
CASE
WHEN lev_solo.c_4700_r0410_c0010 IS NULL THEN NULL
WHEN lev_solo.c_4700_r0410_c0010 = 0.0300000000 THEN 'N'
ELSE 'Y'
END AS solo_lev_ratio_adj,
cast(
case
WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_7600a_r0030_c0010_subgr
ELSE lcr_cons.c_7600a_r0030_c0010
END as DECIMAL(38, 10)
) as cons_lc_ratio,
cast(lcr_solo.c_7600a_r0030_c0010 as DECIMAL(38, 10)) as solo_lc_ratio,
cast(
case
WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_8400a_r0220_c0040_subgr
ELSE nsfr_cons.c_8400a_r0220_c0040
END as DECIMAL(38, 10)
) as cons_nsfr_ratio,
cast(nsfr_solo.c_8400a_r0220_c0040 as DECIMAL(38, 10)) as solo_nsfr_ratio,
cast(NULL as STRING) as submitter_comment,
cast(NULL as STRING) as datacollectioncode,
cast(NULL as STRING) as reportingcyclename,
cast(NULL as STRING) as reportingcyclestatus,
cast(NULL as STRING) as modulecode,
cast(NULL as DECIMAL(38, 10)) as moduleversionnumber,
cast(NULL as STRING) as reportingentityname,
cast(NULL as STRING) as entityattributecountry,
cast(NULL as STRING) as entitygroupentityname,
cast(NULL as STRING) as obligationmoduleremittancedate,
cast(NULL as STRING) as obligationmoduleexpected,
cast(NULL as DEcimal(38, 10)) as revalidationversionnumber,
cast(NULL as STRING) as receivedfilesystemfilename,
cast(NULL as STRING) as obligationstatusstatus,
cast(NULL as STRING) as filestatussetsubmissionstatus,
cast(NULL as STRING) as filestatussetvalidationstatus,
cast(NULL as DECIMAL(38, 10)) as numberoferrors,
cast(NULL as DECIMAL(38, 10)) as numberofwarnings,
cast(NULL as Decimal(38, 10)) as delayindays,
cast(NULL as DECIMAL(38, 10)) as failedattempts,
cast(NULL as STRING) as tablename,
cast(NULL as STRING) as tec_source_system,
cast(NULL as STRING) as tec_dataset,
cast(NULL as STRING) as tec_surrogate_key,
cast(NULL as STRING) as tec_crc,
cast(NULL as TIMESTAMP) as tec_ingestion_date,
cast(NULL as STRING) as tec_version_id,
cast(NULL as TIMESTAMP) as tec_execution_date,
cast(NULL as STRING) as tec_run_id,
cast(NULL as TIMESTAMP) as tec_business_date
from SCOPF
left join EXP_COREP_CONS on (
scopf.obligationmodulereferencedate = EXP_COREP_CONS.reference_date
and scopf.mfi_id = EXP_COREP_CONS.riad_code
)
left join EXP_COREP_SOLO on (
scopf.obligationmodulereferencedate = EXP_COREP_SOLO.reference_date
and scopf.mfi_id = EXP_COREP_SOLO.riad_code
)
left join EXP_FINREP_CONS on (
scopf.obligationmodulereferencedate = EXP_FINREP_CONS.reference_date
and scopf.mfi_id = EXP_FINREP_CONS.riad_code
)
left join EXP_FINREP_SOLO on (
scopf.obligationmodulereferencedate = EXP_FINREP_SOLO.reference_date
and scopf.mfi_id = EXP_FINREP_SOLO.riad_code
)
left join EXP_LEV_CONS on (
scopf.obligationmodulereferencedate = EXP_LEV_CONS.reference_date
and scopf.mfi_id = EXP_LEV_CONS.riad_code
)
left join EXP_LEV_SOLO on (
scopf.obligationmodulereferencedate = EXP_LEV_SOLO.reference_date
and scopf.mfi_id = EXP_LEV_SOLO.riad_code
)
left join EXP_LCR_CONS on (
scopf.obligationmodulereferencedate = EXP_LCR_CONS.reference_date
and scopf.mfi_id = EXP_LCR_CONS.riad_code
)
left join EXP_LCR_SOLO on (
scopf.obligationmodulereferencedate = EXP_LCR_SOLO.reference_date
and scopf.mfi_id = EXP_LCR_SOLO.riad_code
)
left join EXP_NSFR_CONS on (
scopf.obligationmodulereferencedate = EXP_NSFR_CONS.reference_date
and scopf.mfi_id = EXP_NSFR_CONS.riad_code
)
left join EXP_NSFR_SOLO on (
scopf.obligationmodulereferencedate = EXP_NSFR_SOLO.reference_date
and scopf.mfi_id = EXP_NSFR_SOLO.riad_code
)
left join corep_cons_c100 on (
SCOPF.obligationmodulereferencedate = corep_cons_c100.reported_period
and SCOPF.mfi_id = corep_cons_c100.riad_code
)
left join corep_cons_c300 on (
SCOPF.obligationmodulereferencedate = corep_cons_c300.reported_period
and SCOPF.mfi_id = corep_cons_c300.riad_code
)
left join corep_cons_c200 on (
SCOPF.obligationmodulereferencedate = corep_cons_c200.reported_period
and SCOPF.mfi_id = corep_cons_c200.riad_code
)
left join corep_solo_c100 on (
SCOPF.obligationmodulereferencedate = corep_solo_c100.reported_period
and SCOPF.mfi_id = corep_solo_c100.riad_code
)
left join corep_solo_c200 on (
SCOPF.obligationmodulereferencedate = corep_solo_c200.reported_period
and SCOPF.mfi_id = corep_solo_c200.riad_code
)
left join corep_solo_c300 on (
SCOPF.obligationmodulereferencedate = corep_solo_c300.reported_period
and SCOPF.mfi_id = corep_solo_c300.riad_code
)
left join finrep_cons on (
SCOPF.obligationmodulereferencedate = finrep_cons.reported_period
and SCOPF.mfi_id = finrep_cons.riad_code
)
left join finrep_solo on (
SCOPF.obligationmodulereferencedate = finrep_solo.reported_period
and SCOPF.mfi_id = finrep_solo.riad_code
)
left join lev_cons on (
SCOPF.obligationmodulereferencedate = lev_cons.reported_period
and SCOPF.mfi_id = lev_cons.riad_code
)
left join lev_solo on (
SCOPF.obligationmodulereferencedate = lev_solo.reported_period
and SCOPF.mfi_id = lev_solo.riad_code
)
left join lcr_cons on (
SCOPF.obligationmodulereferencedate = lcr_cons.reported_period
and SCOPF.mfi_id = lcr_cons.riad_code
)
left join lcr_solo on (
SCOPF.obligationmodulereferencedate = lcr_solo.reported_period
and SCOPF.mfi_id = lcr_solo.riad_code
)
left join nsfr_cons on (
SCOPF.obligationmodulereferencedate = nsfr_cons.reported_period
and SCOPF.mfi_id = nsfr_cons.riad_code
)
left join nsfr_solo on (
SCOPF.obligationmodulereferencedate = nsfr_solo.reported_period
and SCOPF.mfi_id = nsfr_solo.riad_code
)
left join liq_subgroup_data_riad on (
SCOPF.obligationmodulereferencedate = liq_subgroup_data_riad.reported_period
and SCOPF.mfi_id = liq_subgroup_data_riad.riad_code
)
left join max_dates on (
SCOPF.obligationmodulereferencedate = max_dates.reported_period
and SCOPF.mfi_id = max_dates.riad_code
)
order by 1,2"

259
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml Normal file
View File

@@ -0,0 +1,259 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_SUBA_DEVO_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_SUBA_DEVO
output_table: RQSD_SUBA_DEVO
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'obligationmodulereferencedate'
column_header: 'obligationmodulereferencedate'
- type: 'csv_header'
value: 'reportingentitycollectionuniqueid'
column_header: 'reportingentitycollectionuniqueid'
- type: 'csv_header'
value: 'receivedfileversionnumber'
column_header: 'receivedfileversionnumber'
- type: 'csv_header'
value: 'receivedfilereceiveddate'
column_header: 'receivedfilereceiveddate'
- type: 'csv_header'
value: 'revalidationdate'
column_header: 'revalidationdate'
- type: 'csv_header'
value: 'ref_date'
column_header: 'ref_date'
- type: 'csv_header'
value: 'inst_comp_key'
column_header: 'inst_comp_key'
- type: 'csv_header'
value: 'mfi_id'
column_header: 'mfi_id'
- type: 'csv_header'
value: 'legal_entity_id'
column_header: 'legal_entity_id'
- type: 'csv_header'
value: 'inst_name'
column_header: 'inst_name'
- type: 'csv_header'
value: 'currency'
column_header: 'currency'
- type: 'csv_header'
value: 'reported_by_supervisor'
column_header: 'reported_by_supervisor'
- type: 'csv_header'
value: 'confirmed_by_supervisor'
column_header: 'confirmed_by_supervisor'
- type: 'csv_header'
value: 'exp_corep_cons'
column_header: 'exp_corep_cons'
- type: 'csv_header'
value: 'exp_corep_solo'
column_header: 'exp_corep_solo'
- type: 'csv_header'
value: 'exp_finrep_cons'
column_header: 'exp_finrep_cons'
- type: 'csv_header'
value: 'exp_finrep_solo'
column_header: 'exp_finrep_solo'
- type: 'csv_header'
value: 'exp_lev_cons'
column_header: 'exp_lev_cons'
- type: 'csv_header'
value: 'exp_lev_solo'
column_header: 'exp_lev_solo'
- type: 'csv_header'
value: 'exp_lcr_cons'
column_header: 'exp_lcr_cons'
- type: 'csv_header'
value: 'exp_lcr_solo'
column_header: 'exp_lcr_solo'
- type: 'csv_header'
value: 'exp_nsfr_cons'
column_header: 'exp_nsfr_cons'
- type: 'csv_header'
value: 'exp_nsfr_solo'
column_header: 'exp_nsfr_solo'
- type: 'csv_header'
value: 'cons_cet1_amt'
column_header: 'cons_cet1_amt'
- type: 'csv_header'
value: 'cons_tier1_amt'
column_header: 'cons_tier1_amt'
- type: 'csv_header'
value: 'cons_tot_cap_amt'
column_header: 'cons_tot_cap_amt'
- type: 'csv_header'
value: 'cons_cet1_ratio'
column_header: 'cons_cet1_ratio'
- type: 'csv_header'
value: 'cons_tier1_ratio'
column_header: 'cons_tier1_ratio'
- type: 'csv_header'
value: 'cons_tot_cap_ratio'
column_header: 'cons_tot_cap_ratio'
- type: 'csv_header'
value: 'cons_risk_wght_assets'
column_header: 'cons_risk_wght_assets'
- type: 'csv_header'
value: 'solo_cet1_amt'
column_header: 'solo_cet1_amt'
- type: 'csv_header'
value: 'solo_tier1_amt'
column_header: 'solo_tier1_amt'
- type: 'csv_header'
value: 'solo_tot_cap_amt'
column_header: 'solo_tot_cap_amt'
- type: 'csv_header'
value: 'solo_cet1_ratio'
column_header: 'solo_cet1_ratio'
- type: 'csv_header'
value: 'solo_tier1_ratio'
column_header: 'solo_tier1_ratio'
- type: 'csv_header'
value: 'solo_tot_cap_ratio'
column_header: 'solo_tot_cap_ratio'
- type: 'csv_header'
value: 'solo_risk_wght_assets'
column_header: 'solo_risk_wght_assets'
- type: 'csv_header'
value: 'cons_tot_assets'
column_header: 'cons_tot_assets'
- type: 'csv_header'
value: 'solo_tot_assets'
column_header: 'solo_tot_assets'
- type: 'csv_header'
value: 'cons_lev_ratio_full'
column_header: 'cons_lev_ratio_full'
- type: 'csv_header'
value: 'cons_lev_ratio_trans'
column_header: 'cons_lev_ratio_trans'
- type: 'csv_header'
value: 'cons_lev_ratio_req'
column_header: 'cons_lev_ratio_req'
- type: 'csv_header'
value: 'cons_lev_ratio_adj'
column_header: 'cons_lev_ratio_adj'
- type: 'csv_header'
value: 'solo_lev_ratio_full'
column_header: 'solo_lev_ratio_full'
- type: 'csv_header'
value: 'solo_lev_ratio_trans'
column_header: 'solo_lev_ratio_trans'
- type: 'csv_header'
value: 'solo_lev_ratio_req'
column_header: 'solo_lev_ratio_req'
- type: 'csv_header'
value: 'solo_lev_ratio_adj'
column_header: 'solo_lev_ratio_adj'
- type: 'csv_header'
value: 'cons_lc_ratio'
column_header: 'cons_lc_ratio'
- type: 'csv_header'
value: 'solo_lc_ratio'
column_header: 'solo_lc_ratio'
- type: 'csv_header'
value: 'cons_nsfr_ratio'
column_header: 'cons_nsfr_ratio'
- type: 'csv_header'
value: 'solo_nsfr_ratio'
column_header: 'solo_nsfr_ratio'
- type: 'csv_header'
value: 'submitter_comment'
column_header: 'submitter_comment'
- type: 'csv_header'
value: 'datacollectioncode'
column_header: 'datacollectioncode'
- type: 'csv_header'
value: 'reportingcyclename'
column_header: 'reportingcyclename'
- type: 'csv_header'
value: 'reportingcyclestatus'
column_header: 'reportingcyclestatus'
- type: 'csv_header'
value: 'modulecode'
column_header: 'modulecode'
- type: 'csv_header'
value: 'moduleversionnumber'
column_header: 'moduleversionnumber'
- type: 'csv_header'
value: 'reportingentityname'
column_header: 'reportingentityname'
- type: 'csv_header'
value: 'entityattributecountry'
column_header: 'entityattributecountry'
- type: 'csv_header'
value: 'entitygroupentityname'
column_header: 'entitygroupentityname'
- type: 'csv_header'
value: 'obligationmoduleremittancedate'
column_header: 'obligationmoduleremittancedate'
- type: 'csv_header'
value: 'obligationmoduleexpected'
column_header: 'obligationmoduleexpected'
- type: 'csv_header'
value: 'revalidationversionnumber'
column_header: 'revalidationversionnumber'
- type: 'csv_header'
value: 'receivedfilesystemfilename'
column_header: 'receivedfilesystemfilename'
- type: 'csv_header'
value: 'obligationstatusstatus'
column_header: 'obligationstatusstatus'
- type: 'csv_header'
value: 'filestatussetsubmissionstatus'
column_header: 'filestatussetsubmissionstatus'
- type: 'csv_header'
value: 'filestatussetvalidationstatus'
column_header: 'filestatussetvalidationstatus'
- type: 'csv_header'
value: 'numberoferrors'
column_header: 'numberoferrors'
- type: 'csv_header'
value: 'numberofwarnings'
column_header: 'numberofwarnings'
- type: 'csv_header'
value: 'delayindays'
column_header: 'delayindays'
- type: 'csv_header'
value: 'failedattempts'
column_header: 'failedattempts'
- type: 'csv_header'
value: 'tablename'
column_header: 'tablename'
- type: 'csv_header'
value: 'tec_source_system'
column_header: 'tec_source_system'
- type: 'csv_header'
value: 'tec_dataset'
column_header: 'tec_dataset'
- type: 'csv_header'
value: 'tec_surrogate_key'
column_header: 'tec_surrogate_key'
- type: 'csv_header'
value: 'tec_crc'
column_header: 'tec_crc'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'tec_version_id'
column_header: 'tec_version_id'
- type: 'csv_header'
value: 'tec_execution_date'
column_header: 'tec_execution_date'
- type: 'csv_header'
value: 'tec_run_id'
column_header: 'tec_run_id'
- type: 'csv_header'
value: 'tec_business_date'
column_header: 'tec_business_date'