From 2c225d68acce24ad3c8cce6311f151094ab9359e Mon Sep 17 00:00:00 2001 From: Grzegorz Michalski Date: Mon, 2 Mar 2026 09:47:35 +0100 Subject: [PATCH] init --- .ci/.gitkeep | 0 .ci/mr_dev_to_latest.sh | 74 + .gitignore | 5 + .gitlab-ci.yml | 96 + README.md | 5 + airflow/TestDags/.gitkeep | 0 airflow/TestDags/IDMC_Airflow_Test.py | 120 + airflow/TestDags/IICS_Airflow_Demo.py | 142 + .../TestDags/archive/devo_connector_test.py | 75 + .../archive/devo_replicator_scheduler_rar.py | 158 + .../TestDags/archive/devo_replicator_test.py | 112 + .../devo_replicator_withSQLOperator.py | 132 + .../archive/dmarsdb1_data_replicator_test.py | 65 + .../archive/old_devo_rqsd_mrds_workflow.py | 171 + .../archive/rqsd_devo_replicator_2.py | 320 + .../archive/test_oracle_connection.py | 18 + .../TestDags/ods_exdi_multi_task_processor.py | 244 + airflow/TestDags/pen_test_demo.py | 44 + .../TestDags/rqsd_casper_connector_test.py | 95 + .../t_MOPDB_RQSD_DEVO_OBSERVATIONS.py | 117 + airflow/devo_replicator/.gitkeep | 0 .../devo_replicator/devo_replicator_core.py | 346 ++ .../devo_replicator_core_pandas.py | 239 + .../devo_replicator_core_pyarrow.py | 244 + .../devo_replicator_scheduler_rar.py | 181 + .../devo_replicator_trigger.py | 255 + .../devo_replicator_trigger_mopdb.py | 257 + .../devo_replicator_trigger_rar.py | 257 + .../devo_replicator_trigger_rqsd.py | 257 + .../devo_replicator/devo_table_generator.py | 1080 ++++ .../devo_table_generator_trigger_mopdb.py | 255 + .../devo_table_generator_trigger_rar.py | 257 + .../devo_table_generator_trigger_rqsd.py | 239 + airflow/mopdb/CSDB/DEBT/w_MOPDB_CSDB_DEBT.py | 146 + .../w_MOPDB_LM_ADHOC_ADJUSTMENT.py | 156 + .../BALANCE_SHEET/w_MOPDB_LM_BALANCESHEET.py | 152 + .../w_MOPDB_LM_CSM_ADJUSTMENT.py | 147 + .../w_MOPDB_LM_CURRENT_ACCOUNTS.py | 157 + .../mopdb/LM/FORECAST/w_MOPDB_LM_FORECAST.py | 149 + .../w_MOPDB_LM_QRE_ADJUSTMENT.py | 155 + .../w_MOPDB_LM_STANDING_FACILITY.py | 175 + airflow/mopdb/RQSD/.gitkeep | 0 airflow/mopdb/RQSD/RQSD_PROCESS/.gitkeep | 0 .../RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS.py | 316 + .../w_MOPDB_RQSD_PROCESS_MANUAL.py | 216 + airflow/mopdb/TOP/.gitkeep | 0 .../w_MOPDB_TOP_AGGREGATED_ALLOTMENT.py | 156 + .../TOP/ALLOTMENT/w_MOPDB_TOP_ALLOTMENT.py | 156 + airflow/ods/c2d/uc/config/common_ucdb.xsd | 1534 +++++ .../ods/c2d/uc/config/disseminationFile.xsd | 124 + .../uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml | 45 + airflow/ods/c2d/uc/config/ucdb_cl.xsd | 968 +++ .../uc/config/xsd/UseOfCollateralMessage.xsd | 2412 +++++++ .../ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml | 386 ++ ...ation_DISS-FR-2023-02-16-34270_example.xml | 179 + airflow/ods/csdb/debt/.gitkeep | 0 airflow/ods/csdb/debt/config/.gitkeep | 0 .../debt/config/m_ODS_CSDB_DEBT_PARSE.yaml | 398 ++ .../yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml | 400 ++ airflow/ods/csdb/full_ratings/.gitkeep | 0 airflow/ods/csdb/full_ratings/config/.gitkeep | 0 .../m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml | 103 + .../m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml | 130 + .../m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml | 106 + .../m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml | 131 + .../w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py | 420 ++ .../dags/w_ODS_CSDB_RATINGS_FULL_CORE.py | 388 ++ .../config/m_ODS_FXCD_F_BRANCH_PARSE.yml | 29 + .../config/m_ODS_FXCD_F_CLEARER_PARSE.yml | 53 + .../m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml | 35 + .../config/m_ODS_FXCD_F_CONSTANT_PARSE.yml | 29 + .../m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml | 70 + .../config/m_ODS_FXCD_F_COUNTRY_PARSE.yml | 85 + .../m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml | 35 + .../config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml | 35 + ...m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml | 32 + .../m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml | 32 + .../config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml | 78 + ...ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml | 35 + .../m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml | 44 + ...DS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml | 32 + .../m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml | 41 + .../config/m_ODS_FXCD_F_ENTITY_PARSE.yml | 56 + .../m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml | 35 + .../config/m_ODS_FXCD_F_ISSUER_PARSE.yml | 77 + .../m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml | 38 + ...m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml | 29 + ...m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml | 47 + .../config/m_ODS_FXCD_F_RATING_PARSE.yml | 32 + .../m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml | 29 + .../ods/lm/adhoc_adjustments/config/.gitkeep | 0 .../config/adhoc_adjustments.xsd | 129 + .../m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml | 78 + .../dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py | 520 ++ airflow/ods/lm/balancesheet/config/.gitkeep | 0 .../lm/balancesheet/config/balancesheet.xsd | 102 + .../config/m_ODS_LM_BALANCESHEET_PARSE.yaml | 82 + .../dags/w_ODS_LM_BALANCESHEET.py | 520 ++ airflow/ods/lm/csm_adjustment/config/.gitkeep | 0 .../csm_adjustment/config/csm_adjustment.xsd | 129 + .../config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml | 83 + .../dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py | 520 ++ .../ods/lm/current_accounts/config/.gitkeep | 0 .../config/current_accounts.xsd | 106 + .../m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml | 91 + .../dags/w_ODS_LM_CURRENT_ACCOUNT.py | 520 ++ airflow/ods/lm/forecast/config/forecast.xsd | 91 + .../config/m_ODS_LM_FORECAST_PARSE.yaml | 63 + .../ods/lm/forecast/dags/w_ODS_LM_FORECAST.py | 520 ++ airflow/ods/lm/lm_common/.gitkeep | 0 airflow/ods/lm/lm_common/lm.xsd | 95 + .../lm/quarterly_adjustment/config/.gitkeep | 0 .../m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml | 82 + .../config/quarterly_adjustment.xsd | 129 + .../dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py | 520 ++ .../m_ODS_LM_Standing_Facilities_PARSE.yaml | 85 + .../ods/lm/standing_facilities/config/sf.xsd | 102 + .../ods/lm/standing_facilities/dags/.gitkeep | 0 .../dags/w_ODS_LM_STANDING_FACILITIES.py | 519 ++ ...ODS_LM_STANDING_FACILITIES_event_driven.py | 354 ++ .../config/yaml/flow_config_casper_test.yaml | 6 + .../config/yaml/m_ODS_RQSD_FX.yaml | 19 + .../config/yaml/m_ODS_RQSD_FX_PARSE.yaml | 34 + .../config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml | 31 + .../yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml | 181 + .../config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml | 736 +++ .../yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml | 259 + .../rqsd_process/dags/w_ODS_RQSD_PROCESS.py | 524 ++ .../dags/w_ODS_RQSD_PROCESS_MANUAL.py | 424 ++ airflow/ods/tms/README.md | 236 + .../TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr | 1641 +++++ .../TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml | 15 + .../w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr | 3143 ++++++++++ .../w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml | 20 + .../tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr | 338 + .../tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml | 5 + .../TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr | 248 + .../TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml | 5 + .../TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr | 248 + .../TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json | 13 + .../TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml | 12 + .../ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr | 896 +++ .../ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml | 20 + .../TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr | 418 ++ .../TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml | 9 + .../ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr | 145 + .../ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml | 5 + .../tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr | 161 + .../tms/TMS-layouts/w_ODS_TMS_CALENDAR.yml | 5 + .../tms/TMS-layouts/w_ODS_TMS_CASHFLOW.fkr | 3521 +++++++++++ .../tms/TMS-layouts/w_ODS_TMS_CASHFLOW.yml | 16 + .../TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.fkr | 470 ++ .../TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.yml | 8 + .../w_ODS_TMS_CLIENTINSTRUCTION.fkr | 1481 +++++ .../w_ODS_TMS_CLIENTINSTRUCTION.yml | 20 + .../w_ODS_TMS_CPBLOCKEDISSUERS.fkr | 129 + .../w_ODS_TMS_CPBLOCKEDISSUERS.yml | 5 + .../TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.fkr | 477 ++ .../TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.yml | 17 + .../w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.fkr | 1235 ++++ .../w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.yml | 5 + ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.fkr | 320 + ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.yml | 691 ++ ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.fkr | 336 + ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.yml | 2008 ++++++ ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.fkr | 304 + ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.yml | 453 ++ ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.fkr | 335 + ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.yml | 1701 +++++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.fkr | 590 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.yml | 358 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.fkr | 608 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.yml | 306 + ..._ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.fkr | 623 ++ ..._ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.yml | 173 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.fkr | 607 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.yml | 244 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.fkr | 558 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.yml | 529 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.fkr | 607 ++ ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.yml | 258 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.fkr | 558 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.yml | 110 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.fkr | 575 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.yml | 226 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.fkr | 576 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.yml | 799 +++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.fkr | 592 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.yml | 80 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.fkr | 591 ++ .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.yml | 125 + .../w_ODS_TMS_EFFECTIVEPERMISSIONS.fkr | 362 ++ .../w_ODS_TMS_EFFECTIVEPERMISSIONS.yml | 8 + .../w_ODS_TMS_EFFECTIVEROLEPROFILE.b64 | 2 + .../w_ODS_TMS_EFFECTIVEROLEPROFILE.fkr | 357 ++ .../w_ODS_TMS_EFFECTIVEROLEPROFILE.yml | 5 + .../TMS-layouts/w_ODS_TMS_FINMESSAGELOG.fkr | 1270 ++++ .../TMS-layouts/w_ODS_TMS_FINMESSAGELOG.yml | 12 + .../tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.fkr | 279 + .../tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.yml | 12 + .../w_ODS_TMS_INSTRUMENTBONDCASHFLOW.fkr | 1331 ++++ .../w_ODS_TMS_INSTRUMENTBONDCASHFLOW.yml | 5 + .../w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.fkr | 211 + .../w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.yml | 5 + .../w_ODS_TMS_INSTRUMENT_REPORT.fkr | 977 +++ .../w_ODS_TMS_INSTRUMENT_REPORT.yml | 5 + .../tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.fkr | 482 ++ .../tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.yml | 23 + .../tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.fkr | 650 ++ .../tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.yml | 23 + .../tms/TMS-layouts/w_ODS_TMS_PARAMETER.fkr | 162 + .../tms/TMS-layouts/w_ODS_TMS_PARAMETER.yml | 5 + .../w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.fkr | 284 + .../w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.yml | 14 + .../w_ODS_TMS_PRICE_RATE_REPORT.fkr | 769 +++ .../w_ODS_TMS_PRICE_RATE_REPORT.yml | 23 + .../tms/TMS-layouts/w_ODS_TMS_PROPERTY.fkr | 242 + .../tms/TMS-layouts/w_ODS_TMS_PROPERTY.yml | 5 + .../w_ODS_TMS_PSPPBLOCKEDINSTRUMENT.fkr | 129 + .../tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.fkr | 606 ++ .../tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.yml | 5 + .../w_ODS_TMS_RAR_CLIENTGROUPMAP.fkr | 261 + .../w_ODS_TMS_RAR_CLIENTGROUPMAP.yml | 5 + .../w_ODS_TMS_RAR_ECBINSTRUMENTS.fkr | 209 + .../w_ODS_TMS_RAR_ECBINSTRUMENTS.yml | 20 + .../w_ODS_TMS_RAR_FRM_HOLIDAYS.fkr | 161 + .../w_ODS_TMS_RAR_FRM_HOLIDAYS.yml | 5 + .../w_ODS_TMS_RAR_FUTURE_BONDS.fkr | 145 + .../w_ODS_TMS_RAR_FUTURE_BONDS.yml | 5 + .../w_ODS_TMS_RAR_INSTRUMENT_REPORT.fkr | 1377 ++++ .../w_ODS_TMS_RAR_INSTRUMENT_REPORT.yml | 5 + .../tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.fkr | 465 ++ .../tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.yml | 5 + .../TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.fkr | 129 + .../TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.yml | 5 + .../TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.fkr | 1809 ++++++ .../TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.yml | 12 + .../TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.fkr | 659 ++ .../TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.yml | 5 + .../TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.fkr | 513 ++ .../TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.yml | 5 + .../w_ODS_TMS_RAR_PORTFOLIOACCESS.fkr | 129 + .../w_ODS_TMS_RAR_PORTFOLIOACCESS.yml | 5 + .../w_ODS_TMS_RAR_PORTFOLIOTREE.fkr | 129 + .../w_ODS_TMS_RAR_PORTFOLIOTREE.yml | 5 + .../tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.fkr | 769 +++ .../tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.yml | 23 + .../w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.fkr | 129 + .../w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.yml | 8 + .../w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.fkr | 129 + .../w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.yml | 5 + .../w_ODS_TMS_RAR_RARTRANSLIMITCOND.fkr | 2737 ++++++++ .../w_ODS_TMS_RAR_RARTRANSLIMITCOND.yml | 9 + .../w_ODS_TMS_RAR_REINSTRUMENTDELIVERY.fkr | 145 + .../TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.fkr | 2161 +++++++ .../TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.yml | 5 + .../w_ODS_TMS_RAR_TRANSACTIONSLOG.fkr | 2321 +++++++ .../w_ODS_TMS_RAR_TRANSACTIONSLOG.yml | 17 + .../w_ODS_TMS_RAR_UMICREDITCLIENT.fkr | 1373 ++++ .../w_ODS_TMS_RAR_UMICREDITCLIENT.yml | 5 + .../w_ODS_TMS_RAR_UMISECURITYCODE.fkr | 193 + .../w_ODS_TMS_RAR_UMISECURITYCODE.yml | 5 + .../w_ODS_TMS_RAR_UMISECURITYINFO.fkr | 5533 +++++++++++++++++ .../w_ODS_TMS_RAR_UMISECURITYINFO.yml | 5 + .../TMS-layouts/w_ODS_TMS_RECONCILIATION.fkr | 508 ++ .../TMS-layouts/w_ODS_TMS_RECONCILIATION.yml | 17 + .../w_ODS_TMS_ROLEPORTFOLIOPROFILE.fkr | 179 + .../w_ODS_TMS_ROLEPORTFOLIOPROFILE.yml | 5 + .../ods/tms/TMS-layouts/w_ODS_TMS_RULES.fkr | 2674 ++++++++ .../ods/tms/TMS-layouts/w_ODS_TMS_RULES.yml | 5 + .../w_ODS_TMS_SDM_ENTITY_STATE.fkr | 364 ++ .../w_ODS_TMS_SDM_ENTITY_STATE.yml | 17 + .../w_ODS_TMS_SECURITYPOSITION.fkr | 808 +++ .../w_ODS_TMS_SECURITYPOSITION.yml | 20 + .../w_ODS_TMS_SETTLEMENTCASHFLOW.fkr | 1466 +++++ .../w_ODS_TMS_SETTLEMENTCASHFLOW.yml | 13 + .../TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.fkr | 2051 ++++++ .../TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.yml | 13 + ..._ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.fkr | 769 +++ ..._ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.yml | 18 + .../tms/TMS-layouts/w_ODS_TMS_TRANSACTION.fkr | 1595 +++++ .../tms/TMS-layouts/w_ODS_TMS_TRANSACTION.yml | 14 + .../w_ODS_TMS_TRANSACTIONSLOG_CURR.fkr | 2286 +++++++ .../w_ODS_TMS_TRANSACTIONSLOG_CURR.yml | 16 + .../w_ODS_TMS_TRANSACTION_ADHOC.fkr | 2961 +++++++++ .../TMS-layouts/w_ODS_TMS_USERINFORMATION.fkr | 276 + .../TMS-layouts/w_ODS_TMS_USERINFORMATION.yml | 5 + .../TMS-layouts/w_ODS_TMS_USER_ACCOUNT.fkr | 330 + .../TMS-layouts/w_ODS_TMS_USER_ACCOUNT.yml | 5 + airflow/ods/tms/TMS-tables/TMS_BRANCH.sql | 8 + .../TMS-tables/w_ODS_TMS_ACMCURRENCYFLOW.sql | 101 + .../w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql | 193 + .../ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY.sql | 20 + .../TMS-tables/w_ODS_TMS_ACTIVITYLOGDUE.sql | 14 + .../tms/TMS-tables/w_ODS_TMS_ACTIVITY_LOG.sql | 14 + .../ods/tms/TMS-tables/w_ODS_TMS_BALANCE.sql | 54 + .../tms/TMS-tables/w_ODS_TMS_BLACKOUT_LOG.sql | 21 + .../ods/tms/TMS-tables/w_ODS_TMS_CALENDAR.sql | 9 + .../ods/tms/TMS-tables/w_ODS_TMS_CASHFLOW.sql | 219 + .../TMS-tables/w_ODS_TMS_CLIENTACCOUNT.sql | 28 + .../w_ODS_TMS_CLIENTINSTRUCTION.sql | 91 + .../TMS-tables/w_ODS_TMS_CPBLOCKEDISSUERS.sql | 6 + .../w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.sql | 76 + ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.sql | 11 + ..._ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.sql | 12 + ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.sql | 12 + ...ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.sql | 13 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.sql | 26 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.sql | 29 + ..._ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.sql | 30 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.sql | 27 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.sql | 28 + ...w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.sql | 27 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.sql | 27 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.sql | 28 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.sql | 27 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.sql | 28 + .../w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.sql | 28 + .../w_ODS_TMS_EFFECTIVEPERMISSIONS.sql | 21 + .../w_ODS_TMS_EFFECTIVEROLEPROFILE.sql | 20 + .../TMS-tables/w_ODS_TMS_FINMESSAGELOG.sql | 77 + .../tms/TMS-tables/w_ODS_TMS_HISTORY_LOG.sql | 16 + .../w_ODS_TMS_INSTRUMENTBONDCASHFLOW.sql | 82 + .../w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.sql | 12 + .../w_ODS_TMS_INSTRUMENT_REPORT.sql | 60 + .../tms/TMS-tables/w_ODS_TMS_MOPDBPRICES.sql | 31 + .../tms/TMS-tables/w_ODS_TMS_PARAMETER.sql | 9 + .../w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.sql | 7 + .../w_ODS_TMS_PRICE_RATE_REPORT.sql | 17 + .../ods/tms/TMS-tables/w_ODS_TMS_PROPERTY.sql | 14 + .../tms/TMS-tables/w_ODS_TMS_RAR_CLIENT.sql | 16 + .../w_ODS_TMS_RAR_CLIENTGROUPMAP.sql | 9 + .../w_ODS_TMS_RAR_ECBINSTRUMENTS.sql | 10 + .../TMS-tables/w_ODS_TMS_RAR_FRM_HOLIDAYS.sql | 9 + .../TMS-tables/w_ODS_TMS_RAR_FUTURE_BONDS.sql | 8 + .../w_ODS_TMS_RAR_INSTRUMENT_REPORT.sql | 58 + .../tms/TMS-tables/w_ODS_TMS_RAR_LIMIT.sql | 14 + .../TMS-tables/w_ODS_TMS_RAR_LIMITACCESS.sql | 7 + .../tms/TMS-tables/w_ODS_TMS_RAR_LIMITLOG.sql | 111 + .../TMS-tables/w_ODS_TMS_RAR_MARKETINFO.sql | 27 + .../TMS-tables/w_ODS_TMS_RAR_PORTFOLIO.sql | 17 + .../w_ODS_TMS_RAR_PORTFOLIOACCESS.sql | 7 + .../w_ODS_TMS_RAR_PORTFOLIOTREE.sql | 7 + .../TMS-tables/w_ODS_TMS_RECONCILIATION.sql | 30 + .../w_ODS_TMS_ROLEPORTFOLIOPROFILE.sql | 10 + .../ods/tms/TMS-tables/w_ODS_TMS_RULES.sql | 166 + .../TMS-tables/w_ODS_TMS_SDM_ENTITY_STATE.sql | 21 + .../TMS-tables/w_ODS_TMS_SECURITYPOSITION.sql | 49 + .../w_ODS_TMS_SETTLEMENTCASHFLOW.sql | 90 + .../TMS-tables/w_ODS_TMS_SETTLEMENTLOG.sql | 126 + ..._ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.sql | 17 + .../tms/TMS-tables/w_ODS_TMS_TRANSACTION.sql | 139 + .../TMS-tables/w_ODS_TMS_USERINFORMATION.sql | 15 + .../tms/TMS-tables/w_ODS_TMS_USER_ACCOUNT.sql | 10 + airflow/ods/tms/config/TMS.yml | 14 + airflow/ods/tms/generate_tms_ods_dags.py | 539 ++ .../m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE.yaml | 184 + airflow/ods/top/allotment/config/.gitkeep | 0 .../config/m_ODS_TOP_ALLOTMENT_PARSE.yaml | 139 + .../allotment_modifications/config/.gitkeep | 0 ..._ODS_TOP_ALLOTMENT_MODIFICATION_PARSE.yaml | 78 + .../.gitkeep | 0 .../config/.gitkeep | 0 ...LOTMENT_MODIFICATION_AGGREGATED_PARSE.yaml | 66 + .../config/m_ODS_TOP_ACCOUNCEMENT_PARSE.yaml | 195 + airflow/ods/top/top_common/.gitkeep | 0 airflow/ods/top/top_common/TOPV2.xsd | 967 +++ airflow/tmp/.gitkeep | 0 .../tmp/oci_principal_authentication_debug.py | 73 + airflow/tmp/rotate_airflow_logs_to_oci.py | 195 + dbt/.gitignore | 4 + dbt/README.md | 15 + dbt/__profiles.yml | 47 + dbt/_profiles.yml | 47 + dbt/analyses/.gitkeep | 0 dbt/dbt_project.yml | 68 + dbt/dxc/remove_dbt_env_variables.ps1 | 39 + dbt/dxc/set_dbt_env_variables.ps1 | 38 + dbt/macros/.gitkeep | 0 dbt/macros/check_data_exists.sql | 19 + dbt/macros/common/check_model_exists.sql | 16 + dbt/macros/common/get_child_models.sql | 15 + dbt/macros/common/get_parent_models.sql | 12 + .../common/get_table_qualified_name.sql | 12 + .../control_external_run_end.sql | 22 + .../control_external_run_start.sql | 13 + .../control_tables/control_model_end.sql | 66 + .../control_tables/control_model_start.sql | 16 + dbt/macros/control_tables/control_run_end.sql | 49 + .../control_tables/control_run_start.sql | 12 + .../control_tables/control_source_count.sql | 7 + .../control_tables/get_task_history_key.sql | 20 + .../get_workflow_history_key.sql | 17 + .../control_tables/get_workflow_status.sql | 20 + .../insert_A_MOPDB_LOAD_HISTORY.sql | 25 + .../control_tables/insert_A_TASK_HISTORY.sql | 12 + .../insert_A_TASK_HISTORY_SOURCE.sql | 33 + .../insert_A_TASK_HISTORY_TARGET.sql | 13 + .../insert_A_WORKFLOW_HISTORY.sql | 13 + .../control_tables/update_A_TASK_HISTORY.sql | 16 + .../update_A_WORKFLOW_HISTORY.sql | 14 + dbt/macros/convert_timezone.sql | 12 + .../data_transform/clean_boolean_fxcd.sql | 8 + .../create_table_from_source.sql | 21 + .../data_transform/create_table_target.sql | 9 + .../filter_workflow_history_key.sql | 52 + .../filter_workflow_history_max_key.sql | 52 + .../get_ODS_wf_start_max_key.sql | 65 + .../data_transform/sequence_next_value.sql | 33 + .../dbt_overrides/conditional_truncate.sql | 10 + .../dbt_overrides/generate_schema_name.sql | 14 + .../oracle__snapshot_hash_arguments.sql | 9 + .../oracle__snapshot_staging_table.sql | 214 + .../dbt_overrides/unique_key_join_on.sql | 14 + dbt/macros/error_handlers/check_null.sql | 7 + dbt/macros/error_handlers/check_variable.sql | 7 + dbt/macros/get_main_task_name.sql | 5 + dbt/macros/has_tag.sql | 7 + dbt/macros/logging/elog.sql | 23 + dbt/macros/m_current_timestamp.sql | 22 + .../materialization/table_if_rows_exist.sql | 64 + .../mopdb/get_sql_t_collateral_default.sql | 268 + .../mopdb/get_sql_t_collateral_ma_dissem.sql | 100 + .../mopdb/get_sql_t_collateral_nma_acc.sql | 71 + .../get_sql_t_collateral_nma_decc_dissem.sql | 101 + .../mopdb/get_sql_t_collateral_nma_dissem.sql | 106 + .../mopdb/get_sql_t_collateral_prehook.sql | 126 + .../oracle/list_relations_without_caching.sql | 46 + dbt/models/control_tables/_sources.yml | 38 + dbt/models/example/my_first_dbt_model.sql | 27 + dbt/models/example/my_second_dbt_model.sql | 6 + dbt/models/example/schema.yml | 21 + .../m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ.sql | 37 + ...RAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ.sql | 41 + ...TERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ.sql | 72 + ...L_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql | 40 + ...AL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ.sql | 77 + ...T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ.sql | 47 + ...GGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql | 42 + ..._UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ.sql | 81 + ..._CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ.sql | 172 + .../m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ.sql | 130 + ...R_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ.sql | 60 + ...STR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ.sql | 65 + ..._DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ.sql | 60 + ...ER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ.sql | 63 + ...ATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ.sql | 61 + ...ING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ.sql | 60 + ...LA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ.sql | 66 + ...TS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ.sql | 21 + ...M_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ.sql | 39 + ...NT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ.sql | 42 + ..._ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ.sql | 41 + ...SM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ.sql | 42 + ...RE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ.sql | 44 + ..._MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ.sql | 39 + ...ACILITIES_OU_LM_STANDING_FACILITIES_SQ.sql | 43 + ..._MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ.sql | 284 + ...MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ.sql | 294 + ..._MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ.sql | 40 + ...TNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql | 27 + ...HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ.sql | 42 + ..._PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql | 63 + ...2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql | 54 + ...Y_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql | 70 + ...S_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql | 55 + ...ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql | 69 + ...URRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql | 69 + ...AD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ.sql | 25 + ...ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql | 69 + ...X_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql | 69 + ...ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql | 69 + ...X_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql | 69 + ...D_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql | 69 + ...OPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ.sql | 274 + ...UNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ.sql | 42 + ..._USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ.sql | 33 + ...DITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ.sql | 32 + ...CURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ.sql | 135 + ...OUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ.sql | 216 + ...OPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ.sql | 41 + .../m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ.sql | 36 + ...OPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ.sql | 29 + ..._CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ.sql | 53 + ...LIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ.sql | 34 + ...NSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ.sql | 128 + .../m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ.sql | 47 + ...USTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ.sql | 52 + ...ISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ.sql | 43 + ...PROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ.sql | 42 + ...MS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ.sql | 38 + ..._LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ.sql | 35 + ...ESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ.sql | 30 + ...PROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ.sql | 23 + ...ITYPOSITION_OU_TMS_SECURITYPOSITION_SQ.sql | 59 + ...LOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ.sql | 59 + ...AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ.sql | 37 + ...IONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ.sql | 35 + ...DB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ.sql | 52 + ..._T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ.sql | 66 + ...OPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ.sql | 28 + ...PILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ.sql | 37 + .../m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING.sql | 40 + .../targets/m_MOPDB_C2D_UC_AGGREGATED.sql | 198 + .../targets/m_MOPDB_C2D_UC_ALL_SOURCES.sql | 30 + ...PDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY.sql | 160 + .../m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT.sql | 140 + .../m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_OLD.sql | 142 + ...NSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL.sql | 63 + ..._INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL.sql | 58 + ...UER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL.sql | 63 + ...SSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL.sql | 58 + ...T_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT.sql | 62 + ...RATING_OU_CSDB_RATINGS_RT_PARTY_RATING.sql | 61 + ...T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION.sql | 62 + ...PORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN.sql | 19 + ...B_LM_T_BALANCESHEET_OU_LM_BALANCESHEET.sql | 37 + ...RRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS.sql | 44 + ...ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT.sql | 36 + ...C_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT.sql | 46 + ...C_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT.sql | 45 + .../m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST.sql | 34 + ...G_FACILITIES_OU_LM_STANDING_FACILITIES.sql | 37 + ...PEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL.sql | 540 ++ .../m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT.sql | 273 + ...C_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL.sql | 34 + ..._PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK.sql | 21 + ...LNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK.sql | 24 + ...NZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK.sql | 20 + ...Y_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql | 23 + ...NTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql | 60 + ...ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql | 23 + ...TY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql | 61 + ...L_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql | 66 + ..._RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS.sql | 22 + ...SD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS.sql | 94 + ...NNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql | 94 + ...SD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS.sql | 94 + ...NNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql | 94 + ...RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS.sql | 164 + ...m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA.sql | 112 + ...AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT.sql | 29 + ...DIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE.sql | 23 + ...QUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED.sql | 24 + ...ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW.sql | 118 + ...RGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP.sql | 210 + ...m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY.sql | 37 + .../m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH.sql | 25 + ...m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR.sql | 26 + ...S_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT.sql | 45 + ...T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP.sql | 28 + ...NTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION.sql | 109 + .../m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT.sql | 41 + ...T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE.sql | 46 + ...ERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS.sql | 39 + ...OLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE.sql | 37 + ...B_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG.sql | 34 + ...S_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS.sql | 25 + ...ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS.sql | 25 + ...LIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE.sql | 28 + ...CURITYPOSITION_OU_TMS_SECURITYPOSITION.sql | 67 + ..._ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT.sql | 56 + ...NS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR.sql | 33 + ...CATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS.sql | 28 + ...MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT.sql | 48 + ...TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT.sql | 60 + ...m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM.sql | 24 + ...COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED.sql | 31 + dbt/models/rar/_sequences.yml | 12 + .../rar/lookups/LKP_ASSET_CODE_TYPE_NO.sql | 21 + .../LKP_MU_ASSET_CODE_MAP_ASSET_FK.sql | 12 + ...T_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ.sql | 42 + .../m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ.sql | 168 + ...C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ.sql | 112 + ...D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ.sql | 17 + ..._SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ.sql | 220 + ..._C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT_SQ.sql | 15 + ..._C2D_EA_INTEREST_RATE_SWAP_PROVIDER_SQ.sql | 15 + ...2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK_SQ.sql | 16 + ...U_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER_SQ.sql | 15 + ...LOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR_SQ.sql | 16 + ...RELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER_SQ.sql | 16 + ...FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql | 68 + ...ATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_SQ.sql | 21 + ...E_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD_SQ.sql | 182 + ...SSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET.sql | 214 + .../m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD.sql | 173 + ...SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET.sql | 703 +++ ...ASSET_LOAD_TE_ASSET_TRANSACTIONAL_DATA.sql | 190 + ...LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT.sql | 87 + ..._C2D_EA_SF_TMS_UMI_SF_LED_LLMT_ERR_LOG.sql | 33 + ..._C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR.sql | 23 + ...OAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET.sql | 141 + ..._OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT.sql | 23 + ..._OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER.sql | 23 + ...F_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK.sql | 23 + ...A_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER.sql | 23 + ...PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR.sql | 23 + ...H_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER.sql | 23 + ...RS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql | 63 + ...T_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD.sql | 173 + ...T_RATING_SF_C2D_EA_ELIGIBLE_ASSET_GUAR.sql | 201 + ...NST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LT.sql | 189 + ...NST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_ST.sql | 184 + ...ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD.sql | 78 + dbt/models/sources/c2d.yml | 7 + dbt/models/sources/dw_rar.yml | 7 + dbt/models/sources/dw_ref_main.yml | 7 + dbt/models/sources/dw_tmp.yml | 7 + dbt/models/sources/mpec.yaml | 7 + dbt/models/sources/ods.yaml | 186 + dbt/models/sources/ou_c2d.yml | 25 + dbt/models/sources/ou_casdb.yml | 8 + dbt/models/sources/ou_lm.yml | 41 + dbt/models/sources/ou_mrr.yml | 10 + dbt/models/sources/ou_riad.yml | 16 + dbt/models/sources/ou_rqsd.yml | 9 + dbt/models/sources/ou_tms.yml | 37 + dbt/models/sources/ou_top.yml | 11 + dbt/models/sources/riad.yml | 7 + dbt/models/sources/sf_c2d.yml | 15 + dbt/models/sources/sf_led.yml | 8 + dbt/models/sources/sf_tms.yml | 9 + dbt/models/views/.gitkeep | 0 ...FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql | 63 + ...1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql | 9 + ...FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql | 64 + ...1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql | 9 + ...ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW.sql | 107 + ...X_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql | 155 + ...ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql | 9 + dbt/profiles.yml | 62 + dbt/seeds/.gitkeep | 0 dbt/snapshots/.gitkeep | 0 ..._ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot.sql | 346 ++ dbt/snapshots/m_DWH_FXCD_NH_F_BRANCH.sql | 35 + dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER.sql | 51 + .../m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT.sql | 39 + dbt/snapshots/m_DWH_FXCD_NH_F_CONSTANT.sql | 35 + .../m_DWH_FXCD_NH_F_COUNTERPARTY.sql | 62 + dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY.sql | 71 + .../m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT.sql | 39 + dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP.sql | 39 + .../m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT.sql | 37 + .../m_DWH_FXCD_NH_F_CTP_NCB_LIMIT.sql | 37 + dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY.sql | 70 + .../m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP.sql | 37 + .../m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER.sql | 47 + ...m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP.sql | 36 + .../m_DWH_FXCD_NH_F_ELIGIBILITY_MAP.sql | 39 + dbt/snapshots/m_DWH_FXCD_NH_F_ENTITY.sql | 54 + .../m_DWH_FXCD_NH_F_EQUIVALENCE_RULE.sql | 37 + dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER.sql | 70 + .../m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT.sql | 38 + .../m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT.sql | 38 + .../m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE.sql | 50 + dbt/snapshots/m_DWH_FXCD_NH_F_RATING.sql | 36 + .../m_DWH_FXCD_NH_F_RATING_AGENCY.sql | 38 + dbt/tests/.gitkeep | 0 python/code_generation/generate_code.py | 73 + python/code_generation/ods_mopdb_plain.txt | 30 + .../code_generation/templates/dag_Template.py | 95 + .../code_generation/templates/db_Template.sql | 2 + .../templates/m_Template_sq.sql | 11 + .../templates/m_Template_target.sql | 13 + python/connectors/casper/casper_rqsd.py | 201 + .../connectors/casper/config/env_config.yaml | 27 + python/connectors/devo/config/env_config.yaml | 25 + .../devo/config/env_config_rqsd.yaml | 25 + python/connectors/devo/devo_connector.py | 259 + python/connectors/tms/TMSDBT.py | 294 + python/connectors/tms/TMSQuery.py | 197 + python/connectors/tms/sample_DAG.py | 355 ++ python/devo_replicator/config/.gitkeep | 0 python/devo_replicator/config/env_config.yaml | 86 + .../devo_replicator/data_replicator/.gitkeep | 0 .../data_replicator/diag_s3_access.py | 65 + .../data_replicator/impala_refresher.py | 129 + .../data_replicator/list_s3_files_fast.py | 128 + .../devo_replicator/table_generator/.gitkeep | 0 .../table_generator/FlowOptions.py | 145 + .../table_generator/devo_query.py | 73 + .../table_generator/ranger_test.py | 69 + .../table_generator/ranger_updater.py | 250 + .../table_generator/ranger_updater_old.py | 345 + .../tableBuilderProcessor_3.py | 793 +++ .../table_generator/tableBuilderQueries.py | 130 + .../table_generator/testScript.py | 54 + python/mrds_common/.gitignore | 6 + python/mrds_common/CHANGELOG.md | 72 + python/mrds_common/README.md | 328 + python/mrds_common/mrds/__init__.py | 1 + python/mrds_common/mrds/cli.py | 117 + python/mrds_common/mrds/core.py | 366 ++ python/mrds_common/mrds/docs/rqsd_sample.yaml | 186 + python/mrds_common/mrds/docs/upload.py | 50 + .../mrds_common/mrds/processors/__init__.py | 15 + python/mrds_common/mrds/processors/base.py | 211 + .../mrds/processors/csv_processor.py | 52 + .../mrds/processors/xml_processor.py | 30 + python/mrds_common/mrds/utils/__init__.py | 0 python/mrds_common/mrds/utils/csv_utils.py | 69 + python/mrds_common/mrds/utils/manage_files.py | 177 + python/mrds_common/mrds/utils/manage_runs.py | 97 + python/mrds_common/mrds/utils/objectstore.py | 53 + python/mrds_common/mrds/utils/oraconn.py | 38 + python/mrds_common/mrds/utils/secrets.py | 46 + .../mrds_common/mrds/utils/security_utils.py | 106 + .../mrds_common/mrds/utils/sql_statements.py | 68 + python/mrds_common/mrds/utils/static_vars.py | 6 + python/mrds_common/mrds/utils/utils.py | 83 + python/mrds_common/mrds/utils/vault.py | 23 + python/mrds_common/mrds/utils/xml_utils.py | 177 + python/mrds_common/setup.py | 50 + python/mrds_common/tox.ini | 17 + 715 files changed, 130067 insertions(+) create mode 100644 .ci/.gitkeep create mode 100644 .ci/mr_dev_to_latest.sh create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 README.md create mode 100644 airflow/TestDags/.gitkeep create mode 100644 airflow/TestDags/IDMC_Airflow_Test.py create mode 100644 airflow/TestDags/IICS_Airflow_Demo.py create mode 100644 airflow/TestDags/archive/devo_connector_test.py create mode 100644 airflow/TestDags/archive/devo_replicator_scheduler_rar.py create mode 100644 airflow/TestDags/archive/devo_replicator_test.py create mode 100644 airflow/TestDags/archive/devo_replicator_withSQLOperator.py create mode 100644 airflow/TestDags/archive/dmarsdb1_data_replicator_test.py create mode 100644 airflow/TestDags/archive/old_devo_rqsd_mrds_workflow.py create mode 100644 airflow/TestDags/archive/rqsd_devo_replicator_2.py create mode 100644 airflow/TestDags/archive/test_oracle_connection.py create mode 100644 airflow/TestDags/ods_exdi_multi_task_processor.py create mode 100644 airflow/TestDags/pen_test_demo.py create mode 100644 airflow/TestDags/rqsd_casper_connector_test.py create mode 100644 airflow/TestDags/t_MOPDB_RQSD_DEVO_OBSERVATIONS.py create mode 100644 airflow/devo_replicator/.gitkeep create mode 100644 airflow/devo_replicator/devo_replicator_core.py create mode 100644 airflow/devo_replicator/devo_replicator_core_pandas.py create mode 100644 airflow/devo_replicator/devo_replicator_core_pyarrow.py create mode 100644 airflow/devo_replicator/devo_replicator_scheduler_rar.py create mode 100644 airflow/devo_replicator/devo_replicator_trigger.py create mode 100644 airflow/devo_replicator/devo_replicator_trigger_mopdb.py create mode 100644 airflow/devo_replicator/devo_replicator_trigger_rar.py create mode 100644 airflow/devo_replicator/devo_replicator_trigger_rqsd.py create mode 100644 airflow/devo_replicator/devo_table_generator.py create mode 100644 airflow/devo_replicator/devo_table_generator_trigger_mopdb.py create mode 100644 airflow/devo_replicator/devo_table_generator_trigger_rar.py create mode 100644 airflow/devo_replicator/devo_table_generator_trigger_rqsd.py create mode 100644 airflow/mopdb/CSDB/DEBT/w_MOPDB_CSDB_DEBT.py create mode 100644 airflow/mopdb/LM/ADH_ADJUSTMENTS/w_MOPDB_LM_ADHOC_ADJUSTMENT.py create mode 100644 airflow/mopdb/LM/BALANCE_SHEET/w_MOPDB_LM_BALANCESHEET.py create mode 100644 airflow/mopdb/LM/CSM_ADJUSTMENTS/w_MOPDB_LM_CSM_ADJUSTMENT.py create mode 100644 airflow/mopdb/LM/CURRENT_ACCOUNTS/w_MOPDB_LM_CURRENT_ACCOUNTS.py create mode 100644 airflow/mopdb/LM/FORECAST/w_MOPDB_LM_FORECAST.py create mode 100644 airflow/mopdb/LM/QRE_ADJUSTMENTS/w_MOPDB_LM_QRE_ADJUSTMENT.py create mode 100644 airflow/mopdb/LM/STANDING_FACILITY/w_MOPDB_LM_STANDING_FACILITY.py create mode 100644 airflow/mopdb/RQSD/.gitkeep create mode 100644 airflow/mopdb/RQSD/RQSD_PROCESS/.gitkeep create mode 100644 airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS.py create mode 100644 airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS_MANUAL.py create mode 100644 airflow/mopdb/TOP/.gitkeep create mode 100644 airflow/mopdb/TOP/AGGREGATED_ALLOTMENT/w_MOPDB_TOP_AGGREGATED_ALLOTMENT.py create mode 100644 airflow/mopdb/TOP/ALLOTMENT/w_MOPDB_TOP_ALLOTMENT.py create mode 100644 airflow/ods/c2d/uc/config/common_ucdb.xsd create mode 100644 airflow/ods/c2d/uc/config/disseminationFile.xsd create mode 100644 airflow/ods/c2d/uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml create mode 100644 airflow/ods/c2d/uc/config/ucdb_cl.xsd create mode 100644 airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd create mode 100644 airflow/ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml create mode 100644 airflow/ods/c2d/uc/test_data/MOC-CollateralDataDissemination-UCDB_dissemination_DISS-FR-2023-02-16-34270_example.xml create mode 100644 airflow/ods/csdb/debt/.gitkeep create mode 100644 airflow/ods/csdb/debt/config/.gitkeep create mode 100644 airflow/ods/csdb/debt/config/m_ODS_CSDB_DEBT_PARSE.yaml create mode 100644 airflow/ods/csdb/debt_daily/config/yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml create mode 100644 airflow/ods/csdb/full_ratings/.gitkeep create mode 100644 airflow/ods/csdb/full_ratings/config/.gitkeep create mode 100644 airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml create mode 100644 airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml create mode 100644 airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml create mode 100644 airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml create mode 100644 airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py create mode 100644 airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_CORE.py create mode 100644 airflow/ods/fxcd/BRANCH/config/m_ODS_FXCD_F_BRANCH_PARSE.yml create mode 100644 airflow/ods/fxcd/CLEARER/config/m_ODS_FXCD_F_CLEARER_PARSE.yml create mode 100644 airflow/ods/fxcd/CLEARER_NCB_LIMIT/config/m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/CONSTANT/config/m_ODS_FXCD_F_CONSTANT_PARSE.yml create mode 100644 airflow/ods/fxcd/COUNTERPARTY/config/m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml create mode 100644 airflow/ods/fxcd/COUNTRY/config/m_ODS_FXCD_F_COUNTRY_PARSE.yml create mode 100644 airflow/ods/fxcd/COUNTRY_NCB_LIMIT/config/m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/CTP_GROUP/config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml create mode 100644 airflow/ods/fxcd/CTP_GROUP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/CTP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/ELIGIBILITY/config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml create mode 100644 airflow/ods/fxcd/ELIGIBILITY_GROUP_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml create mode 100644 airflow/ods/fxcd/ELIGIBILITY_ISSUER/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml create mode 100644 airflow/ods/fxcd/ELIGIBILITY_ISSUER_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml create mode 100644 airflow/ods/fxcd/ELIGIBILITY_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml create mode 100644 airflow/ods/fxcd/ENTITY/config/m_ODS_FXCD_F_ENTITY_PARSE.yml create mode 100644 airflow/ods/fxcd/EQUIVALENCE_RULE/config/m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml create mode 100644 airflow/ods/fxcd/ISSUER/config/m_ODS_FXCD_F_ISSUER_PARSE.yml create mode 100644 airflow/ods/fxcd/ISSUER_NCB_LIMIT/config/m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/LIQUIDITY_NCB_LIMIT/config/m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml create mode 100644 airflow/ods/fxcd/NCB_PORTFOLIO_SHARE/config/m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml create mode 100644 airflow/ods/fxcd/RATING/config/m_ODS_FXCD_F_RATING_PARSE.yml create mode 100644 airflow/ods/fxcd/RATING_AGENCY/config/m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml create mode 100644 airflow/ods/lm/adhoc_adjustments/config/.gitkeep create mode 100644 airflow/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd create mode 100644 airflow/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml create mode 100644 airflow/ods/lm/adhoc_adjustments/dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py create mode 100644 airflow/ods/lm/balancesheet/config/.gitkeep create mode 100644 airflow/ods/lm/balancesheet/config/balancesheet.xsd create mode 100644 airflow/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml create mode 100644 airflow/ods/lm/balancesheet/dags/w_ODS_LM_BALANCESHEET.py create mode 100644 airflow/ods/lm/csm_adjustment/config/.gitkeep create mode 100644 airflow/ods/lm/csm_adjustment/config/csm_adjustment.xsd create mode 100644 airflow/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml create mode 100644 airflow/ods/lm/csm_adjustment/dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py create mode 100644 airflow/ods/lm/current_accounts/config/.gitkeep create mode 100644 airflow/ods/lm/current_accounts/config/current_accounts.xsd create mode 100644 airflow/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml create mode 100644 airflow/ods/lm/current_accounts/dags/w_ODS_LM_CURRENT_ACCOUNT.py create mode 100644 airflow/ods/lm/forecast/config/forecast.xsd create mode 100644 airflow/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml create mode 100644 airflow/ods/lm/forecast/dags/w_ODS_LM_FORECAST.py create mode 100644 airflow/ods/lm/lm_common/.gitkeep create mode 100644 airflow/ods/lm/lm_common/lm.xsd create mode 100644 airflow/ods/lm/quarterly_adjustment/config/.gitkeep create mode 100644 airflow/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml create mode 100644 airflow/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd create mode 100644 airflow/ods/lm/quarterly_adjustment/dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py create mode 100644 airflow/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml create mode 100644 airflow/ods/lm/standing_facilities/config/sf.xsd create mode 100644 airflow/ods/lm/standing_facilities/dags/.gitkeep create mode 100644 airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES.py create mode 100644 airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES_event_driven.py create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml create mode 100644 airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS.py create mode 100644 airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS_MANUAL.py create mode 100644 airflow/ods/tms/README.md create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.b64 create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_PSPPBLOCKEDINSTRUMENT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_REINSTRUMENTDELIVERY.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION_ADHOC.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.yml create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.fkr create mode 100644 airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.yml create mode 100644 airflow/ods/tms/TMS-tables/TMS_BRANCH.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMCURRENCYFLOW.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITYLOGDUE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY_LOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_BALANCE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_BLACKOUT_LOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_CALENDAR.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_CASHFLOW.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTACCOUNT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTINSTRUCTION.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_CPBLOCKEDISSUERS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEPERMISSIONS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEROLEPROFILE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_FINMESSAGELOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_HISTORY_LOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_REPORT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_MOPDBPRICES.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_PARAMETER.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_PRICE_RATE_REPORT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_PROPERTY.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENTGROUPMAP.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_ECBINSTRUMENTS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FRM_HOLIDAYS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FUTURE_BONDS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_INSTRUMENT_REPORT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMIT.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITACCESS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITLOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_MARKETINFO.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIO.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOACCESS.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOTREE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RECONCILIATION.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_ROLEPORTFOLIOPROFILE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_RULES.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_SDM_ENTITY_STATE.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_SECURITYPOSITION.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTCASHFLOW.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTLOG.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_TRANSACTION.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_USERINFORMATION.sql create mode 100644 airflow/ods/tms/TMS-tables/w_ODS_TMS_USER_ACCOUNT.sql create mode 100644 airflow/ods/tms/config/TMS.yml create mode 100644 airflow/ods/tms/generate_tms_ods_dags.py create mode 100644 airflow/ods/top/aggregated_allotment/config/m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE.yaml create mode 100644 airflow/ods/top/allotment/config/.gitkeep create mode 100644 airflow/ods/top/allotment/config/m_ODS_TOP_ALLOTMENT_PARSE.yaml create mode 100644 airflow/ods/top/allotment_modifications/config/.gitkeep create mode 100644 airflow/ods/top/allotment_modifications/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_PARSE.yaml create mode 100644 airflow/ods/top/allotment_modifications_aggregated/.gitkeep create mode 100644 airflow/ods/top/allotment_modifications_aggregated/config/.gitkeep create mode 100644 airflow/ods/top/allotment_modifications_aggregated/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_AGGREGATED_PARSE.yaml create mode 100644 airflow/ods/top/announcement/config/m_ODS_TOP_ACCOUNCEMENT_PARSE.yaml create mode 100644 airflow/ods/top/top_common/.gitkeep create mode 100644 airflow/ods/top/top_common/TOPV2.xsd create mode 100644 airflow/tmp/.gitkeep create mode 100644 airflow/tmp/oci_principal_authentication_debug.py create mode 100644 airflow/tmp/rotate_airflow_logs_to_oci.py create mode 100644 dbt/.gitignore create mode 100644 dbt/README.md create mode 100644 dbt/__profiles.yml create mode 100644 dbt/_profiles.yml create mode 100644 dbt/analyses/.gitkeep create mode 100644 dbt/dbt_project.yml create mode 100644 dbt/dxc/remove_dbt_env_variables.ps1 create mode 100644 dbt/dxc/set_dbt_env_variables.ps1 create mode 100644 dbt/macros/.gitkeep create mode 100644 dbt/macros/check_data_exists.sql create mode 100644 dbt/macros/common/check_model_exists.sql create mode 100644 dbt/macros/common/get_child_models.sql create mode 100644 dbt/macros/common/get_parent_models.sql create mode 100644 dbt/macros/common/get_table_qualified_name.sql create mode 100644 dbt/macros/control_tables/control_external_run_end.sql create mode 100644 dbt/macros/control_tables/control_external_run_start.sql create mode 100644 dbt/macros/control_tables/control_model_end.sql create mode 100644 dbt/macros/control_tables/control_model_start.sql create mode 100644 dbt/macros/control_tables/control_run_end.sql create mode 100644 dbt/macros/control_tables/control_run_start.sql create mode 100644 dbt/macros/control_tables/control_source_count.sql create mode 100644 dbt/macros/control_tables/get_task_history_key.sql create mode 100644 dbt/macros/control_tables/get_workflow_history_key.sql create mode 100644 dbt/macros/control_tables/get_workflow_status.sql create mode 100644 dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql create mode 100644 dbt/macros/control_tables/insert_A_TASK_HISTORY.sql create mode 100644 dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql create mode 100644 dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql create mode 100644 dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql create mode 100644 dbt/macros/control_tables/update_A_TASK_HISTORY.sql create mode 100644 dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql create mode 100644 dbt/macros/convert_timezone.sql create mode 100644 dbt/macros/data_transform/clean_boolean_fxcd.sql create mode 100644 dbt/macros/data_transform/create_table_from_source.sql create mode 100644 dbt/macros/data_transform/create_table_target.sql create mode 100644 dbt/macros/data_transform/filter_workflow_history_key.sql create mode 100644 dbt/macros/data_transform/filter_workflow_history_max_key.sql create mode 100644 dbt/macros/data_transform/get_ODS_wf_start_max_key.sql create mode 100644 dbt/macros/data_transform/sequence_next_value.sql create mode 100644 dbt/macros/dbt_overrides/conditional_truncate.sql create mode 100644 dbt/macros/dbt_overrides/generate_schema_name.sql create mode 100644 dbt/macros/dbt_overrides/oracle__snapshot_hash_arguments.sql create mode 100644 dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql create mode 100644 dbt/macros/dbt_overrides/unique_key_join_on.sql create mode 100644 dbt/macros/error_handlers/check_null.sql create mode 100644 dbt/macros/error_handlers/check_variable.sql create mode 100644 dbt/macros/get_main_task_name.sql create mode 100644 dbt/macros/has_tag.sql create mode 100644 dbt/macros/logging/elog.sql create mode 100644 dbt/macros/m_current_timestamp.sql create mode 100644 dbt/macros/materialization/table_if_rows_exist.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_default.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_ma_dissem.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_nma_acc.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_nma_decc_dissem.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_nma_dissem.sql create mode 100644 dbt/macros/mopdb/get_sql_t_collateral_prehook.sql create mode 100644 dbt/macros/oracle/list_relations_without_caching.sql create mode 100644 dbt/models/control_tables/_sources.yml create mode 100644 dbt/models/example/my_first_dbt_model.sql create mode 100644 dbt/models/example/my_second_dbt_model.sql create mode 100644 dbt/models/example/schema.yml create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ.sql create mode 100644 dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_C2D_UC_AGGREGATED.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_C2D_UC_ALL_SOURCES.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_OLD.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM.sql create mode 100644 dbt/models/mopdb/targets/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED.sql create mode 100644 dbt/models/rar/_sequences.yml create mode 100644 dbt/models/rar/lookups/LKP_ASSET_CODE_TYPE_NO.sql create mode 100644 dbt/models/rar/lookups/LKP_MU_ASSET_CODE_MAP_ASSET_FK.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_SQ.sql create mode 100644 dbt/models/rar/source_qualifiers/m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD_SQ.sql create mode 100644 dbt/models/rar/targets/m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET.sql create mode 100644 dbt/models/rar/targets/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD.sql create mode 100644 dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET.sql create mode 100644 dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_TRANSACTIONAL_DATA.sql create mode 100644 dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT.sql create mode 100644 dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT_ERR_LOG.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER.sql create mode 100644 dbt/models/rar/targets/m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql create mode 100644 dbt/models/rar/targets/m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD.sql create mode 100644 dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_GUAR.sql create mode 100644 dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LT.sql create mode 100644 dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_ST.sql create mode 100644 dbt/models/rar/targets/m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD.sql create mode 100644 dbt/models/sources/c2d.yml create mode 100644 dbt/models/sources/dw_rar.yml create mode 100644 dbt/models/sources/dw_ref_main.yml create mode 100644 dbt/models/sources/dw_tmp.yml create mode 100644 dbt/models/sources/mpec.yaml create mode 100644 dbt/models/sources/ods.yaml create mode 100644 dbt/models/sources/ou_c2d.yml create mode 100644 dbt/models/sources/ou_casdb.yml create mode 100644 dbt/models/sources/ou_lm.yml create mode 100644 dbt/models/sources/ou_mrr.yml create mode 100644 dbt/models/sources/ou_riad.yml create mode 100644 dbt/models/sources/ou_rqsd.yml create mode 100644 dbt/models/sources/ou_tms.yml create mode 100644 dbt/models/sources/ou_top.yml create mode 100644 dbt/models/sources/riad.yml create mode 100644 dbt/models/sources/sf_c2d.yml create mode 100644 dbt/models/sources/sf_led.yml create mode 100644 dbt/models/sources/sf_tms.yml create mode 100644 dbt/models/views/.gitkeep create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql create mode 100644 dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql create mode 100644 dbt/profiles.yml create mode 100644 dbt/seeds/.gitkeep create mode 100644 dbt/snapshots/.gitkeep create mode 100644 dbt/snapshots/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_BRANCH.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CONSTANT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_COUNTERPARTY.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_CTP_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_MAP.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ENTITY.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_EQUIVALENCE_RULE.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_RATING.sql create mode 100644 dbt/snapshots/m_DWH_FXCD_NH_F_RATING_AGENCY.sql create mode 100644 dbt/tests/.gitkeep create mode 100644 python/code_generation/generate_code.py create mode 100644 python/code_generation/ods_mopdb_plain.txt create mode 100644 python/code_generation/templates/dag_Template.py create mode 100644 python/code_generation/templates/db_Template.sql create mode 100644 python/code_generation/templates/m_Template_sq.sql create mode 100644 python/code_generation/templates/m_Template_target.sql create mode 100644 python/connectors/casper/casper_rqsd.py create mode 100644 python/connectors/casper/config/env_config.yaml create mode 100644 python/connectors/devo/config/env_config.yaml create mode 100644 python/connectors/devo/config/env_config_rqsd.yaml create mode 100644 python/connectors/devo/devo_connector.py create mode 100644 python/connectors/tms/TMSDBT.py create mode 100644 python/connectors/tms/TMSQuery.py create mode 100644 python/connectors/tms/sample_DAG.py create mode 100644 python/devo_replicator/config/.gitkeep create mode 100644 python/devo_replicator/config/env_config.yaml create mode 100644 python/devo_replicator/data_replicator/.gitkeep create mode 100644 python/devo_replicator/data_replicator/diag_s3_access.py create mode 100644 python/devo_replicator/data_replicator/impala_refresher.py create mode 100644 python/devo_replicator/data_replicator/list_s3_files_fast.py create mode 100644 python/devo_replicator/table_generator/.gitkeep create mode 100644 python/devo_replicator/table_generator/FlowOptions.py create mode 100644 python/devo_replicator/table_generator/devo_query.py create mode 100644 python/devo_replicator/table_generator/ranger_test.py create mode 100644 python/devo_replicator/table_generator/ranger_updater.py create mode 100644 python/devo_replicator/table_generator/ranger_updater_old.py create mode 100644 python/devo_replicator/table_generator/tableBuilderProcessor_3.py create mode 100644 python/devo_replicator/table_generator/tableBuilderQueries.py create mode 100644 python/devo_replicator/table_generator/testScript.py create mode 100644 python/mrds_common/.gitignore create mode 100644 python/mrds_common/CHANGELOG.md create mode 100644 python/mrds_common/README.md create mode 100644 python/mrds_common/mrds/__init__.py create mode 100644 python/mrds_common/mrds/cli.py create mode 100644 python/mrds_common/mrds/core.py create mode 100644 python/mrds_common/mrds/docs/rqsd_sample.yaml create mode 100644 python/mrds_common/mrds/docs/upload.py create mode 100644 python/mrds_common/mrds/processors/__init__.py create mode 100644 python/mrds_common/mrds/processors/base.py create mode 100644 python/mrds_common/mrds/processors/csv_processor.py create mode 100644 python/mrds_common/mrds/processors/xml_processor.py create mode 100644 python/mrds_common/mrds/utils/__init__.py create mode 100644 python/mrds_common/mrds/utils/csv_utils.py create mode 100644 python/mrds_common/mrds/utils/manage_files.py create mode 100644 python/mrds_common/mrds/utils/manage_runs.py create mode 100644 python/mrds_common/mrds/utils/objectstore.py create mode 100644 python/mrds_common/mrds/utils/oraconn.py create mode 100644 python/mrds_common/mrds/utils/secrets.py create mode 100644 python/mrds_common/mrds/utils/security_utils.py create mode 100644 python/mrds_common/mrds/utils/sql_statements.py create mode 100644 python/mrds_common/mrds/utils/static_vars.py create mode 100644 python/mrds_common/mrds/utils/utils.py create mode 100644 python/mrds_common/mrds/utils/vault.py create mode 100644 python/mrds_common/mrds/utils/xml_utils.py create mode 100644 python/mrds_common/setup.py create mode 100644 python/mrds_common/tox.ini diff --git a/.ci/.gitkeep b/.ci/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.ci/mr_dev_to_latest.sh b/.ci/mr_dev_to_latest.sh new file mode 100644 index 0000000..c52af21 --- /dev/null +++ b/.ci/mr_dev_to_latest.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +REPO_URL="https://oauth2:${GL_TOKEN}@gitlab.sofa.dev/mrds/mrds_elt.git" +DEV_BRANCH="dev" +TARGET_BRANCH="latest" +WORK_DIR="mrds_elt_merge" +MERGE_BRANCH="dev_to_latest_merge" +DIRECTORIES=( + "airflow/devo_replicator" + "airflow/ods/rqsd" + "airflow/mopdb/RQSD" + "dbt" + "python/connectors/devo" + "python/devo_replicator" + "python/mrds_common" + ".ci" + ".gitlab-ci.yml" + ".ci/mr_dev_to_latest.sh" +) + +echo "Setting up working directory..." +git branch -D "$MERGE_BRANCH" 2>/dev/null || true +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" +cd "$WORK_DIR" + +echo "Cloning repository..." +git clone "$REPO_URL" . +git config --global user.email "cicd@sofa.dev" +git config --global user.name "CICD Pipeline" + +echo "Creating merge branch from $TARGET_BRANCH..." +git checkout "$TARGET_BRANCH" +git checkout -b "$MERGE_BRANCH" + +echo "Fetching $DEV_BRANCH branch..." +git fetch origin "$DEV_BRANCH" + +echo "Selecting changes from $DEV_BRANCH for specific directories..." +CHANGES_FOUND=false + +for dir in "${DIRECTORIES[@]}"; do + echo "Copying $dir from $DEV_BRANCH to $MERGE_BRANCH" + git checkout "origin/$DEV_BRANCH" -- "$dir" +done + +if ! git diff --quiet "$TARGET_BRANCH"; then + git commit -am "Update selected directories from $DEV_BRANCH" + CHANGES_FOUND=true +fi + +if ! git diff --quiet "$TARGET_BRANCH"; then + CHANGES_FOUND=true +fi + +if [ "$CHANGES_FOUND" = true ]; then + echo "Pushing merge branch..." + git push -f -u "$REPO_URL" "$MERGE_BRANCH" + + echo "Creating merge request..." + if [ -z "$GL_TOKEN" ]; then + echo "GL_TOKEN environment variable not set. Cannot create merge request." + exit 1 + else + curl -X POST \ + -H "PRIVATE-TOKEN: $GL_TOKEN" \ + "https://gitlab.sofa.dev/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \ + -d "source_branch=${MERGE_BRANCH}&target_branch=${TARGET_BRANCH}&title=Requested merge of selected directories from ${DEV_BRANCH} to ${TARGET_BRANCH}&description=This MR contains selected directories from ${DEV_BRANCH} branch." + fi +else + echo "No changes found between $DEV_BRANCH and $TARGET_BRANCH for the specified directories." + echo "Skipping merge request creation." + exit 0 +fi \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94c94c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.log +.venv +.tox +*.egg-info/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..7907937 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,96 @@ +stages: + - analyze + - build + - merge + +variables: + TARGET_PROJECT: "mrds%2Foci-terraform%2Fmars%2Fmars-airflow" + GITLAB_API_URL: "https://gitlab.sofa.dev/api/v4/projects" + + +sonarqube_check_latest_sources: + stage: analyze + image: artifactory.sofa.dev/docker-remote/sonarsource/sonar-scanner-cli:latest + script: + - sonar-scanner + -Dsonar.projectKey=$SONAR_PROJECT_KEY + -Dsonar.projectBaseDir=. + -Dsonar.sources=. + -Dsonar.host.url=$SONAR_HOST_URL + -Dsonar.login=$SONAR_TOKEN + rules: + - if: '$CI_COMMIT_BRANCH == "latest"' + +build_airflow_from_dev_branch: + stage: build + when: manual + script: + - echo "Triggering pipeline in $TARGET_PROJECT on branch $TARGET_BRANCH..." + - apt-get update && apt-get install curl -y + - echo "#############################################################" + - echo "" + - echo -e "\e[31mContinue to https://gitlab.sofa.dev/mrds/oci-terraform/mars/mars-airflow/-/pipelines to see the build process progress, its result and to trigger deployment to DEV\e[0m" + - echo "" + - echo "#############################################################" + - | + set -e + RESPONSE=$(curl --silent --show-error --fail --request POST \ + --form token="$TRIGGER_TOKEN" \ + --form ref="dev" \ + --form "variables[COMMIT_MESSAGE]=$CI_COMMIT_MESSAGE" \ + --form "variables[COMMIT_SHA]=$CI_COMMIT_SHA" \ + --form "variables[MR_AUTHOR]=$GITLAB_USER_NAME" \ + "$GITLAB_API_URL/$TARGET_PROJECT/trigger/pipeline") || { + echo "ERROR: Failed to trigger remote pipeline!" + echo "$RESPONSE" + exit 1 + } + echo "$RESPONSE" + only: + - dev + +build_airflow_from_latest_branch: + stage: build + script: + - echo "Triggering pipeline in $TARGET_PROJECT on branch $TARGET_BRANCH..." + - apt-get update && apt-get install curl -y + - echo "#############################################################" + - echo "" + - echo -e "\e[31mContinue to https://gitlab.sofa.dev/mrds/oci-terraform/mars/mars-airflow/-/pipelines to see the build process and its result\e[0m" + - echo "" + - echo "#############################################################" + - | + set -e + RESPONSE=$(curl --silent --show-error --fail --request POST \ + --form token="$TRIGGER_TOKEN" \ + --form ref="main" \ + --form "variables[COMMIT_MESSAGE]=$CI_COMMIT_MESSAGE" \ + --form "variables[COMMIT_SHA]=$CI_COMMIT_SHA" \ + --form "variables[MR_AUTHOR]=$GITLAB_USER_NAME" \ + "$GITLAB_API_URL/$TARGET_PROJECT/trigger/pipeline") || { + echo "ERROR: Failed to trigger remote pipeline!" + echo "$RESPONSE" + exit 1 + } + echo "$RESPONSE" + only: + - latest + +create_mr_dev_to_latest: + stage: merge + script: + - apt-get update && apt-get install curl git sed -y + - | + if [ "$CI_COMMIT_BRANCH" != "dev" ]; then + echo -e "\e[31mERROR: Merge requests to latest branch are only allowed from dev branch.\e[0m" + exit 1 + fi + - bash .ci/mr_dev_to_latest.sh | tee mr_dev_to_latest.log + when: manual + allow_failure: false + only: + - dev + artifacts: + paths: + - mr_dev_to_latest.log + expire_in: 7 days \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..d367fdb --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# MRDS_ELT +## Building and deployment rules and procedures +https://europeancentralbank.atlassian.net/wiki/spaces/MRDS/pages/588221438/Gitlab+and+CI+CD+setup +# +# \ No newline at end of file diff --git a/airflow/TestDags/.gitkeep b/airflow/TestDags/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/TestDags/IDMC_Airflow_Test.py b/airflow/TestDags/IDMC_Airflow_Test.py new file mode 100644 index 0000000..5911bb9 --- /dev/null +++ b/airflow/TestDags/IDMC_Airflow_Test.py @@ -0,0 +1,120 @@ +import json +import sys +import time +import re +import requests +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.models import Variable +from airflow.operators.python_operator import PythonOperator + + +# from infromatic team, : connect to infromatica (akash) + +# Utility to make task_id Airflow-safe +def sanitize_task_id(task_id: str) -> str: + sanitized = re.sub(r'[^a-zA-Z0-9_]+', '_', task_id) + if not re.match(r'^[a-zA-Z0-9]', sanitized): + sanitized = 'task_' + sanitized + return sanitized + +# Fetch parameters from Airflow Variables +iics_username = Variable.get("iics_username") +iics_password = Variable.get("iics_password") +task_type = Variable.get("task_type", default_var="MTT") +base_url = Variable.get("iics_base_url", default_var="") + +# Task name +CDI_task_name = "CDI_task" + +# Default DAG args +default_args = { + 'owner': 'infa', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), + 'start_date': datetime.now() - timedelta(seconds=10), +} + +# API logic (same as before) +def get_session_id(un, pw): + data = {'@type': 'login', 'username': un, 'password': pw} + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + r = requests.post(base_url, data=json.dumps(data), headers=headers) + if r.status_code == 200: + return r.json()["icSessionId"], r.json()["serverUrl"] + else: + print('API call failed:', r.status_code) + print(r.text) + sys.exit(1) + +def start_job(session_id, server_url, taskname, taskType): + job_start_url = server_url + "/api/v2/job" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + data = {'@type': 'job', 'taskName': taskname, 'taskType': taskType} + r = requests.post(job_start_url, data=json.dumps(data), headers=headers) + if r.status_code == 200: + response_content = r.json() + print(f"Job {taskname} started successfully") + return response_content['taskId'], response_content['runId'] + else: + print('Job failed to start:', r.status_code) + print(r.text) + sys.exit(1) + +def get_status(server_url, session_id, task_id, run_id): + job_activity_url = server_url + "/api/v2/activity/activityMonitor" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + r = requests.get(job_activity_url, headers=headers) + if r.status_code == 200: + for obj in r.json(): + if obj['taskId'] == task_id and obj['runId'] == run_id: + return obj['executionState'] + else: + print('Failed to get status:', r.status_code) + print(r.text) + sys.exit(1) + +def execute_task(task_name): + session_id, server_url = get_session_id(iics_username, iics_password) + task_id, run_id = start_job(session_id, server_url, task_name, task_type) + + log_url = f"{server_url}/api/v2/activity/activityLog/" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + + while True: + time.sleep(15) + status = get_status(server_url, session_id, task_id, run_id) + print(f"Task status: {status}") + if status not in {"RUNNING", "INITIALIZED", "STOPPING", "QUEUED"}: + # Fetch logs on completion + url = f"{log_url}?taskId={task_id}&runId={run_id}" + r = requests.get(url, headers=headers) + logs = r.json() + for obj in logs: + log_id = obj['id'] + log_detail = requests.get(f"{log_url}{log_id}/sessionLog", headers=headers) + print(log_detail.text) + break + +# DAG with no schedule (manual trigger) +dag = DAG( + 'IDMC_Airflow_Test', + default_args=default_args, + description='Simplified DAG with one CDI task', + schedule_interval=None, + catchup=False +) + +safe_task_id = sanitize_task_id(CDI_task_name) + +run_cdi_task = PythonOperator( + task_id=safe_task_id, + python_callable=execute_task, + op_kwargs={'task_name': CDI_task_name}, + dag=dag +) diff --git a/airflow/TestDags/IICS_Airflow_Demo.py b/airflow/TestDags/IICS_Airflow_Demo.py new file mode 100644 index 0000000..7d75448 --- /dev/null +++ b/airflow/TestDags/IICS_Airflow_Demo.py @@ -0,0 +1,142 @@ +import json +import sys +import time +import re +import requests +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.models import Variable +from airflow.operators.dummy_operator import DummyOperator +from airflow.operators.python_operator import PythonOperator + + +# from infromatic team, : connect to infromatica (akash) + +# Utility to make task_id Airflow-safe +def sanitize_task_id(task_id: str) -> str: + # Replace invalid characters with underscores + sanitized = re.sub(r'[^a-zA-Z0-9_]+', '_', task_id) + # Ensure task_id starts with a letter or number + if not re.match(r'^[a-zA-Z0-9]', sanitized): + sanitized = 'task_' + sanitized + return sanitized + +# Fetch parameters from Airflow Variables +iics_username = Variable.get("iics_username") +iics_password = Variable.get("iics_password") +task_type = Variable.get("task_type", default_var="MTT") +base_url = Variable.get("iics_base_url", default_var="https://dm-us.informaticacloud.com/ma/api/v2/user/login") + +# Load task names from Airflow Variables +CDI_task_name = json.loads(Variable.get("CDI_task_name", default_var='["Task_Date_Dim", "Task_Items", "Task_Store_Sales"]')) +CDI_E_task_name = json.loads(Variable.get("CDI_E_task_name", default_var='["Task_Total_Store_Sales_IWDEMO"]')) + +# Default DAG args +default_args = { + 'owner': 'infa', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), + 'start_date': datetime.now() - timedelta(seconds=10), + 'schedule': '@daily' +} + +# API logic +def get_session_id(un, pw): + session_id = '' + data = {'@type': 'login', 'username': un, 'password': pw} + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + r = requests.post(base_url, data=json.dumps(data), headers=headers) + if r.status_code == 200: + session_id = r.json()["icSessionId"] + server_url = r.json()["serverUrl"] + else: + print('API call failed:', r.status_code) + print(r.text) + sys.exit(1) + return session_id, server_url + +def start_job(session_id, server_url, taskname, taskType): + job_start_url = server_url + "/api/v2/job" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + data = {'@type': 'job', 'taskName': taskname, 'taskType': taskType} + r = requests.post(job_start_url, data=json.dumps(data), headers=headers) + if r.status_code == 200: + response_content = r.json() + print("Job", taskname, "started successfully") + return response_content['taskId'], response_content['runId'], response_content['taskName'] + else: + print('Job failed to start:', r.status_code) + print(r.text) + +def get_status(server_url, session_id): + job_activity_url = server_url + "/api/v2/activity/activityMonitor" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + r = requests.get(job_activity_url, headers=headers) + if r.status_code == 200: + for obj in r.json(): + return obj['taskId'], obj['executionState'], obj['taskName'], obj['runId'] + else: + print('Failed to get status:', r.status_code) + print(r.text) + +def execute_task(task_name): + session_id, server_url = get_session_id(iics_username, iics_password) + task_id, run_id, _ = start_job(session_id, server_url, task_name, task_type) + + log_url = f"{server_url}/api/v2/activity/activityLog/" + headers = {'Content-Type': 'application/json', 'icSessionId': session_id, 'Accept': 'application/json'} + + while True: + time.sleep(15) + task_status = get_status(server_url, session_id) + if not task_status or task_status[1] not in {"RUNNING", "INITIALIZED", "STOPPING", "QUEUED"}: + # Fetch log + url = f"{log_url}?taskId={task_id}&runId={run_id}" + r = requests.get(url, headers=headers) + logs = r.json() + for obj in logs: + log_id = obj['id'] + log_detail = requests.get(f"{log_url}{log_id}/sessionLog", headers=headers) + print(log_detail.text) + break + +# Define DAG +dag = DAG( + 'IICS_Airflow_Demo', + default_args=default_args, + description='A Sample IICS Airflow DAG', + schedule_interval='@daily', + catchup=False +) + +# Task group 1: CDI Tasks +cdi_start = DummyOperator(task_id='cdi_start', dag=dag) +cdi_end = DummyOperator(task_id='cdi_end', dag=dag) + +for i in CDI_task_name: + safe_task_id = 'IICS_CDI_' + sanitize_task_id(i) + print(f"Creating task: {safe_task_id} for original task name: {i}") + cdi_task = PythonOperator( + task_id=safe_task_id, + python_callable=execute_task, + op_kwargs={'task_name': i}, + dag=dag + ) + cdi_start >> cdi_task >> cdi_end + +# Task group 2: CDI_E Tasks +for j in CDI_E_task_name: + safe_task_id = 'IICS_CDI_E_' + sanitize_task_id(j) + print(f"Creating E task: {safe_task_id} for original task name: {j}") + cdi_e_task = PythonOperator( + task_id=safe_task_id, + python_callable=execute_task, + op_kwargs={'task_name': j}, + dag=dag + ) + cdi_end >> cdi_e_task diff --git a/airflow/TestDags/archive/devo_connector_test.py b/airflow/TestDags/archive/devo_connector_test.py new file mode 100644 index 0000000..b340bf5 --- /dev/null +++ b/airflow/TestDags/archive/devo_connector_test.py @@ -0,0 +1,75 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from datetime import datetime, timedelta +import logging + +# Importing custom modules +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') +sys.path.append('/opt/airflow/python/connectors/devo') + +# Import the main function from your script +from devo_connector import main as devo_main + + ### DEVO CONNECTOR WITH STATIC workflow (task 3) + + +# Default DAG arguments +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 3, + 'retry_delay': timedelta(minutes=5), +} + +with DAG( + dag_id='devo_connector_test', + default_args=default_args, + description='Run devo RQSD data ingestion workflow', + schedule_interval=None, # we can set later + #start_date=datetime(2025, 10, 7), + catchup=False, + tags=['Devo', 'RQSD', 'Connector'], +) as dag: + + def run_devo_connector_rqsd(**context): + try: + # Pick env from ENV variables + env = os.getenv("MRDS_ENV") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + if not all([username, password, tnsalias]): + raise ValueError( + "Missing one or more required environment variables: " + "MRDS_LOADER_DB_USER, MRDS_LOADER_DB_PASS, MRDS_LOADER_DB_TNS" + ) + + logging.info( + f"Starting Casper RQSD workflow from Airflow DAG for env '{env}'" + ) + + workflow_context = {"run_id": 34, "a_workflow_history_key": 6} + flow_config_path = "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/flow_config_rqsd_observations.yaml" + env_config_path = "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml" + #env = "tst" + # flow_config_rqsd_observations.yaml + + logging.info("Starting Devo RQSD workflow from Airflow DAG") + devo_main(workflow_context, flow_config_path, env_config_path, env) + logging.info("Devo RQSD workflow completed successfully") + except Exception as e: + logging.error(f"Error running Devo RQSD workflow: {e}", exc_info=True) + raise + + run_devo = PythonOperator( + task_id='run_devo_connector_rqsd', + python_callable=run_devo_connector_rqsd, + ) diff --git a/airflow/TestDags/archive/devo_replicator_scheduler_rar.py b/airflow/TestDags/archive/devo_replicator_scheduler_rar.py new file mode 100644 index 0000000..f31fd50 --- /dev/null +++ b/airflow/TestDags/archive/devo_replicator_scheduler_rar.py @@ -0,0 +1,158 @@ +import os +import sys +import logging + +from airflow.decorators import dag +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow import DAG +from airflow.decorators import task +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from datetime import datetime, timedelta +from airflow.operators.python import BranchPythonOperator +from airflow.operators.empty import EmptyOperator + +from mrds.utils import oraconn + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +DAG_NAME = "dev_replicator_scheduler_rar" +TARGET_DAG_ID = "devo_replicator_trigger_rar" + +def get_devo_replica_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rar ORDER BY OWNER, TABLE_NAME") + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting MOPDB table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +def check_table_precondition(table_full_name): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + sql = """ + WITH LAST_UPDATE_ORACLE AS ( + SELECT max(process_end) as process_end + FROM CT_RAR.A_RAR_FOR_DISC_MONITORING + WHERE upper(owner||'.'||TARGET_TABLE_NAME) = upper(:table_name) + AND PROCESS_END is not null AND PROCESS_SUCCESSFUL='Y' + ), + LAST_UPDATE_DEVO AS ( + SELECT CASE WHEN last_status = 'FINISHED' THEN LAST_END_TIME ELSE TO_DATE('01-JAN-1999', 'DD-MON-YYYY') END as process_end + FROM CT_MRDS.a_devo_replica_mgmt_rar + WHERE OWNER || '.' || TABLE_NAME = :table_name + ) + SELECT CASE WHEN (SELECT process_end FROM LAST_UPDATE_ORACLE) > (SELECT process_end FROM LAST_UPDATE_DEVO) + THEN 'Y' ELSE 'N' END AS TRIGGER_DEVO_REPLICATOR FROM dual + """ + cursor.execute(sql, table_name=table_full_name) + result = cursor.fetchone() + status = result[0] if result else 'N' + logging.info(f"Precondition for {table_full_name}: {status}") + cursor.close() + return {"table": table_full_name, "trigger": status} + except Exception as e: + logging.error(f"Error checking precondition for {table_full_name}: {e}") + return {"table": table_full_name, "trigger": 'ERROR'} + finally: + if oracle_conn: + oracle_conn.close() + +def get_tables_to_trigger(precondition_results): + triggered_tables = [r["table"] for r in precondition_results if r["trigger"] == "Y"] + logging.info(f"Tables meeting precondition: {triggered_tables}") + return [{"owner_table": table_name} for table_name in triggered_tables] + +def branch_on_tables(ti): + precondition_results = ti.xcom_pull(task_ids='check_all_tables') + tables_to_trigger = [r["table"] for r in precondition_results if r["trigger"] == "Y"] + if tables_to_trigger: + return "trigger_devo_replicators" + else: + return "no_table_updated" + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=2), +} + +with DAG( + dag_id=DAG_NAME, + default_args=default_args, + schedule_interval=None, + catchup=False, + tags=['DevoScheduler', 'DevoReplicatorTrigger'] +) as dag: + + @task() + def fetch_tables(): + return get_devo_replica_table_options() + + @task() + def check_all_tables(table_list): + results = [check_table_precondition(tbl) for tbl in table_list] + count_y = sum(1 for r in results if r["trigger"] == "Y") + count_n = sum(1 for r in results if r["trigger"] == "N") + logging.info(f"Precondition results: {results}") + logging.info(f"Tables with trigger = 'Y': {count_y}") + logging.info(f"Tables with trigger = 'N': {count_n}") + return results + + @task() + def output_tables_to_trigger(precondition_results): + return get_tables_to_trigger(precondition_results) + + branch_task = BranchPythonOperator( + task_id="branch_trigger_check", + python_callable=branch_on_tables, + provide_context=True, + ) + + no_table_updated = EmptyOperator(task_id="no_table_updated") + + tables = fetch_tables() + precondition_results = check_all_tables(tables) + tables_to_trigger = output_tables_to_trigger(precondition_results) + + trigger_dag = TriggerDagRunOperator.partial( + task_id="trigger_devo_replicators", + trigger_dag_id=TARGET_DAG_ID, + execution_date="{{ ds }}" + ).expand(conf=tables_to_trigger) + + # Dependencies for branching + tables >> precondition_results >> tables_to_trigger >> branch_task + branch_task >> [trigger_dag, no_table_updated] + + + + +""" +1. fetch_tables gets the list of tables. +2. check_all_tables checks each table’s trigger status and logs counts. +3. output_tables_to_trigger prepares the mapped parameter list for triggering downstream DAGs. +4. branch_on_tables decides the path: + "trigger_devo_replicators" if any table triggers. + "no_table_updated" otherwise. +5. BranchPythonOperator implements the conditional branching. +6. TriggerDagRunOperator dynamically triggers a run of devo_replicator_trigger_rar per qualifying table. +7. EmptyOperator represents the "no tables to trigger" branch. + +""" \ No newline at end of file diff --git a/airflow/TestDags/archive/devo_replicator_test.py b/airflow/TestDags/archive/devo_replicator_test.py new file mode 100644 index 0000000..7af140f --- /dev/null +++ b/airflow/TestDags/archive/devo_replicator_test.py @@ -0,0 +1,112 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.providers.oracle.hooks.oracle import OracleHook +from airflow.utils.dates import days_ago +from datetime import timedelta +import logging + + +p_run_id = 1234 +p_service_name = 'MyService' +p_table_owner = 'MY_SCHEMA' +p_table_name = 'MY_TABLE' +p_objectstore_uri = 's3://bucket/uri' # subject to change appropriate for RAR/MOPDB + + +def start_log_table_task(**context): + proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.start_log_table(:1, :2, :3, :4); END;" + try: + oracle_hook = OracleHook(oracle_conn_id='oracle_default') + conn = oracle_hook.get_conn() + cursor = conn.cursor() + cursor.execute(proc_call, [p_run_id, p_service_name, p_table_owner, p_table_name]) + conn.commit() + cursor.close() + conn.close() + logging.info("start_log_table executed successfully.") + except Exception as e: + logging.error("Failed to execute start_log_table: %s", e, exc_info=True) + raise + +def export_table_task(**context): + proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.export_table(:1, :2, :3, :4); END;" + try: + oracle_hook = OracleHook(oracle_conn_id='oracle_default') + conn = oracle_hook.get_conn() + cursor = conn.cursor() + cursor.execute(proc_call, [p_service_name, p_table_owner, p_table_name, p_objectstore_uri]) + conn.commit() + cursor.close() + conn.close() + logging.info("export_table executed successfully.") + except Exception as e: + logging.error("Failed to execute export_table: %s", e, exc_info=True) + raise + +def devo_impyla_task(**context): + # Placeholder for Impyla (Devo) code + # Example for future: + # from impala.dbapi import connect + # conn = connect(host="...", port=21050) + # cursor = conn.cursor() + # cursor.execute("...") + logging.info("Impyla (Devo) task placeholder ran. Please implement.") + +def end_log_table_task(**context): + proc_call = "BEGIN MRDS_LOADER.DATA_REPLICATOR.end_log_table(:1, :2, :3); END;" + try: + oracle_hook = OracleHook(oracle_conn_id='oracle_default') + conn = oracle_hook.get_conn() + cursor = conn.cursor() + cursor.execute(proc_call, [p_service_name, p_table_owner, p_table_name]) + conn.commit() + cursor.close() + conn.close() + logging.info("end_log_table executed successfully.") + except Exception as e: + logging.error("Failed to execute end_log_table: %s", e, exc_info=True) + raise + + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 2, + 'retry_delay': timedelta(minutes=5), +} + + +with DAG( + dag_id='rqsd_devo_replicator_test_old', + default_args=default_args, + description='Run Devo replicator workflow', + schedule_interval=None, + catchup=False, + tags=['Devo', 'RQSD', 'Replicator'], +) as dag: + + t1 = PythonOperator( + task_id='start_log_table', + python_callable=start_log_table_task, + ) + + t2 = PythonOperator( + task_id='export_table', + python_callable=export_table_task, + ) + + t3 = PythonOperator( + task_id='devo_impyla', + python_callable=devo_impyla_task, + ) + + t4 = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + ) + + t1 >> t2 >> t3 >> t4 + diff --git a/airflow/TestDags/archive/devo_replicator_withSQLOperator.py b/airflow/TestDags/archive/devo_replicator_withSQLOperator.py new file mode 100644 index 0000000..51a01d7 --- /dev/null +++ b/airflow/TestDags/archive/devo_replicator_withSQLOperator.py @@ -0,0 +1,132 @@ +from airflow import DAG +#from airflow.providers.oracle.operators.oracle import SQLExecuteQueryOperator +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.utils.dates import days_ago +from datetime import timedelta +import logging + +# Importing custom modules +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + +# Import your functions +from mrds.utils.manage_runs import init_workflow, finalise_workflow +from devo_replicator.data_replicator.impala_refresher import main as impala_main + +#step 5) Devo replication + +## DEVO REPLICATOR WITH SQLOperator +### check the oracle connection, fixed params --> test cnx +### pick it from a file, + + + + +# TASK : +# - retrive directly from config file the param {0} and {1} based dev/test + +# need to be passed from infromatic (WLA call) to dags +# wla to airflow, cnx done + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 2, + 'retry_delay': timedelta(minutes=5), +} + +with DAG( + dag_id='rqsd_devo_replicator_2', + default_args=default_args, + description='Run Devo replicator workflow', + schedule_interval=None, + catchup=False, + tags=['Devo', 'RQSD', 'Replicator'], +) as dag: + + def init_step(**context): + env = os.getenv("MRDS_ENV") + corporate_store= "corporate store is 'crp_mopdb' for mopdb and 'crp_rar' for rar" + config_path = "/opt/airflow/python/devo_replicator/config/env_config.yaml" + + p_service_name = 'MOPDB' + p_table_owner = 'MPEC' + p_table_name = 'T_MPEC' + + #parse the config yml and filter by dev or test and mopdb or rar + p_objectstore_uri = 'https://devo-crp-ffppyd8q.bucket.vpce-040b28f5818b670c1-owicl3ow.s3.eu-central-1.vpce.amazonaws.com/mopdb/db' # subject to change as appropriate + p_run_id = str(context['ti'].run_id) + print(f"=== DEBUG INFO : {p_run_id} ===") + context['ti'].xcom_push(key='p_run_id', value=p_run_id) + + init_step = PythonOperator( + task_id='init_step', + python_callable=init_step, + provide_context=True, + ) + + t1 = SQLExecuteQueryOperator( + task_id='start_log_table', + oracle_conn_id='oracle_default', + # failed ,open up the cnx + sql="BEGIN MRDS_LOADER.DATA_REPLICATOR.start_log_table(:p_run_id, :p_service_name, :p_table_owner, :p_table_name); END;", + parameters={ + 'p_run_id': p_run_id, + 'p_service_name': p_service_name, + 'p_table_owner': p_table_owner, + 'p_table_name': p_table_name + }, + #oracle_conn_id='oracle_default' + ) + + t2 = SQLExecuteQueryOperator( + task_id='export_table', + oracle_conn_id='oracle_default', + sql="BEGIN MRDS_LOADER.DATA_REPLICATOR.export_table(:p_service_name, :p_table_owner, :p_table_name, :p_objectstore_uri); END;", + parameters={ + 'p_service_name': p_service_name, + 'p_table_owner': p_table_owner, + 'p_table_name': p_table_name, + 'p_objectstore_uri': p_objectstore_uri + }, + #oracle_conn_id='oracle_default' + ) + + # Leaving the Devo/Impyla task as a PythonOperator (placeholder) + from airflow.operators.python import PythonOperator + def devo_impyla_task(**context): + + status = impala_main(env_config_path, env, table, corporate_store) + logging.info("Impyla (Devo) task placeholder ran. Please implement.") + + + + # get details-data from impala ( its pending ) + + t3 = PythonOperator( + task_id='devo_impyla', + python_callable=devo_impyla_task, + ) + + + # push to s3, we need to call the proc + t4 = SQLExecuteQueryOperator( + task_id='end_log_table', + oracle_conn_id='oracle_default', + sql="BEGIN MRDS_LOADER.DATA_REPLICATOR.end_log_table(:p_service_name, :p_table_owner, :p_table_name); END;", + parameters={ + 'p_service_name': p_service_name, + 'p_table_owner': p_table_owner, + 'p_table_name': p_table_name + }, + #oracle_conn_id='oracle_default' + ) + + # t4 need to be executed always if we succeed or not ( if t1 failed then go directly to t4) + # t5 that will check if any of previous dag failed put everything will be read + + init_step >> t1 >> t2 >> t3 >> t4 diff --git a/airflow/TestDags/archive/dmarsdb1_data_replicator_test.py b/airflow/TestDags/archive/dmarsdb1_data_replicator_test.py new file mode 100644 index 0000000..26261e7 --- /dev/null +++ b/airflow/TestDags/archive/dmarsdb1_data_replicator_test.py @@ -0,0 +1,65 @@ +from airflow import DAG +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.operators.python import PythonOperator +from datetime import datetime, timedelta +import logging + +logger = logging.getLogger(__name__) + +## OLD ( Package repliction) + +def test_oracle_connection(**context): + """Test Oracle connection and log the result""" + conn_id = "marsdb_loader" + from airflow.providers.oracle.hooks.oracle import OracleHook + + try: + logger.debug("Attempting to connect to Oracle database...") + hook = OracleHook(oracle_conn_id=conn_id) + conn = hook.get_conn() + cursor = conn.cursor() + cursor.execute("SELECT 1 FROM dual") + result = cursor.fetchone() + logger.info(f"Connection test successful. Result: {result}") + cursor.close() + conn.close() + except Exception as e: + logger.error(f"Connection test failed: {str(e)}") + raise + +default_args = { + 'depends_on_past': False, + 'start_date': datetime(2025, 6, 25), + 'retries': 1, + 'retry_delay': timedelta(seconds=15), +} + +with DAG( + 'oracle_plsql_test_dag', + default_args=default_args, + schedule_interval=None, + catchup=False, +) as dag: + + test_connection = PythonOperator( + task_id='test_oracle_connection', + python_callable=test_oracle_connection, + ) + +# With named parameter + run_plsql = SQLExecuteQueryOperator( + task_id='run_plsql_procedure', + conn_id="marsdb_loader", + sql=""" + BEGIN + DATA_REPLICATOR.export_table( + p_table_owner => 'c2d', + p_table_name => 't_all_assets_servicer', + p_objectstore_uri => 'https://oci-test-sani.bucket.vpce-0b3a5f000733397b0-kxlyoh5z.s3.eu-central-1.vpce.amazonaws.com/', + p_date_column => 'SNAPSHOT_DATE' + ); + END; + """, + ) + + test_connection >> run_plsql diff --git a/airflow/TestDags/archive/old_devo_rqsd_mrds_workflow.py b/airflow/TestDags/archive/old_devo_rqsd_mrds_workflow.py new file mode 100644 index 0000000..ae8814c --- /dev/null +++ b/airflow/TestDags/archive/old_devo_rqsd_mrds_workflow.py @@ -0,0 +1,171 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from datetime import datetime, timedelta +import logging + +### DEVO CONNECTOR WITH DYNAMIC WORKFLOW CONTEXT & HISTORY KEY + +# Importing custom modules +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + +# Import your functions +from mrds.utils.manage_runs import init_workflow, finalise_workflow +from devo_connector import main as devo_main +from devo_connector_v2 import run as devo_main2 +from mrds.core import main as mrds_main + +# Default arguments +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 3, + 'retry_delay': timedelta(minutes=5), +} + +# Dynamic name extraction from basename +dag_id = os.path.splitext(os.path.basename(__file__))[0] + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='Run devo RQSD data ingestion workflow with MRDS processing', + schedule_interval=None, + catchup=False, + tags=["Devo", "RQSD", "MRDS", "Connector"], + params={ + "source_filename": "", + "config_file": "", + }, +) as dag: + + def run_devo_connector_rqsd(**context): + """Run Devo RQSD connector workflow""" + try: + env = os.getenv("MRDS_ENV") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + raise ValueError( + "Missing one or more required environment variables: " + "MRDS_LOADER_DB_USER, MRDS_LOADER_DB_PASS, MRDS_LOADER_DB_TNS" + ) + + logging.info(f"Starting Devo RQSD workflow from Airflow DAG for env '{env}'") + + database_name = 'MOPDB' + workflow_name = 'w_MOPDB_RQSD_PROCESS' + workflow_run_id = str(context['ti'].run_id) + + + #comment + a_workflow_history_key = init_workflow(database_name, workflow_name, workflow_run_id) + logging.info(f"Initialized workflow with history key: {a_workflow_history_key}") + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + flow_config_path = "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/flow_config_rqsd_observations.yaml" + env_config_path = "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml" + + logging.info("Starting Devo RQSD workflow from Airflow DAG") + count = devo_main2(workflow_context, flow_config_path, env_config_path, env) + print("=================================================================") + print(f"Devo RQSD workflow completed successfully with count : {count}") + logging.info(f"Devo RQSD workflow completed successfully with count : {count}") + + # Push the workflow context and history key to XCom for downstream tasks + context['ti'].xcom_push(key='workflow_history_key', value=a_workflow_history_key) + context['ti'].xcom_push(key='workflow_context', value=workflow_context) + + except Exception as e: + logging.error(f"Error running Devo RQSD workflow: {e}", exc_info=True) + # If init_workflow succeeded but workflow failed, finalize with FAILED status + if 'a_workflow_history_key' in locals(): + try: + finalise_workflow(a_workflow_history_key, "FAILED") + except Exception as finalise_error: + logging.error(f"Failed to finalise workflow after error: {finalise_error}") + raise + + def run_mrds_task(**context): + """Run MRDS processing task""" + try: + + ti = context.get('ti') + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='run_devo_connector_rqsd') + if not workflow_context: + raise ValueError("No workflow_context from Task 1") + + print("=== workflow_context ====:",workflow_context) + + source_filename = "RQSD_OBSERVATIONS.csv" + config_file = "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/flow_config_devo_process.yaml" + + print("---- run_mrds_task ----") + print("source_filename :", source_filename) + print("config_file = ", config_file) + print("------------------------") + + + if not source_filename: + raise ValueError("No source_filename provided in DAG run params.") + if not config_file: + raise ValueError("No config_file path provided in DAG run params.") + + logging.info(f"Starting MRDS task with source_filename: {source_filename}, config_file: {config_file}") + + # Run MRDS with the workflow context from the previous task + mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=True) + + logging.info("MRDS task completed successfully") + + except Exception as e: + logging.error(f"Error running MRDS task: {e}", exc_info=True) + raise + + def finalise_workflow_task(**context): + """Finalize workflow with SUCCESS status""" + # Pull the workflow_history_key from XCom pushed by the main task + ti = context['ti'] + a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='run_devo_connector_rqsd') + + if a_workflow_history_key is None: + raise ValueError("No workflow history key found in XCom; cannot finalise workflow") + + # Call finalise with SUCCESS status + finalise_workflow(a_workflow_history_key, "SUCCESS") + logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS") + + # Task definitions + run_devo = PythonOperator( + task_id='run_devo_connector_rqsd', + python_callable=run_devo_connector_rqsd, + provide_context=True, + ) + + run_mrds = PythonOperator( + task_id='run_mrds_task', + python_callable=run_mrds_task, + provide_context=True, + ) + + finalize = PythonOperator( + task_id='finalise_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + ) + + # Task dependencies + run_devo >> run_mrds >> finalize diff --git a/airflow/TestDags/archive/rqsd_devo_replicator_2.py b/airflow/TestDags/archive/rqsd_devo_replicator_2.py new file mode 100644 index 0000000..91de458 --- /dev/null +++ b/airflow/TestDags/archive/rqsd_devo_replicator_2.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import os +import sys +import logging +import yaml +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.python import PythonOperator +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator + +try: + from airflow.exceptions import AirflowFailException +except Exception: # fallback for older Airflow + from airflow.exceptions import AirflowException as AirflowFailException + +# --- Custom module paths (as in snippet) --- +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') +sys.path.append('/opt/airflow/python/devo_replicator/data_replicator') + +# --- custom imports --- +from mrds.utils import oraconn +from impala_refresher import main as impala_main + +# --- Config path --- +ENV_CONFIG_PATH = "/opt/airflow/python/devo_replicator/config/env_config.yaml" + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 2, + 'retry_delay': timedelta(minutes=5), +} + +with DAG( + dag_id='rqsd_devo_replicator_2', + default_args=default_args, + description='Run Devo replicator workflow', + schedule=None, + catchup=False, + tags=['Devo', 'RQSD', 'Replicator'], +) as dag: + + # ------------------------------- + # 1) Init: read config + set XCom + # ------------------------------- + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + + conf = (dag_run.conf or {}) if dag_run else {} + + env = conf.get("env") or os.getenv("MRDS_ENV", "dev").lower() + if env not in {"dev", "tst"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev' or 'tst'.") + + # hardcoded the mopdb + # ==================================== + store = "mopdb" + # ==================================== + + if store not in {"mopdb", "rar"}: + raise ValueError(f"Unsupported store '{store}'. Expected 'mopdb' or 'rar'.") + + p_service_name = "MOPDB" if store == "mopdb" else "RAR" + p_table_owner = "MPEC" + p_table_name = "T_MPEC" + + with open(ENV_CONFIG_PATH, "r") as f: + cfg = yaml.safe_load(f) + + env_cfg = cfg[env] + store_cfg = cfg[store] + + p_objectstore_uri = env_cfg["S3_LOCATION_URI"].replace("{0}",store.lower()) + + p_run_id = str(ti.run_id) + logging.info("=== init_step === env=%s store=%s run_id=%s", env, store, p_run_id) + logging.info("objectstore_uri=%s", p_objectstore_uri) + + xcom = { + "env": env, + "store": store, + "config_path": ENV_CONFIG_PATH, + "p_run_id": p_run_id, + "p_service_name": p_service_name, + "p_table_owner": p_table_owner, + "p_table_name": p_table_name, + "p_objectstore_uri": p_objectstore_uri, + "corporate_store": store_cfg["corporate_store"], # "crp_mopdb" or "crp_rar" + } + + print(" ============= DEBUG PARAMS ============= ") + print(xcom) + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # ------------------------------------ + # 2) log table (Oracle procedure) + # ------------------------------------ + def start_log_table_task(**context): + ti = context["ti"] + + # Get parameters from XCom + p_run_id = ti.xcom_pull(task_ids='init_step', key='p_run_id') + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + # Create Oracle connection for this task + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + logging.info("Oracle connection established successfully for start_log_table") + + # Execute Oracle procedure using oraconn.run_proc() directly + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.start_log_table', + [p_run_id, p_service_name, p_table_owner, p_table_name] + ) + oracle_conn.commit() + logging.info("start_log_table procedure executed successfully") + + except Exception as e: + logging.error(f"Error in start_log_table: {e}") + raise + finally: + if oracle_conn: + try: + oracle_conn.close() + logging.info("Oracle connection closed for start_log_table") + except Exception as e: + logging.error(f"Error closing connection in start_log_table: {e}") + + t1 = PythonOperator( + task_id='start_log_table', + python_callable=start_log_table_task, + ) + + # --------------------------------------------------------- + # 3) Export table (Oracle procedure writes to object store) + # --------------------------------------------------------- + def export_table_task(**context): + ti = context["ti"] + + # Get parameters from XCom + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + p_objectstore_uri = ti.xcom_pull(task_ids='init_step', key='p_objectstore_uri') + + # Create Oracle connection for this task + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + logging.info("Oracle connection established successfully for export_table") + + # Execute Oracle procedure using oraconn.run_proc() directly + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.export_table', + [p_service_name, p_table_owner, p_table_name, p_objectstore_uri] + ) + oracle_conn.commit() + logging.info("export_table procedure executed successfully") + + except Exception as e: + logging.error(f"Error in export_table: {e}") + raise + finally: + if oracle_conn: + try: + oracle_conn.close() + logging.info("Oracle connection closed for export_table") + except Exception as e: + logging.error(f"Error closing connection in export_table: {e}") + + t2 = PythonOperator( + task_id='export_table', + python_callable=export_table_task, + trigger_rule=TriggerRule.ALL_DONE, # Continue even if t1 failed + ) + + # --------------------------------------------- + # 4) Devo / Impyla refresh (Python Package) + # --------------------------------------------- + def devo_impyla_task(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + corporate_store = ti.xcom_pull(task_ids='init_step', key='corporate_store') + config_path = ti.xcom_pull(task_ids='init_step', key='config_path') + owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + table = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + # For Impala: corporate_store.table_name (e.g., crp_mopdb.T_MPEC) + # NOT corporate_store.owner.table_name (which would be crp_mopdb.MPEC.T_MPEC - INVALID) + table_name = table # Just "T_MPEC" + + logging.info( + "Starting Impyla refresh with env=%s store=%s corporate_store=%s table=%s", + env, store, corporate_store, table_name + ) + logging.info("Will execute: INVALIDATE METADATA %s.%s", corporate_store, table_name) + logging.info("Will execute: COMPUTE STATS %s.%s", corporate_store, table_name) + + try: + # This should result in queries like: + # INVALIDATE METADATA crp_mopdb.T_MPEC + # COMPUTE STATS crp_mopdb.T_MPEC + status = impala_main(config_path, env, table_name, corporate_store) + logging.info("Impyla (Devo) task finished successfully. Status: %s", status) + return status + except Exception as e: + logging.error(f"Error in devo_impyla_task: {e}") + raise + + t3 = PythonOperator( + task_id='devo_impyla', + python_callable=devo_impyla_task, + trigger_rule=TriggerRule.ALL_DONE, # Continue even if t2 failed + ) + + # ------------------------------------- + # 5) End log table (always executes after t1, t2, t3 complete - regardless of success/failure) + # ------------------------------------- + def end_log_table_task(**context): + ti = context["ti"] + + # Get parameters from XCom + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + # Create Oracle connection for this task + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + logging.info("Oracle connection established successfully for end_log_table") + + # Execute Oracle procedure using oraconn.run_proc() directly + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.end_log_table', + [p_service_name, p_table_owner, p_table_name] + ) + oracle_conn.commit() + logging.info("end_log_table procedure executed successfully") + + except Exception as e: + logging.error(f"Error in end_log_table: {e}") + # Don't raise the exception since this is a cleanup task + logging.info("Continuing despite end_log_table error (cleanup task)") + finally: + if oracle_conn: + try: + oracle_conn.close() + logging.info("Oracle connection closed for end_log_table") + except Exception as e: + logging.error(f"Error closing connection in end_log_table: {e}") + + t4 = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + trigger_rule=TriggerRule.ALL_DONE, # Run after t1, t2, t3 complete (success or failure) + ) + + # ----------------------------------------------------- + # 6) Check and fail the DAG if any of t1..t3 actually failed + # This task always runs after t4, but will fail the DAG if needed + # ----------------------------------------------------- + def fail_if_any_failed(**context): + dag_run = context['dag_run'] + check_tasks = ['start_log_table', 'export_table', 'devo_impyla'] + failed = [] + + for tid in check_tasks: + ti_up = dag_run.get_task_instance(tid) + if ti_up and ti_up.state == 'failed': + failed.append(tid) + + if failed: + error_msg = f"Critical task(s) failed: {', '.join(failed)}. DAG execution failed." + logging.error(error_msg) + raise AirflowFailException(error_msg) + + logging.info("All critical tasks completed successfully: %s", check_tasks) + + t5 = PythonOperator( + task_id='fail_if_any_failed', + python_callable=fail_if_any_failed, + trigger_rule=TriggerRule.ALL_DONE, # Always run after t4 + ) + + # --------- + # Task Dependencies - SEQUENTIAL + # --------- + # Sequential flow: init -> t1 -> t2 -> t3 + init >> t1 >> t2 >> t3 + + # t4 runs after t1, t2, t3 are all done (regardless of success/failure) + [t1, t2, t3] >> t4 + + # t5 always runs after t4 to check for failures and fail the DAG if needed + t4 >> t5 diff --git a/airflow/TestDags/archive/test_oracle_connection.py b/airflow/TestDags/archive/test_oracle_connection.py new file mode 100644 index 0000000..70401ec --- /dev/null +++ b/airflow/TestDags/archive/test_oracle_connection.py @@ -0,0 +1,18 @@ +from airflow import DAG +#from airflow.providers.oracle.operators.oracle import OracleOperator +from airflow.operators.bash import BashOperator +from datetime import datetime +from airflow import DAG +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator + +with DAG( + 'test_oracle_connection', + start_date=datetime(2025, 6, 13), + schedule_interval=None +) as dag: + test_query = SQLExecuteQueryOperator( + task_id='test_oracle_query', + conn_id='oracle_default', + sql='SELECT 1 FROM DUAL' + ) + diff --git a/airflow/TestDags/ods_exdi_multi_task_processor.py b/airflow/TestDags/ods_exdi_multi_task_processor.py new file mode 100644 index 0000000..2bc4271 --- /dev/null +++ b/airflow/TestDags/ods_exdi_multi_task_processor.py @@ -0,0 +1,244 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from datetime import datetime, timedelta +import logging + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Importing custom modules +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') + +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + + +# Configuration Dictionary - First key will be mandatory, +# Workflow: Init → First Task (Sequential) → Parallel Tasks → Finalize +# Input: Only needs PARSE.yaml config file and source filename + +TASK_CONFIGS = { + "m_ODS_EXDI_TASK1": { + "source_filename": "EXDI_TASK1.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/exdi/exdi_process/config/yaml/m_ODS_EXDI_TASK1_PARSE.yaml" + }, + "m_ODS_EXDI_TASK2": { + "source_filename": "EXDI_TASK2.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/exdi/exdi_process/config/yaml/m_ODS_EXDI_TASK2_PARSE.yaml" + }, + "m_ODS_EXDI_TASK3": { + "source_filename": "EXDI_TASK3.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/exdi/exdi_process/config/yaml/m_ODS_EXDI_TASK3_PARSE.yaml" + } +} + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +dag_id = os.path.splitext(os.path.basename(__file__))[0] + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id +} + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='Run EXDI data processing workflow with MRDS - Multi-task', + schedule_interval=None, + catchup=False, + tags=["EXDI", "MRDS", "Multi-Task", "ODS"] +) as dag: + + def init_workflow_task(**context): + """Initialize workflow and set up context""" + try: + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing_vars = [] + if not username: missing_vars.append("MRDS_LOADER_DB_USER") + if not password: missing_vars.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing_vars.append("MRDS_LOADER_DB_TNS") + raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}") + + workflow_run_id = str(context['ti'].run_id) + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, workflow_run_id) + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Push context to XCom for downstream tasks + ti = context['ti'] + ti.xcom_push(key='workflow_history_key', value=a_workflow_history_key) + ti.xcom_push(key='workflow_context', value=workflow_context) + ti.xcom_push(key='env', value=env) + + logging.info("Workflow initialization completed successfully") + + except Exception as e: + logging.error(f"Error initializing workflow: {e}", exc_info=True) + raise + + def run_mrds_task(**context): + """Run MRDS processing task for EXDI""" + try: + ti = context['ti'] + task_id = context['task'].task_id + + # Extract task name from task_id + task_name = task_id.replace('_PARSE', '') if task_id.endswith('_PARSE') else task_id + + # Get task configuration + task_config = TASK_CONFIGS.get(task_name) + if not task_config: + raise ValueError(f"No configuration found for task: {task_name}") + + source_filename = task_config["source_filename"] + config_file = task_config["config_file"] + + # Get context from init task + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow') + + if not workflow_context: + raise ValueError("No workflow_context from init task") + + # Verify config file exists + if not os.path.exists(config_file): + raise FileNotFoundError(f"PARSE config file not found: {config_file}") + + logging.info(f"Processing EXDI MRDS with source_filename: {source_filename}, config_file: {config_file}") + + # Run MRDS - file should already be in bucket from EXDI flow + mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=False) + + logging.info(f"EXDI MRDS task completed successfully for {task_name}") + + # Push success status to XCom + ti.xcom_push(key='mrds_success', value=True) + ti.xcom_push(key='task_status', value='SUCCESS') + + return "SUCCESS" + + except Exception as e: + logging.error(f"Error running EXDI MRDS task: {e}", exc_info=True) + + # Push failure status to XCom + ti = context['ti'] + ti.xcom_push(key='mrds_success', value=False) + ti.xcom_push(key='task_status', value='FAILED') + ti.xcom_push(key='error_message', value=str(e)) + + raise + + def finalise_workflow_task(**context): + """Finalize workflow based on overall execution results""" + try: + ti = context['ti'] + dag_run = context['dag_run'] + + a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='init_workflow') + + if a_workflow_history_key is None: + raise ValueError("No workflow history key found in XCom; cannot finalise workflow") + + # Check all task statuses + workflow_success = True + failure_reasons = [] + + for task_name in TASK_CONFIGS.keys(): + mrds_task_id = f'{task_name}_PARSE' + mrds_task = dag_run.get_task_instance(mrds_task_id) + + if mrds_task.state == 'failed': + workflow_success = False + try: + error_msg = ti.xcom_pull(key='error_message', task_ids=mrds_task_id) + failure_reasons.append(f"{task_name}: MRDS task failed - {error_msg}") + except: + failure_reasons.append(f"{task_name}: MRDS task failed") + + # Finalize workflow + if workflow_success: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info(f"Finalised EXDI workflow with history key {a_workflow_history_key} as SUCCESS") + else: + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error(f"Finalised EXDI workflow with history key {a_workflow_history_key} as FAILED") + raise AirflowFailException(f"EXDI Workflow failed: {', '.join(failure_reasons)}") + + except AirflowFailException: + raise + except Exception as e: + logging.error(f"Error finalizing EXDI workflow: {e}", exc_info=True) + try: + if 'a_workflow_history_key' in locals() and a_workflow_history_key: + mrds_finalise_workflow(a_workflow_history_key, "N") + except: + pass + raise AirflowFailException(f"EXDI Workflow finalization failed: {e}") + + # Create tasks + init_workflow = PythonOperator( + task_id='init_workflow', + python_callable=init_workflow_task, + provide_context=True, + ) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Get task names - first task runs sequentially, others in parallel + task_names = list(TASK_CONFIGS.keys()) + first_task_name = task_names[0] + parallel_task_names = task_names[1:] + + # Create first task (sequential) + first_mrds_task = PythonOperator( + task_id=f'{first_task_name}_PARSE', + python_callable=run_mrds_task, + provide_context=True, + ) + + # Set dependencies for first task + init_workflow >> first_mrds_task >> finalize_workflow + + # Create parallel tasks + for task_name in parallel_task_names: + mrds_task = PythonOperator( + task_id=f'{task_name}_PARSE', + python_callable=run_mrds_task, + provide_context=True, + ) + # Parallel tasks start after first task completes + first_mrds_task >> mrds_task >> finalize_workflow + + logging.info(f"EXDI DAG created with {len(TASK_CONFIGS)} tasks: {list(TASK_CONFIGS.keys())}") diff --git a/airflow/TestDags/pen_test_demo.py b/airflow/TestDags/pen_test_demo.py new file mode 100644 index 0000000..5cf74b6 --- /dev/null +++ b/airflow/TestDags/pen_test_demo.py @@ -0,0 +1,44 @@ +from datetime import datetime +from airflow import DAG +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator + + +# RUN it if aiflow wokrs or not. + +def print_statement(): + print("Hello from the Python function!!!!!") + +def print_message(): + print("Last message from Python!, hope things are going good") + +with DAG( + 'demo_task_workflow', + start_date=datetime(2025, 6, 13), + schedule_interval=None, + catchup=False, +) as dag: + + task1 = BashOperator( + task_id='print_with_bash', + bash_command='echo "Lets begin"', + ) + + task2 = PythonOperator( + task_id='print_with_python', + python_callable=print_statement, + ) + + task3 = BashOperator( + task_id='another_bash_task', + bash_command='echo "So far so good!"', + ) + + task4 = PythonOperator( + task_id='another_python_task', + python_callable=print_message, + ) + + + task1 >> task2 >> task3 >> task4 + diff --git a/airflow/TestDags/rqsd_casper_connector_test.py b/airflow/TestDags/rqsd_casper_connector_test.py new file mode 100644 index 0000000..2cc688c --- /dev/null +++ b/airflow/TestDags/rqsd_casper_connector_test.py @@ -0,0 +1,95 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from datetime import datetime, timedelta +import logging + +from mrds.utils.manage_runs import init_workflow, finalise_workflow + +### CASPER CONNECTOR WITH DYNAMIC WORKFLOW CONTEXT & HISTORY KEY + + +# Importing custom module#s +sys.path.append('/opt/airflow/python/connectors/casper') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + +"""username = os.getenv("MRDS_LOADER_DB_USER") +password = os.getenv("MRDS_LOADER_DB_PASS") +tnsalias = os.getenv("MRDS_LOADER_DB_TNS") """ + +# connstr = f"{username}/{password}@{tnsalias}" + +# Importing the main function from casper script +from casper_rqsd import main as casper_main + +# Default DAG arguments +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 2, + 'retry_delay': timedelta(minutes=5), +} + +with DAG( + dag_id='rqsd_casper_connector_test', + default_args=default_args, + description='Run Casper RQSD data ingestion workflow', + schedule_interval=None, # we can set later + #start_date=datetime(2025, 10, 7), + catchup=False, + tags=['Casper', 'RQSD', 'Connector'], +) as dag: + + + def run_casper_rqsd(**context): + try: + #workflow_context = {"run_id": 34, "a_workflow_history_key": 6} + flow_config_path = "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/flow_config_casper.yaml" + env_config_path = "/opt/airflow/python/connectors/casper/config/env_config.yaml" + + + workflow_run_id = str(context['ti'].run_id) + + a_workflow_history_key = init_workflow(database_name, workflow_name, workflow_run_id) + logging.info(f"Initialized workflow with history key: {a_workflow_history_key}") + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Pick env from ENV variables + env = os.getenv("MRDS_ENV") + #env = os.getenv("MRDS_ENV", "lab") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + if not all([username, password, tnsalias]): + raise ValueError( + "Missing one or more required environment variables: " + "MRDS_LOADER_DB_USER, MRDS_LOADER_DB_PASS, MRDS_LOADER_DB_TNS" + ) + + logging.info( + f"Starting Casper RQSD workflow from Airflow DAG for env '{env}'" + ) + + #Calling main() + casper_main(workflow_context, flow_config_path, env_config_path, env) + + logging.info("Casper RQSD workflow completed successfully") + except Exception as e: + logging.error(f"Error running Casper RQSD workflow: {e}", exc_info=True) + raise + + run_casper = PythonOperator( + task_id='run_casper_rqsd', + python_callable=run_casper_rqsd, + provide_context=True, + ) \ No newline at end of file diff --git a/airflow/TestDags/t_MOPDB_RQSD_DEVO_OBSERVATIONS.py b/airflow/TestDags/t_MOPDB_RQSD_DEVO_OBSERVATIONS.py new file mode 100644 index 0000000..6b6baca --- /dev/null +++ b/airflow/TestDags/t_MOPDB_RQSD_DEVO_OBSERVATIONS.py @@ -0,0 +1,117 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from datetime import datetime, timedelta +import logging + +### DEVO CONNECTOR WITH DYNAMIC WORKFLOW CONTEXT & HISTORY KEY + +# Importing custom modules +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/connectors/devo') + +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + + +# Import your functions from manage_runs and devo_connector as before +from mrds.utils.manage_runs import init_workflow, finalise_workflow +from devo_connector import main as devo_main + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 3, + 'retry_delay': timedelta(minutes=5), +} + +# dynamic name extracton from basename +dag_id = os.path.splitext(os.path.basename(__file__))[0] + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='Run devo RQSD data ingestion workflow', + schedule_interval=None, + catchup=False, + tags=["Devo", "RQSD", "Connector"], +) as dag: + + def run_devo_connector_rqsd(**context): + try: + env = os.getenv("MRDS_ENV") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + if not all([username, password, tnsalias]): + raise ValueError( + "Missing one or more required environment variables: " + "MRDS_LOADER_DB_USER, MRDS_LOADER_DB_PASS, MRDS_LOADER_DB_TNS" + ) + + logging.info(f"Starting Casper RQSD workflow from Airflow DAG for env '{env}'") + print("======== THIS ... =========") + print("======== THIS ... =========") + database_name = 'MOPDB' + workflow_name = 'w_MOPDB_RQSD_PROCESS' + workflow_run_id = str(context['ti'].run_id) + + a_workflow_history_key = init_workflow(database_name, workflow_name, workflow_run_id) + logging.info(f"Initialized workflow with history key: {a_workflow_history_key}") + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + flow_config_path = "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/flow_config_rqsd_observations.yaml" + env_config_path = "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml" + + logging.info("Starting Devo RQSD workflow from Airflow DAG") + devo_main(workflow_context, flow_config_path, env_config_path, env) + logging.info("Devo RQSD workflow completed successfully") + + # Push the workflow history key to XCom for downstream tasks + context['ti'].xcom_push(key='workflow_history_key', value=a_workflow_history_key) + + except Exception as e: + logging.error(f"Error running Devo RQSD workflow: {e}", exc_info=True) + # If init_workflow succeeded but workflow failed, finalize with FAILED status + # Attempt to retrieve the key to finalize + if 'a_workflow_history_key' in locals(): + try: + finalise_workflow(a_workflow_history_key, "FAILED") + except Exception as finalise_error: + logging.error(f"Failed to finalise workflow after error: {finalise_error}") + raise + + def finalise_workflow_task(**context): + # Pull the workflow_history_key from XCom pushed by the main task + ti = context['ti'] + a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='run_devo_connector_rqsd') + + if a_workflow_history_key is None: + raise ValueError("No workflow history key found in XCom; cannot finalise workflow") + + # Call finalise with SUCCESS status + finalise_workflow(a_workflow_history_key, "SUCCESS") + logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS") + + run_devo = PythonOperator( + task_id='run_devo_connector_rqsd', + python_callable=run_devo_connector_rqsd, + provide_context=True, + ) + + finalize = PythonOperator( + task_id='finalise_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + ) + + run_devo >> finalize \ No newline at end of file diff --git a/airflow/devo_replicator/.gitkeep b/airflow/devo_replicator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/devo_replicator/devo_replicator_core.py b/airflow/devo_replicator/devo_replicator_core.py new file mode 100644 index 0000000..74a6e47 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_core.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import os +import sys +import logging +import yaml +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.python import PythonOperator + +try: + from airflow.exceptions import AirflowFailException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/python/devo_replicator/data_replicator') + +from mrds.utils import oraconn +from impala_refresher import main as impala_main + +from mrds.utils.security_utils import get_verified_run_id, verify_run_id + +ENV_CONFIG_PATH = "/opt/airflow/python/devo_replicator/config/env_config.yaml" + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_replicator_core', + default_args=default_args, + description='Core Devo replicator workflow for single table', + schedule=None, + catchup=False, + tags=['DevoReplicator'], + max_active_runs=10, + max_active_tasks=16, +) as dag: + + # Init - read config from context + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = conf.get("store") + if not store: + raise ValueError("store parameter is required") + store = store.lower() + + owner_table = conf.get("owner_table") + if not owner_table or '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst","acc","prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + if store not in {"mopdb", "rar", "rqsd"}: + raise ValueError(f"Unsupported store '{store}'. Expected 'mopdb', 'rar', 'rqsd'.") + + if store == "mopdb": + p_service_name = "MOPDB" + elif store == "rar": + p_service_name = "RAR" + elif store == "rqsd": + p_service_name = "RQSD" + + with open(ENV_CONFIG_PATH, "r") as f: + cfg = yaml.safe_load(f) + + env_cfg = cfg[env] + store_cfg = cfg[store] + p_objectstore_uri = env_cfg["S3_LOCATION_URI"].replace("{0}", store.lower()) + + # Get verified run_id using security utilities + p_run_id = get_verified_run_id(context) + + logging.info("=== init_step === env=%s store=%s table=%s.%s run_id=%s", + env, store, table_owner, table_name, p_run_id) + + xcom = { + "env": env, + "store": store, + "config_path": ENV_CONFIG_PATH, + "p_run_id": p_run_id, + "p_service_name": p_service_name, + "p_table_owner": table_owner, + "p_table_name": table_name, + "p_objectstore_uri": p_objectstore_uri, + "corporate_store": store_cfg["corporate_store"], + "owner_table": owner_table, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Start log table + def start_log_table_task(**context): + ti = context["ti"] + p_run_id = ti.xcom_pull(task_ids='init_step', key='p_run_id') + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.start_log_table', + [p_run_id, p_service_name, p_table_owner, p_table_name] + ) + oracle_conn.commit() + logging.info("start_log_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in start_log_table: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='start_log_table', + python_callable=start_log_table_task, + ) + + # Export table + def export_table_task(**context): + ti = context["ti"] + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + p_objectstore_uri = ti.xcom_pull(task_ids='init_step', key='p_objectstore_uri') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.export_table', + [p_service_name, p_table_owner, p_table_name, p_objectstore_uri] + ) + oracle_conn.commit() + logging.info("export_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in export_table: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='export_table', + python_callable=export_table_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Check if previous tasks succeeded before triggering child DAG + def check_previous_tasks_success(**context): + ti = context["ti"] + dag_run = context['dag_run'] + store = ti.xcom_pull(task_ids='init_step', key='store') + + check_tasks = ['start_log_table', 'export_table'] + failed = [] + + for tid in check_tasks: + ti_up = dag_run.get_task_instance(tid) + if ti_up and ti_up.state != 'success': + failed.append(f"{tid}:{ti_up.state}") + + if failed: + error_msg = f"Cannot proceed with {store} table generator. Previous tasks not successful: {', '.join(failed)}" + logging.error(error_msg) + raise AirflowFailException(error_msg) + + logging.info(f"All previous tasks succeeded. Ready to trigger {store} table generator.") + return True + + t3_check = PythonOperator( + task_id='check_previous_tasks_success', + python_callable=check_previous_tasks_success, + trigger_rule=TriggerRule.ALL_DONE, + ) + + def drop_table(**context): + ti = context["ti"] + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + p_objectstore_uri = ti.xcom_pull(task_ids='init_step', key='p_objectstore_uri') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.clear_s3_bucket', + [p_service_name, p_table_owner, p_table_name + '_COPY', p_objectstore_uri] + ) + oracle_conn.commit() + logging.info("clear_s3_bucket for table {0} procedure executed successfully".format(p_table_name[:-5].lower())) + except Exception as e: + logging.error(f"Error in clear_s3_bucket: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t3_drop = PythonOperator( + task_id='drop_table', + python_callable=drop_table, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Trigger table generator DAG based on store + def trigger_table_generator(**context): + from airflow.api.common.trigger_dag import trigger_dag + + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + # Determine target DAG based on store + if store == "mopdb": + target_dag_id = 'devo_table_generator_trigger_mopdb' + elif store == "rar": + target_dag_id = 'devo_table_generator_trigger_rar' + elif store == "rqsd": + target_dag_id = 'devo_table_generator_trigger_rqsd' + else: + raise ValueError(f"Unsupported store: {store}") + + # Add _COPY suffix to table name for the target table + owner_table_with_copy = f"{table_owner}.{table_name}_COPY" + + # Create configuration dictionary + trigger_conf = { + "owner_table": owner_table_with_copy + } + + logging.info(f"Triggering {target_dag_id} with conf: {trigger_conf}") + + try: + dag_run = trigger_dag( + dag_id=target_dag_id, + conf=trigger_conf, + execution_date=None, + replace_microseconds=False + ) + + logging.info(f"Successfully triggered {target_dag_id}, run_id: {dag_run.run_id}") + ti.xcom_push(key='triggered_dag_run_id', value=dag_run.run_id) + ti.xcom_push(key='triggered_dag_id', value=target_dag_id) + + return dag_run.run_id + except Exception as e: + logging.error(f"Error triggering {target_dag_id}: {e}") + raise + + t3_trigger = PythonOperator( + task_id='trigger_table_generator', + python_callable=trigger_table_generator, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # End log table + def end_log_table_task(**context): + ti = context["ti"] + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.end_log_table', + [p_service_name, p_table_owner, p_table_name] + ) + oracle_conn.commit() + logging.info("end_log_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in end_log_table: {e}") + logging.info("Continuing despite end_log_table error (cleanup task)") + finally: + if oracle_conn: + oracle_conn.close() + + t4 = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Check status and fail if needed + def fail_if_any_failed(**context): + dag_run = context['dag_run'] + check_tasks = ['start_log_table', 'export_table', 'check_previous_tasks_success', 'trigger_table_generator'] + failed = [] + + for tid in check_tasks: + ti_up = dag_run.get_task_instance(tid) + if ti_up and ti_up.state == 'failed': + failed.append(tid) + + if failed: + error_msg = f"Critical task(s) failed: {', '.join(failed)}. DAG execution failed." + logging.error(error_msg) + raise AirflowFailException(error_msg) + + logging.info("All critical tasks completed successfully: %s", check_tasks) + + t5 = PythonOperator( + task_id='fail_if_any_failed', + python_callable=fail_if_any_failed, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Dependencies + init >> t1 >> t2 >> t3_check >> t3_drop >> t3_trigger + [t1, t2, t3_trigger] >> t4 + t4 >> t5 diff --git a/airflow/devo_replicator/devo_replicator_core_pandas.py b/airflow/devo_replicator/devo_replicator_core_pandas.py new file mode 100644 index 0000000..3703656 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_core_pandas.py @@ -0,0 +1,239 @@ +from airflow import DAG +from airflow.operators.python_operator import PythonOperator +from airflow.hooks.S3_hook import S3Hook +from datetime import datetime, timedelta +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import io +import os +import logging +import sys + +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + + +SERVICE_NAME = "SERVICE_NAME" +OWNER = "C2D" +TABLE_NAME = "T_CEPH" +METADATA_OWNER = "CT_MOPDB" +METADATA_TABLE = "mopdb_metadata_inventory" + +USE_LOCAL_STORAGE = True +LOCAL_OUTPUT_DIR = "/tmp/devo_replicator_output" + +S3_BUCKET = "bucket-name" +S3_PREFIX = "devo/replicator/C2D/T_CEPH/" +AWS_CONN_ID = "aws_default" + +DEFAULT_ARGS = { + "owner": "airflow", + "depends_on_past": False, + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=5), +} +DAG_ID = "devo_replicator_pandas" +SCHEDULE_INTERVAL = None +CHUNK_SIZE = 100000 + + +def query_oracle_template(owner, table_name): + try: + input_query = """SELECT + COLUMN_NAME, + DATA_TYPE, + CHAR_LENGTH, + DATA_PRECISION, + DATA_SCALE, + COLUMN_ID, + CASE + WHEN DATA_TYPE = 'DATE' OR DATA_TYPE LIKE '%TIMESTAMP%' THEN + 'CAST(' || COLUMN_NAME || ' AS VARCHAR2(100)) AS ' || COLUMN_NAME + WHEN DATA_TYPE = 'VARCHAR2' OR DATA_TYPE LIKE '%CHAR%' THEN + 'CAST(' || COLUMN_NAME || ' AS VARCHAR2(' || CAST(CHAR_LENGTH AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE IN ('NUMBER', 'DECIMAL') AND DATA_PRECISION IS NOT NULL AND DATA_SCALE IS NOT NULL THEN + 'CAST(' || COLUMN_NAME || ' AS ' || DATA_TYPE || '(' || CAST(DATA_PRECISION AS INT) || ',' || CAST(DATA_SCALE AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE IN ('NUMBER', 'DECIMAL') AND DATA_PRECISION IS NOT NULL THEN + 'CAST(' || COLUMN_NAME || ' AS ' || DATA_TYPE || '(' || CAST(DATA_PRECISION AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE = 'CLOB' THEN + 'TO_CHAR(SUBSTR(' || COLUMN_NAME || ', 1, 32767)) AS ' || COLUMN_NAME + ELSE + COLUMN_NAME + END AS casting + FROM {0}.{1} + WHERE OWNER = '{2}' AND TABLE_NAME = '{3}' AND A_VALID_TO > SYSDATE + ORDER BY COLUMN_ID""".format(METADATA_OWNER, METADATA_TABLE, owner, table_name) + + conn = connect('MRDS_LOADER') + df = pd.read_sql(input_query, conn) + + if df.empty: + raise ValueError(f"No metadata found for {owner}.{table_name}") + + output_query = 'SELECT ' + ', \n'.join(df['casting'].tolist()) + ' FROM {0}.{1}'.format(owner, table_name) + + column_metadata = {} + for _, row in df.iterrows(): + col_name = row['COLUMN_NAME'] + data_type = row['DATA_TYPE'] + + if data_type in ('NUMBER', 'DECIMAL', 'FLOAT', 'BINARY_FLOAT', 'BINARY_DOUBLE'): + if pd.notna(row['DATA_SCALE']) and row['DATA_SCALE'] > 0: + column_metadata[col_name] = 'float64' + elif pd.notna(row['DATA_PRECISION']) and row['DATA_PRECISION'] <= 9: + column_metadata[col_name] = 'Int32' + elif pd.notna(row['DATA_PRECISION']) and row['DATA_PRECISION'] <= 18: + column_metadata[col_name] = 'Int64' + else: + column_metadata[col_name] = 'float64' + elif data_type == 'DATE' or 'TIMESTAMP' in data_type: + column_metadata[col_name] = 'string' + else: + column_metadata[col_name] = 'string' + + logging.info(f"Generated query template with {len(df)} columns") + return output_query, column_metadata + + except Exception as e: + logging.error(f"Error in query_oracle_template: {e}") + raise + finally: + if conn: + conn.close() + + +def query_oracle_and_generate_parquet(partition_num, partitions, sql, column_metadata, use_local): + logging.info(f"[Pandas-Partition {partition_num}] Starting processing (Mode: {'LOCAL' if use_local else 'S3'})") + + partition_sql = f"""SELECT /*+ PARALLEL(t, {partitions}) */ * +FROM ( +{sql} +) t +WHERE ORA_HASH(ROWID, {partitions - 1}) = {partition_num}""" + + conn = connect("MRDS_LOADER") + + if use_local: + os.makedirs(LOCAL_OUTPUT_DIR, exist_ok=True) + output_path = os.path.join(LOCAL_OUTPUT_DIR, f"partition_{partition_num:04d}.parquet") + file_handle = open(output_path, 'wb') + write_target = file_handle + else: + s3_hook = S3Hook(aws_conn_id=AWS_CONN_ID) + s3_client = s3_hook.get_conn() + s3_key = f"{S3_PREFIX}partition_{partition_num:04d}.parquet" + buffer = io.BytesIO() + write_target = buffer + + try: + chunk_iterator = pd.read_sql(partition_sql, conn, chunksize=CHUNK_SIZE) + + pqwriter = None + total_rows = 0 + chunk_count = 0 + + for chunk in chunk_iterator: + for col, dtype in column_metadata.items(): + if col in chunk.columns: + try: + if dtype == 'string': + chunk[col] = chunk[col].astype('string') + elif dtype.startswith('Int'): + chunk[col] = pd.to_numeric(chunk[col], errors='coerce').astype(dtype) + elif dtype == 'float64': + chunk[col] = pd.to_numeric(chunk[col], errors='coerce') + except Exception as e: + logging.warning(f"[Pandas-Partition {partition_num}] Could not optimize column {col}: {e}") + + table = pa.Table.from_pandas(chunk, preserve_index=False) + + if pqwriter is None: + pqwriter = pq.ParquetWriter( + write_target, + table.schema, + compression='snappy', + use_dictionary=True, + write_statistics=True, + version='2.6', + data_page_size=1024*1024, + ) + + pqwriter.write_table(table) + total_rows += len(chunk) + chunk_count += 1 + + if chunk_count % 10 == 0: + if use_local: + file_size_mb = os.path.getsize(output_path) / 1024 / 1024 + else: + file_size_mb = write_target.tell() / 1024 / 1024 + logging.info(f"[Pandas-Partition {partition_num}] Processed {total_rows:,} rows, Size: {file_size_mb:.2f} MB") + + if pqwriter: + pqwriter.close() + + if use_local: + file_size_mb = os.path.getsize(output_path) / 1024 / 1024 + logging.info(f"[Pandas-Partition {partition_num}] Completed - {total_rows:,} rows, {file_size_mb:.2f} MB saved to {output_path}") + else: + write_target.seek(0) + buffer_size_mb = write_target.getbuffer().nbytes / 1024 / 1024 + + logging.info(f"[Pandas-Partition {partition_num}] Uploading {buffer_size_mb:.2f} MB to s3://{S3_BUCKET}/{s3_key}") + + s3_client.upload_fileobj(write_target, S3_BUCKET, s3_key) + + logging.info(f"[Pandas-Partition {partition_num}] Completed - {total_rows:,} rows, {buffer_size_mb:.2f} MB uploaded to S3") + + except Exception as e: + logging.error(f"[Pandas-Partition {partition_num}] Error: {e}") + raise + finally: + conn.close() + if use_local: + file_handle.close() + else: + write_target.close() + + +def generate_tasks(dag, partitions, sql, column_metadata, use_local): + tasks = [] + for partition_num in range(partitions): + task = PythonOperator( + task_id=f"generate_parquet_partition_{partition_num}", + python_callable=query_oracle_and_generate_parquet, + op_kwargs={ + "partition_num": partition_num, + "partitions": partitions, + "sql": sql, + "column_metadata": column_metadata, + "use_local": use_local + }, + provide_context=True, + dag=dag, + ) + tasks.append(task) + return tasks + + +with DAG( + dag_id='devo_replicator_pandas', + default_args=DEFAULT_ARGS, + description='Devo replicator using Pandas with dtype optimization', + schedule_interval=SCHEDULE_INTERVAL, + start_date=datetime(2024, 1, 1), + catchup=False, + tags=['DevoReplicator', 'Pandas'], + max_active_runs=1, + max_active_tasks=30, +) as dag: + + query, column_metadata = query_oracle_template(OWNER, TABLE_NAME) + + PARTITIONS = 16 + + partition_tasks = generate_tasks(dag, PARTITIONS, query, column_metadata, USE_LOCAL_STORAGE) diff --git a/airflow/devo_replicator/devo_replicator_core_pyarrow.py b/airflow/devo_replicator/devo_replicator_core_pyarrow.py new file mode 100644 index 0000000..c9cfdff --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_core_pyarrow.py @@ -0,0 +1,244 @@ +from airflow import DAG +from airflow.operators.python_operator import PythonOperator +from airflow.hooks.S3_hook import S3Hook +from datetime import datetime, timedelta +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import io +import logging +import sys + + +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +SERVICE_NAME = "SERVICE_NAME" +OWNER = "C2D" +TABLE_NAME = "T_CEPH" +METADATA_OWNER = "CT_MOPDB" +METADATA_TABLE = "mopdb_metadata_inventory" + +## need to be changed +S3_BUCKET = "bucket-name" +S3_PREFIX = "devo/replicator/C2D/T_CEPH/" +AWS_CONN_ID = "aws_default" + +DEFAULT_ARGS = { + "owner": "airflow", + "depends_on_past": False, + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=5), +} +DAG_ID = "devo_replicator_pyarrow" +SCHEDULE_INTERVAL = None +BATCH_SIZE = 100000 + + +def query_oracle_template(owner, table_name): + try: + input_query = """SELECT + CASE + WHEN DATA_TYPE = 'DATE' OR DATA_TYPE LIKE '%TIMESTAMP%' THEN + 'CAST(' || COLUMN_NAME || ' AS VARCHAR2(100)) AS ' || COLUMN_NAME + WHEN DATA_TYPE = 'VARCHAR2' OR DATA_TYPE LIKE '%CHAR%' THEN + 'CAST(' || COLUMN_NAME || ' AS VARCHAR2(' || CAST(CHAR_LENGTH AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE IN ('NUMBER', 'DECIMAL') AND DATA_PRECISION IS NOT NULL AND DATA_SCALE IS NOT NULL THEN + 'CAST(' || COLUMN_NAME || ' AS ' || DATA_TYPE || '(' || CAST(DATA_PRECISION AS INT) || ',' || CAST(DATA_SCALE AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE IN ('NUMBER', 'DECIMAL') AND DATA_PRECISION IS NOT NULL THEN + 'CAST(' || COLUMN_NAME || ' AS ' || DATA_TYPE || '(' || CAST(DATA_PRECISION AS INT) || ')) AS ' || COLUMN_NAME + WHEN DATA_TYPE = 'CLOB' THEN + 'TO_CHAR(SUBSTR(' || COLUMN_NAME || ', 1, 32767)) AS ' || COLUMN_NAME + ELSE + COLUMN_NAME + END AS casting + FROM {0}.{1} + WHERE OWNER = '{2}' AND TABLE_NAME = '{3}' AND A_VALID_TO > SYSDATE + ORDER BY COLUMN_ID""".format(METADATA_OWNER, METADATA_TABLE, owner, table_name) + + conn = connect('MRDS_LOADER') + df = pd.read_sql(input_query, conn) + + if df.empty: + raise ValueError(f"No metadata found for {owner}.{table_name}") + + output_query = 'SELECT ' + ', \n'.join(df['casting'].tolist()) + ' FROM {0}.{1}'.format(owner, table_name) + + logging.info(f"Generated query template with {len(df)} columns") + return output_query + + except Exception as e: + logging.error(f"Error in query_oracle_template: {e}") + raise + finally: + if conn: + conn.close() + + +def query_oracle_and_generate_parquet(partition_num, partitions, sql): + logging.info(f"[PyArrow-Partition {partition_num}] Starting processing") + + partition_sql = f"""SELECT /*+ PARALLEL(t, {partitions}) */ * +FROM ( +{sql} +) t +WHERE ORA_HASH(ROWID, {partitions - 1}) = {partition_num}""" + + conn = connect("MRDS_LOADER") + cursor = conn.cursor() + cursor.arraysize = BATCH_SIZE + cursor.prefetchrows = BATCH_SIZE + + s3_hook = S3Hook(aws_conn_id=AWS_CONN_ID) + s3_client = s3_hook.get_conn() + s3_key = f"{S3_PREFIX}partition_{partition_num:04d}.parquet" + + buffer = io.BytesIO() + + try: + cursor.execute(partition_sql) + + column_names = [desc[0] for desc in cursor.description] + + first_batch_rows = cursor.fetchmany(BATCH_SIZE) + if not first_batch_rows: + logging.warning(f"[PyArrow-Partition {partition_num}] No data found") + return + + arrow_fields = [] + sample_row = first_batch_rows[0] + + for i, col_name in enumerate(column_names): + sample_val = sample_row[i] + + if sample_val is None: + for row in first_batch_rows[1:]: + if row[i] is not None: + sample_val = row[i] + break + + if isinstance(sample_val, str): + arrow_type = pa.string() + elif isinstance(sample_val, int): + arrow_type = pa.int64() + elif isinstance(sample_val, float): + arrow_type = pa.float64() + elif isinstance(sample_val, (datetime, pd.Timestamp)): + arrow_type = pa.timestamp('ns') + elif isinstance(sample_val, bytes): + arrow_type = pa.binary() + else: + arrow_type = pa.string() + + arrow_fields.append(pa.field(col_name, arrow_type)) + + schema = pa.schema(arrow_fields) + + writer = pq.ParquetWriter( + buffer, + schema, + compression='snappy', + use_dictionary=True, + write_statistics=True, + data_page_size=2*1024*1024, + version='2.6', + ) + + def process_batch(rows): + if not rows: + return None + + columns_data = list(zip(*rows)) + arrays = [] + + for i, col_data in enumerate(columns_data): + try: + arrays.append(pa.array(col_data, type=schema.field(i).type)) + except Exception as e: + logging.warning(f"[PyArrow-Partition {partition_num}] Column {column_names[i]} conversion failed: {e}") + converted = [str(val) if val is not None else None for val in col_data] + arrays.append(pa.array(converted, type=pa.string())) + + return pa.RecordBatch.from_arrays(arrays, schema=schema) + + batch = process_batch(first_batch_rows) + if batch: + writer.write_batch(batch) + + total_rows = len(first_batch_rows) + batch_count = 1 + + while True: + rows = cursor.fetchmany(BATCH_SIZE) + if not rows: + break + + batch = process_batch(rows) + if batch: + writer.write_batch(batch) + + total_rows += len(rows) + batch_count += 1 + + if batch_count % 10 == 0: + buffer_size_mb = buffer.tell() / 1024 / 1024 + logging.info(f"[PyArrow-Partition {partition_num}] Processed {total_rows:,} rows, Buffer size: {buffer_size_mb:.2f} MB") + + writer.close() + + buffer.seek(0) + buffer_size_mb = buffer.getbuffer().nbytes / 1024 / 1024 + + logging.info(f"[PyArrow-Partition {partition_num}] Uploading {buffer_size_mb:.2f} MB to s3://{S3_BUCKET}/{s3_key}") + + s3_client.upload_fileobj(buffer, S3_BUCKET, s3_key) + + logging.info(f"[PyArrow-Partition {partition_num}] Completed - {total_rows:,} rows, {buffer_size_mb:.2f} MB uploaded to S3") + + except Exception as e: + logging.error(f"[PyArrow-Partition {partition_num}] Error: {e}") + raise + finally: + cursor.close() + conn.close() + buffer.close() + + +def generate_tasks(dag, partitions, sql): + tasks = [] + for partition_num in range(partitions): + task = PythonOperator( + task_id=f"generate_parquet_partition_{partition_num}", + python_callable=query_oracle_and_generate_parquet, + op_kwargs={ + "partition_num": partition_num, + "partitions": partitions, + "sql": sql + }, + provide_context=True, + dag=dag, + ) + tasks.append(task) + return tasks + + +with DAG( + dag_id='devo_replicator_pyarrow', + default_args=DEFAULT_ARGS, + description='Devo replicator using PyArrow native processing', + schedule_interval=SCHEDULE_INTERVAL, + start_date=datetime(2024, 1, 1), + catchup=False, + tags=['DevoReplicator', 'PyArrow'], + max_active_runs=1, + max_active_tasks=16, +) as dag: + + query = query_oracle_template(OWNER, TABLE_NAME) + + PARTITIONS = 16 + + partition_tasks = generate_tasks(dag, PARTITIONS, query) diff --git a/airflow/devo_replicator/devo_replicator_scheduler_rar.py b/airflow/devo_replicator/devo_replicator_scheduler_rar.py new file mode 100644 index 0000000..67239f3 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_scheduler_rar.py @@ -0,0 +1,181 @@ +# dags/dev_replicator_scheduler_rar.py +from __future__ import annotations + +import sys +import logging +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.decorators import task +from airflow.operators.python import BranchPythonOperator +from airflow.operators.empty import EmptyOperator +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from datetime import datetime, timedelta + +from mrds.utils import oraconn # your Oracle connection helper + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +DAG_NAME = "devo_replicator_scheduler_rar" +TARGET_DAG_ID = "devo_replicator_trigger" +ORACLE_CONN_NAME = "MRDS_LOADER" + +PRECONDITION_SQL = """ +WITH LAST_UPDATE_ORACLE AS ( + SELECT MAX(process_end) AS process_end + FROM CT_RAR.A_RAR_FOR_DISC_MONITORING + WHERE UPPER(owner || '.' || target_table_name) = UPPER(:table_name) + AND process_end IS NOT NULL + AND process_successful = 'Y' +), +LAST_UPDATE_DEVO AS ( + SELECT CASE + WHEN last_status = 'FINISHED' THEN last_end_time + ELSE TO_DATE('01-JAN-1999', 'DD-MON-YYYY') + END AS process_end + FROM CT_MRDS.a_devo_replica_mgmt_rar + WHERE owner || '.' || table_name = :table_name +) +SELECT CASE + WHEN (SELECT process_end FROM LAST_UPDATE_ORACLE) > + (SELECT process_end FROM LAST_UPDATE_DEVO) + THEN 'Y' ELSE 'N' + END AS trigger_devo_replicator +FROM dual +""" + +def _get_conn(): + return oraconn.connect(ORACLE_CONN_NAME) + +def get_devo_replica_table_options() -> list[str]: + conn = None + cur = None + try: + conn = _get_conn() + cur = conn.cursor() + cur.execute(""" + SELECT OWNER || '.' || TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_rar + ORDER BY OWNER, TABLE_NAME + """) + rows = cur.fetchall() + tables = [r[0] for r in rows] if rows else [] + logging.info("Fetched %d table(s) from replica mgmt.", len(tables)) + return tables + except Exception: + logging.exception("Error getting DEVO replica table options") + return [] + finally: + try: + if cur: cur.close() + except Exception: + pass + if conn: + conn.close() + +def check_table_precondition(table_full_name: str) -> dict: + """Returns {"table": , "trigger": "Y"|"N"}.""" + conn = None + cur = None + try: + conn = _get_conn() + cur = conn.cursor() + cur.execute(PRECONDITION_SQL, {"table_name": table_full_name}) + row = cur.fetchone() + status = (row[0] if row else 'N') or 'N' + logging.info("Precondition for %s: %s", table_full_name, status) + return {"table": table_full_name, "trigger": status} + except Exception: + logging.exception("Error checking precondition for %s", table_full_name) + return {"table": table_full_name, "trigger": "N"} # fail closed + finally: + try: + if cur: cur.close() + except Exception: + pass + if conn: + conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id=DAG_NAME, + description="Checks DEVO replica preconditions; triggers devo_replicator_trigger_rar once if any table is stale", + default_args=default_args, + start_date=datetime.now() - timedelta(days=2), + schedule=None, + catchup=False, + schedule_interval='*/10 * * * *', # every 10 minutes + max_active_runs=1, + tags=["DevoScheduler", "DevoReplicatorTrigger"], +) as dag: + + @task + def fetch_tables() -> list[str]: + tables = get_devo_replica_table_options() + if not tables: + logging.warning("No tables returned from enumeration.") + return tables + + @task + def check_one(table_name: str) -> dict: + return check_table_precondition(table_name) + + @task + def summarize(results: list[dict]) -> dict: + y_tables = [r["table"] for r in results if r and r.get("trigger") == "Y"] + n_tables = [r["table"] for r in results if r and r.get("trigger") == "N"] + + logging.info("Precondition summary -> Y: %d, N: %d", len(y_tables), len(n_tables)) + if y_tables: + logging.info("Tables needing replication: %s", ", ".join(y_tables)) + else: + logging.info("No tables are updated/stale; nothing to trigger.") + + return {"any_true": bool(y_tables), "y_tables": y_tables} + + def decide_branch(summary: dict) -> str: + """Return the EXACT downstream task_id to follow.""" + return "prepare_trigger_conf" if summary.get("any_true") else "no_updates" + + @task + def prepare_trigger_conf(summary: dict) -> dict: + """Single conf payload for the downstream DAG.""" + return {"tables_to_replicate": summary.get("y_tables", [])} + + no_updates = EmptyOperator(task_id="no_updates") + + # Graph + tables = fetch_tables() + results = check_one.expand(table_name=tables) # dynamic mapping across tables + summary = summarize(results) + + branch = BranchPythonOperator( + task_id="branch_on_any", + python_callable=decide_branch, + op_args=[summary], # XComArg from summarize + ) + + + conf_payload = prepare_trigger_conf(summary) + + trigger_devo = TriggerDagRunOperator( + task_id="trigger_devo_replicator_rar", + trigger_dag_id=TARGET_DAG_ID, + wait_for_completion=True, + reset_dag_run=True, + conf=conf_payload, + ) + + # Wire branching — only ONE instance of prepare_trigger_conf is referenced + summary >> branch + branch >> no_updates + branch >> conf_payload >> trigger_devo diff --git a/airflow/devo_replicator/devo_replicator_trigger.py b/airflow/devo_replicator/devo_replicator_trigger.py new file mode 100644 index 0000000..e230eff --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_trigger.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import os +import sys +import logging +import time +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param +from airflow.decorators import task +from airflow.providers.oracle.hooks.oracle import OracleHook + +from mrds.utils import oraconn + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +ORACLE_CONN_ID = "MRDS_LOADER" +# TARGET_DAG_ID = "devo_replicator_trigger_rar" + +def get_rar_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute(""" + SELECT OWNER || '.' || TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_rar + ORDER BY OWNER, TABLE_NAME + """) + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting RAR table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_replicator_trigger', + default_args=default_args, + description='External trigger DAG for RAR tables', + schedule=None, + catchup=False, + tags=['DevoReplicator', 'DevoReplicatorTrigger'], + max_active_runs=1, + params={ + # still allow manual runs from the UI + "owner_table": Param( + default=None, + type=["string", "null"], + description="Select table in format OWNER.TABLE_NAME", + #enum=get_rar_table_options() + ) + } +) as dag: + + # --- Init: read conf --- + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "rar" + owner_table = conf.get("owner_table") # optional single table + tables_to_replicate = conf.get("tables_to_replicate") # optional list of OWNER.TABLE + + # Log what we got + if tables_to_replicate: + logging.info("Received tables_to_replicate from upstream: %d table(s).", len(tables_to_replicate)) + elif owner_table: + logging.info("Received single owner_table from conf: %s", owner_table) + else: + logging.info("No conf provided; manual UI param may be used or fallback to full list in get_table_list.") + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + xcom = { + "env": env, + "store": store, + "owner_table": owner_table, # may be None + "tables_to_replicate": tables_to_replicate # may be None/list + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # --- Build the processing list --- + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + tables_to_replicate = ti.xcom_pull(task_ids='init_step', key='tables_to_replicate') + + # 1) If upstream provided a list, use it + if tables_to_replicate: + logging.info("Using tables_to_replicate list from conf: %d items", len(tables_to_replicate)) + tables = [] + for ot in tables_to_replicate: + if '.' not in ot: + logging.warning("Skipping malformed owner_table (no dot): %s", ot) + continue + table_owner, table_name = ot.split('.', 1) + tables.append((table_owner, table_name)) + ti.xcom_push(key='tables_to_process', value=tables) + return tables + + # 2) Else if a single owner_table provided (manual/programmatic) + if owner_table: + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing single table from conf/params: %s", owner_table) + ti.xcom_push(key='tables_to_process', value=tables) + return tables + + # 3) Else fallback to full list in DB (manual run without conf) + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute(""" + SELECT OWNER, TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_rar + ORDER BY OWNER, TABLE_NAME + """) + tables = cursor.fetchall() + cursor.close() + logging.info("Fallback: Found %d tables for RAR", len(tables)) + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # --- Keep your existing throttled triggering logic unchanged --- + def check_and_trigger(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + threshold = 30 # you were pushing 30; keep it here or push from init + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + while True: + cursor = oracle_conn.cursor() + service_name = store.upper() + sql_query = f""" + SELECT + (SELECT NVL(SUM(MAX_THREADS),0) FROM CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB WHERE LAST_STATUS = 'RUNNING') + + (SELECT NVL(SUM(MAX_THREADS),0) FROM CT_MRDS.A_DEVO_REPLICA_MGMT_RAR WHERE LAST_STATUS = 'RUNNING') + AS TOTAL_RUNNING_THREADS_NOW, + (SELECT COUNT(*) + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' AND TABLE_NAME = '{table_name}' AND LAST_STATUS = 'RUNNING') AS TABLE_IS_ALREADY_RUNNING + FROM DUAL + """ + cursor.execute(sql_query) + total_running_val, table_running_val = cursor.fetchone() + cursor.close() + + logging.info( + "Total running: %d, threshold: %d, table running: %d", + total_running_val or 0, threshold, table_running_val or 0 + ) + + if (total_running_val or 0) > threshold: + logging.info("Threshold exceeded. Waiting 5 minutes...") + time.sleep(300) + continue + + if (table_running_val or 0) >= 1: + logging.info("Table %s.%s already running. Skipping.", table_owner, table_name) + break + + # Trigger the core DAG for this specific table + from airflow.api.common.trigger_dag import trigger_dag + conf = {"store": store, "owner_table": f"{table_owner}.{table_name}"} + trigger_dag( + dag_id='devo_replicator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + break + + logging.info("Total core DAGs triggered: %d", triggered_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + + except Exception as e: + logging.error(f"Error in check_and_trigger: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='check_and_trigger', + python_callable=check_and_trigger, + ) + + init >> t1 >> t2 + + +""" +Reading tables_to_replicate from dag_run.conf in init_step. +Pushing it to XCom (so get_table_list can use it). +Tell get_table_list to prioritize the provided list. +init_step reads tables_to_replicate from dag_run.conf and puts it into XCom. +get_table_list prioritizes that list; falls back to owner_table or full table list only if needed. +check_and_trigger loops over those tables and triggers your core DAG (devo_replicator_core) per table, respecting your concurrency threshold. +""" \ No newline at end of file diff --git a/airflow/devo_replicator/devo_replicator_trigger_mopdb.py b/airflow/devo_replicator/devo_replicator_trigger_mopdb.py new file mode 100644 index 0000000..ca1c931 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_trigger_mopdb.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import os +import sys +import logging +import time +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get MOPDB table options for dropdown +def get_mopdb_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_mopdb ORDER BY OWNER, TABLE_NAME") + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting MOPDB table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_replicator_trigger_mopdb', + default_args=default_args, + description='External trigger DAG for MOPDB tables', + schedule=None, + catchup=False, + tags=['DevoReplicator', 'DevoReplicatorTrigger'], + params={ + "owner_table": Param( + default=None, + type="string", + description="Select table in format OWNER.TABLE_NAME", + enum=get_mopdb_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "mopdb" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + if '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s", + env, store, owner_table) + + xcom = { + "env": env, + "store": store, + "table_owner": table_owner, + "table_name": table_name, + "owner_table": owner_table, + "threshold": 30, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if owner_table: + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_mopdb ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for MOPDB", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Check and trigger core DAG + def check_and_trigger(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + threshold = ti.xcom_pull(task_ids='init_step', key='threshold') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + while True: + cursor = oracle_conn.cursor() + + # Execute SQL query with variable substitution + service_name = store.upper() + sql_query = f""" + SELECT (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB + WHERE LAST_STATUS = 'RUNNING') + + (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_RAR + WHERE LAST_STATUS = 'RUNNING') + AS TOTAL_RUNNING_THREADS_NOW, + (SELECT COUNT(*) FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' AND TABLE_NAME = '{table_name}' AND LAST_STATUS = 'RUNNING') AS TABLE_IS_ALREADY_RUNNING + FROM DUAL + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + total_running_val = result[0] or 0 + table_running_val = result[1] or 0 + cursor.close() + + logging.info("Total running: %d, threshold: %d, table running: %d", + total_running_val, threshold, table_running_val) + + if total_running_val > threshold: + logging.info("Threshold exceeded. Waiting 5 minutes...") + time.sleep(300) + continue + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + break + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_replicator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + break + + logging.info("Total DAGs triggered: %d", triggered_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + + except Exception as e: + logging.error(f"Error in check_and_trigger: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='check_and_trigger', + python_callable=check_and_trigger, + ) + + # Dependencies + init >> t1 >> t2 + +""" +MOPDB Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME) +- Sets store to "mopdb" (fixed for this DAG) +- Sets threshold to 30 (max concurrent running threads) +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_mopdb +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) check_and_trigger +- Loops through each table from the table list +- For each table, enters monitoring loop: + - Executes SQL query to check total running threads across MOPDB+RAR + - Checks if current table is already running + - If total threads > threshold (30): waits 5 minutes and rechecks + - If table already running: skips to next tabl + - If conditions met: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered +- Ensures system doesn't exceed concurrent processing limits +""" \ No newline at end of file diff --git a/airflow/devo_replicator/devo_replicator_trigger_rar.py b/airflow/devo_replicator/devo_replicator_trigger_rar.py new file mode 100644 index 0000000..86f3a83 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_trigger_rar.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import os +import sys +import logging +import time +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get MOPDB table options for dropdown +def get_rar_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rar ORDER BY OWNER, TABLE_NAME") + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting RAR table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_replicator_trigger_rar', + default_args=default_args, + description='External trigger DAG for RAR tables', + schedule=None, + catchup=False, + tags=['DevoReplicator', 'DevoReplicatorTrigger'], + params={ + "owner_table": Param( + default=None, + type="string", + description="Select table in format OWNER.TABLE_NAME", + enum=get_rar_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "rar" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + if '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s", + env, store, owner_table) + + xcom = { + "env": env, + "store": store, + "table_owner": table_owner, + "table_name": table_name, + "owner_table": owner_table, + "threshold": 30, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if owner_table: + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rar ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for RAR", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Check and trigger core DAG + def check_and_trigger(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + threshold = ti.xcom_pull(task_ids='init_step', key='threshold') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + while True: + cursor = oracle_conn.cursor() + + # Execute SQL query with variable substitution + service_name = store.upper() + sql_query = f""" + SELECT (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB + WHERE LAST_STATUS = 'RUNNING') + + (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_RAR + WHERE LAST_STATUS = 'RUNNING') + AS TOTAL_RUNNING_THREADS_NOW, + (SELECT COUNT(*) FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' AND TABLE_NAME = '{table_name}' AND LAST_STATUS = 'RUNNING') AS TABLE_IS_ALREADY_RUNNING + FROM DUAL + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + total_running_val = result[0] or 0 + table_running_val = result[1] or 0 + cursor.close() + + logging.info("Total running: %d, threshold: %d, table running: %d", + total_running_val, threshold, table_running_val) + + if total_running_val > threshold: + logging.info("Threshold exceeded. Waiting 5 minutes...") + time.sleep(300) + continue + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + break + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_replicator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + break + + logging.info("Total DAGs triggered: %d", triggered_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + + except Exception as e: + logging.error(f"Error in check_and_trigger: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='check_and_trigger', + python_callable=check_and_trigger, + ) + + # Dependencies + init >> t1 >> t2 + +""" +RAR Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME) +- Sets store to "rar" (fixed for this DAG) +- Sets threshold to 30 (max concurrent running threads) +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_rar +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) check_and_trigger +- Loops through each table from the table list +- For each table, enters monitoring loop: + - Executes SQL query to check total running threads across MOPDB+RAR + - Checks if current table is already running + - If total threads > threshold (30): waits 5 minutes and rechecks + - If table already running: skips to next tabl + - If conditions met: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered +- Ensures system doesn't exceed concurrent processing limits +""" diff --git a/airflow/devo_replicator/devo_replicator_trigger_rqsd.py b/airflow/devo_replicator/devo_replicator_trigger_rqsd.py new file mode 100644 index 0000000..1b45329 --- /dev/null +++ b/airflow/devo_replicator/devo_replicator_trigger_rqsd.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import os +import sys +import logging +import time +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get RQSD table options for dropdown +def get_rqsd_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rqsd ORDER BY OWNER, TABLE_NAME") + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting RQSD table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_replicator_trigger_rqsd', + default_args=default_args, + description='External trigger DAG for RQSD tables', + schedule=None, + catchup=False, + tags=['DevoReplicator', 'DevoReplicatorTrigger'], + params={ + "owner_table": Param( + default=None, + type="string", + description="Select table in format OWNER.TABLE_NAME", + enum=get_rqsd_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "rqsd" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + if '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s", + env, store, owner_table) + + xcom = { + "env": env, + "store": store, + "table_owner": table_owner, + "table_name": table_name, + "owner_table": owner_table, + "threshold": 30, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if owner_table: + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rqsd ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for RQSD", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Check and trigger core DAG + def check_and_trigger(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + threshold = ti.xcom_pull(task_ids='init_step', key='threshold') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + while True: + cursor = oracle_conn.cursor() + + # Execute SQL query with variable substitution + service_name = store.upper() + sql_query = f""" + SELECT (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB + WHERE LAST_STATUS = 'RUNNING') + + (SELECT CASE WHEN SUM(MAX_THREADS) IS NULL THEN 0 ELSE SUM(MAX_THREADS) END AS RUNNING_THREADS + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_RAR + WHERE LAST_STATUS = 'RUNNING') + AS TOTAL_RUNNING_THREADS_NOW, + (SELECT COUNT(*) FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' AND TABLE_NAME = '{table_name}' AND LAST_STATUS = 'RUNNING') AS TABLE_IS_ALREADY_RUNNING + FROM DUAL + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + total_running_val = result[0] or 0 + table_running_val = result[1] or 0 + cursor.close() + + logging.info("Total running: %d, threshold: %d, table running: %d", + total_running_val, threshold, table_running_val) + + if total_running_val > threshold: + logging.info("Threshold exceeded. Waiting 5 minutes...") + time.sleep(300) + continue + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + break + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_replicator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + break + + logging.info("Total DAGs triggered: %d", triggered_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + + except Exception as e: + logging.error(f"Error in check_and_trigger: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='check_and_trigger', + python_callable=check_and_trigger, + ) + + # Dependencies + init >> t1 >> t2 + +""" +RQSD Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME) +- Sets store to "rqsd" (fixed for this DAG) +- Sets threshold to 30 (max concurrent running threads) +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_rqsd +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) check_and_trigger +- Loops through each table from the table list +- For each table, enters monitoring loop: + - Executes SQL query to check total running threads across MOPDB+RAR + - Checks if current table is already running + - If total threads > threshold (30): waits 5 minutes and rechecks + - If table already running: skips to next tabl + - If conditions met: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered +- Ensures system doesn't exceed concurrent processing limits +""" diff --git a/airflow/devo_replicator/devo_table_generator.py b/airflow/devo_replicator/devo_table_generator.py new file mode 100644 index 0000000..22d4786 --- /dev/null +++ b/airflow/devo_replicator/devo_table_generator.py @@ -0,0 +1,1080 @@ +from __future__ import annotations +import sys + +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/python/devo_replicator/table_generator') +import pandas as pd +import mrds.utils.manage_files as fileManager +import logging +import tableBuilderQueries as tbq +from devo_query import execute_query +import ranger_updater as ranger +import yaml +import numpy as np +from mrds.utils.secrets import get_secret +import os +import logging +import yaml +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.python import PythonOperator +import traceback +try: + from airflow.exceptions import AirflowFailException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + + + +from mrds.utils import oraconn + +ENV_CONFIG_PATH = "/opt/airflow/python/devo_replicator/config/env_config.yaml" + +# Set up basic configuration for logging +logging.basicConfig(level=logging.INFO) + +# Create a logger object +logger = logging.getLogger(__name__) +import re + +#0 utilities +def initialize_config(config_file_path): + # Ensure the file exists + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + + # Load the configuration + with open(config_file_path, "r") as f: + config_data = yaml.safe_load(f) + + return config_data + +def fix_impala_sql(sql: str) -> str: + # List of reserved keywords in Impala that need backticks if used as column names + impala_reserved_keywords = { + 'date', 'value', 'source', 'comment', 'partition', 'row', 'select', 'insert', + 'table', 'external', 'format', 'location', 'stored', 'inputformat', 'outputformat', + 'scenario', 'string', 'int', 'decimal', 'timestamp', 'float', 'double','procedure', 'floor' + } + + # Regex pattern to find column definitions + pattern = re.compile( + r'(?P`?\w+`?)\s+(?P[A-Za-z]+\s*(?:\([^)]+\))?)\s*(?Pcomment\s*\'[^\']*\'|)?', + re.IGNORECASE + ) + + def replace(match): + col = match.group('col').strip('`') + dtype = match.group('type') + comment = match.group('comment') or '' + # Add backticks only if column name is a reserved keyword or contains special chars + if col.lower() in impala_reserved_keywords or not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', col): + col = f'`{col}`' + return f"{col} {dtype} {comment}".strip() + + # Only replace column list part between parentheses + table_def_start = sql.find('(') + table_def_end = sql.find('ROW FORMAT SERDE', table_def_start) + if table_def_start == -1 or table_def_end == -1: + raise ValueError("Invalid SQL format: Missing column definition parentheses.") + + before = sql[:table_def_start + 1] + columns = sql[table_def_start + 1:table_def_end] + after = sql[table_def_end:] + + # Replace all columns inside definition + fixed_columns = pattern.sub(replace, columns) + + # Combine and return + final= before + fixed_columns + after + final=final.replace("\\'", "").replace('\\\\', '\\') + return final + +def applyQueryParameters(query: str, parameters: str) -> str: + """ + Replaces placeholders in the query with values from parameters. + + Parameters: + - query: Original query string with placeholders like $$$1, $$$2, etc. + - parameters: Semicolon-separated string of parameter values. + + Returns: + - String with the query filled with parameter values. + """ + filled_query = query + if parameters: + # Split the parameters string and reverse the list + params_array = parameters.split(';')[::-1] + index = len(params_array) + for param in params_array: + # Replace the placeholder $$$ with the parameter + placeholder = f"$$${index}" + filled_query = filled_query.replace(placeholder, param) + index -= 1 # Decrement the index + return filled_query + +def format_column_definition(row): + if pd.isnull(row['data_description']): + # If data_description is null, only include column_name and data_type_string + return f"{row['column_name']} {row['data_type_string']}" + else: + # If data_description is present, include it with a comment + # Ensure data_description does not contain single quotes + data_description = str(row['data_description']).replace("'", "\\'") + return f"{row['column_name']} {row['data_type_string']} comment '{data_description}'" +#1 receive table name and check for target table and access type + +def execute_oracle_query(sql): + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + options=cursor.execute(sql).fetchall() + oracle_conn.commit() + df = pd.DataFrame(options,columns= [row[0].lower() for row in cursor.description]) + ## fetch db dtypes + cursor.close() + oracle_conn.close() + return df + +def get_target_table(oracle_mgmt_table,source_schema,source_table): + sql=f"SELECT DISTINCT TABLE_ALIAS FROM {oracle_mgmt_table} WHERE OWNER = '{source_schema}' AND TABLE_NAME = '{source_table}'" + df=execute_oracle_query(sql) + return df + +def get_type_ofAccess(oracle_metadata_table,table_owner,source_table,service_name): + sql=f"SELECT DISTINCT RAR3_TYPE_OF_ACCESS FROM {oracle_metadata_table} WHERE A_VALID_TO > SYSDATE AND OWNER = '{table_owner}'AND TABLE_NAME = '{source_table}'" + df=execute_oracle_query(sql) + + return df + +#2 load metadata +def readIGAMRoles( config ): + queryParams = "'" + config['sentry_role_environment'] + "'" + igamRolesQuery = tbq.get_query_igam_roles(config['oracle_igam_table'],config['service_name']) + logger.info(f"Querying the IGAM Table: {igamRolesQuery}") + + queryWithParamsIgamSentry = applyQueryParameters(igamRolesQuery, queryParams) + + logger.info(f"Replaced params to IGAM Table: {queryWithParamsIgamSentry}") + + igamRoleDF = execute_oracle_query(queryWithParamsIgamSentry) + + return igamRoleDF + +def loadMetadataTable( config ): + + metadataQuery = tbq.get_query_metadata(config['oracle_metadata_table'], config['source_schema'], config['source_table']) + + logger.info("Map Oracle metadata (data types) to Hive query: {metadataQuery}") + + jdbcMetaDataDF = execute_oracle_query( metadataQuery) + + logger.info("Fetch all fields for table and concatenate them separated by ','") + tableDataList = jdbcMetaDataDF.apply(format_column_definition, axis=1).tolist() + tableFields = ",".join(tableDataList) + + return tableFields + + +#3 drop table and policies +def deleteExternalTable(config,env_config): + try: + try: + deleted=ranger.delete_policy(config,env_config) + logger.info(f"deleted policies: {deleted}") + except Exception as e: + logger.error("Error in dropping table") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + except RuntimeError as e: + logger.error("Error in dropping table") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + + sql_drop = f"DROP TABLE IF EXISTS {config['corporate_store']}.{config['target_table']}" + execute_query( + sql_drop, + env_config['DEVO_USERNAME'], env_config['IMPALA_HOSTNAME'], env_config['DEVO_SECRET'], + ) + logger.info(f"table {config['corporate_store']}.{config['target_table']} dropped") + +#4 create external table +def createExternalTables(config, tableFields,env_config ): + sql_create = ( + f"CREATE EXTERNAL TABLE {config['corporate_store']}.{config['target_table']} " + f"({tableFields}, {config['tech_meta_data_fields']}) " + "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' " + "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' " + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' " + f"LOCATION '{config['target_s3_bucket']}/{config['target_table']}' " + "TBLPROPERTIES (" + "'external.table.purge'='true', " + "'parquet.compression'='snappy')" + ) + sql_create=fix_impala_sql(sql_create) + execute_query(sql_create,env_config['DEVO_USERNAME'],env_config['HIVE_HOSTNAME'],env_config['DEVO_SECRET']) + +def createTableFromExternal( config, tableFields,env_config ): + sql_create = ( + f"CREATE EXTERNAL TABLE {config['corporate_store']}.{config['target_table']} AS " + f"SELECT * FROM {config['corporate_store']}.{config['target_table']}_EXT" + ) + execute_query(sql_create,env_config['DEVO_USERNAME'],env_config['HIVE_HOSTNAME'],env_config['DEVO_SECRET']) + +#5 create table policies +def accessTypeMapper(config, env_config, igamRoleDF): + + + if (config['access_type'].lower() == '1'): + return accessType_1(config, env_config, igamRoleDF) + elif (config['access_type'].lower() == '2a'): + return accessType_2A(config, env_config, igamRoleDF) + elif (config['access_type'].lower() == '2b'): + return accessType_2B(config, env_config, igamRoleDF) + elif (config['access_type'].lower() == '3'): + return accessType_3(config, env_config, igamRoleDF) + else: + + logger.info(f"Invalid access type ${config['access_type']}. Please check the input param") + raise RuntimeError( + f"Access type error, access type :{config['access_type'].lower()} unsupported" + ) + +def accessType_1(config, env_config, igamRoleDF): + logger.info("Grant privileges for access type 1") + logger.info("Fetch metadata from Oracle for access type 1") + + # ---- Construct query and fetch from Oracle ---- + queryParams = f"'{config['source_schema']}.{config['source_table']}'" + queryMetadataAccessType1 = tbq.get_query_metadata_access_type1(config['oracle_metadata_table']) + queryWithParamsAccessType1 = applyQueryParameters(queryMetadataAccessType1, queryParams) + + logger.info("Metadata table query: " + queryWithParamsAccessType1) + jdbcMetaDataAccessType1DF = execute_oracle_query(queryWithParamsAccessType1) + + # ---- Normalize columns ---- + df = jdbcMetaDataAccessType1DF.copy() + df["rar3_type_of_access"] = df["rar3_type_of_access"].astype(str).str.strip() + df["source"] = df["source"].astype(str).str.strip().str.upper() + igamRoleDF["datasource"] = igamRoleDF["datasource"].astype(str).str.strip().str.upper() + + # ---- Branch A: source != 'RAR' ---- + left_a = ( + df.loc[ + (df["rar3_type_of_access"] == "1") & (df["source"] != config['service_name']), + ["table_name", "source"] + ] + .drop_duplicates() + ) + + branch_a = ( + left_a.merge( + igamRoleDF, + left_on="source", + right_on="datasource", + how="inner" + ) + [["table_name", "source", "subsource_id", "igam_entitlement", "environment"]] + .drop_duplicates() + ) + + # ---- Branch B: source == 'RAR' (CROSS JOIN with igamRoleDF) ---- + left_b = ( + df.loc[ + (df["rar3_type_of_access"] == "1") & (df["source"] == config['service_name']), + ["table_name", "source"] + ] + .drop_duplicates() + ) + + if not left_b.empty: + branch_b = ( + left_b.merge(igamRoleDF, how="cross") + [["table_name", "source", "subsource_id", "igam_entitlement", "environment"]] + .drop_duplicates() + ) + else: + branch_b = pd.DataFrame(columns=["table_name", "source", "subsource_id", "igam_entitlement", "environment"]) + + # ---- UNION (distinct) ---- + typeOneDF = ( + pd.concat([branch_a, branch_b], ignore_index=True) + .drop_duplicates() + .reset_index(drop=True) + ) + + logger.info("typeOneDF:\n%s", typeOneDF) + + # ---- Collect IGAM entitlements ---- + igam_entitlements = ( + typeOneDF["igam_entitlement"] + .dropna() + .astype(str) + .str.strip() + .tolist() + ) + # Extract IGAM entitlements + + # Merge with optional full access list + if config['full_access_entitlement_list'] is None: + combined_entitlements = igam_entitlements + else: + full_access_list_clean = config['full_access_entitlement_list'] + combined_entitlements = igam_entitlements + full_access_list_clean + + # Add table permission groups using YAMLFormatter + params = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + combined_entitlements + ) + + # Generate the final YAML policy + ranger.generate_policy(params,env_config, None) + + + +def accessType_2A(config, env_config, igamRoleDF): + logger.info("Grant privileges for access type 2a") + logger.info("Fetch the metadata in Oracle for access type 2a") + + # ---- Construct query and fetch from Oracle ---- + queryParams = f"'{config['source_schema']}.{config['source_table']}'" + queryMetadataAccessType2a = tbq.get_query_metadata_access_type2a(config['oracle_metadata_table']) + queryWithParamsAccessType2a = applyQueryParameters(queryMetadataAccessType2a, queryParams) + + logger.info("Meta data table query: ") + jdbcMetaDataAccessType2aDF = execute_oracle_query(queryWithParamsAccessType2a) + + # ---- Normalize columns ---- + df = jdbcMetaDataAccessType2aDF.copy() + df["rar3_type_of_access"] = df["rar3_type_of_access"].astype(str).str.strip().str.lower() + df["source"] = df["source"].astype(str).str.strip().str.upper() + + roles = igamRoleDF.copy() + # expected columns in igamRoleDF: rar_subsource_id, igam_entitlement, environment (plus anything else you keep) + roles["subsource_id"] = roles["subsource_id"].astype(str).str.strip().str.upper() + roles["igam_entitlement"] = roles["igam_entitlement"].astype(str).str.strip() + + # ---- Branch A: source != service_name -> INNER JOIN on source == rar_subsource_id ---- + left_a = ( + df.loc[ + (df["rar3_type_of_access"] == "2a") + & (df["source"] != config['service_name'].upper()), + ["table_name", "column_name", "source"] + ] + ) + + branch_a = ( + left_a.merge( + roles, + left_on="source", + right_on="subsource_id", + how="inner" + ) + .drop(columns=["subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + + # ---- Branch B: source == service_name -> CROSS JOIN with igamRoleDF ---- + left_b = ( + df.loc[ + (df["rar3_type_of_access"] == "2a") + & (df["source"] == config['service_name'].upper()), + ["table_name", "column_name", "source"] + ] + ) + + if not left_b.empty: + try: + branch_b = ( + left_b.merge(roles, how="cross") + .drop(columns=["subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + except TypeError: + # pandas < 1.2 fallback + left_b["_cj"] = 1 + roles["_cj"] = 1 + branch_b = ( + left_b.merge(roles, on="_cj") + .drop(columns=["_cj", "subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + # (optional) cleanup if you keep using roles later + roles.drop(columns=["_cj"], inplace=True, errors="ignore") + else: + branch_b = pd.DataFrame(columns=["table_name", "column_name", "igam_entitlement", "environment"]) + + # ---- UNION (ALL) ---- + one_df = ( + pd.concat([branch_a, branch_b], ignore_index=True) + .reset_index(drop=True) + ) + + # ---- Group 1: (table_name, igam_entitlement) -> sorted, comma-joined column_list ---- + tmp = one_df.sort_values(["table_name", "igam_entitlement", "column_name"], kind="mergesort") + new_df = ( + tmp.groupby(["table_name", "igam_entitlement"], as_index=False)["column_name"] + .apply(lambda s: ",".join(s.dropna().astype(str).tolist())) + .rename(columns={"column_name": "column_list"}) + ) + # Columns: table_name, igam_entitlement, column_list + + # ---- Group 2: (table_name, column_list) -> comma-joined igam_entitlement ---- + grouped = ( + new_df.groupby(["table_name", "column_list"], as_index=False)["igam_entitlement"] + .apply(lambda s: ",".join(s.dropna().astype(str).tolist())) + ) + # Columns: table_name, column_list, igam_entitlement + + # ---- ROW_NUMBER() OVER (ORDER BY column_list) -> policy_id ---- + grouped = grouped.sort_values(["column_list"], kind="mergesort") + grouped["policy_id"] = np.arange(1, len(grouped) + 1).astype(int) + + # ---- Emit policies: one per (table_name, column_list) row ---- + for _, row in grouped.iterrows(): + entitlements_list = [e.strip() for e in str(row["igam_entitlement"]).split(",") if e.strip()] + columns_list = [c.strip() for c in str(row["column_list"]).split(",") if c.strip()] + policy_id = str(int(row["policy_id"])) + + params = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], # "2a" + config['source_table'], + entitlements_list, + columns_list=columns_list + ) + ranger.generate_policy(params, env_config, policy_id) + + + # ---- Optional: append full-access YAML if list provided on config ---- + if config["full_access_entitlement_list"] != None: + # If your code already provides a list, use it directly; otherwise split string. + if isinstance(config["full_access_entitlement_list"], list): + full_access_list = config["full_access_entitlement_list"] + else: + full_access_list = [s.strip() for s in str(config["full_access_entitlement_list"]).split(",") if s.strip()] + + params_full = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + full_access_list + ) + ranger.generate_policy(params_full, env_config, "full_access") + + + + +def accessType_2B(config, env_config,igamRoleDF): + logger.info(f"Grant privileges for access type {config['access_type']}") + logger.info("Fetch the metadata in Oracle for access type 2b") + + # --- Validate required columns --- + required = {"environment", "igam_entitlement", "subsource_id"} + missing = required - set(igamRoleDF.columns) + if missing: + raise KeyError(f"igamRoleDF missing required column(s): {sorted(missing)}") + + # --- Normalize to strings (robust against None/NaN) --- + igamRoleDF = igamRoleDF.copy() + igamRoleDF["environment"] = igamRoleDF["environment"].astype(str).str.strip() + igamRoleDF["igam_entitlement"] = igamRoleDF["igam_entitlement"].astype(str).str.strip() + igamRoleDF["subsource_id"] = igamRoleDF["subsource_id"].astype(str).str.strip() + + # --- Aggregation: per (environment, igam_entitlement) collect unique rar_subsource_id list --- + # Keep a stable order by sorting; remove empties. + agg_df = ( + igamRoleDF.loc[igamRoleDF["subsource_id"].ne(""), ["environment", "igam_entitlement", "subsource_id"]] + .drop_duplicates() + .sort_values(["environment", "igam_entitlement", "subsource_id"], kind="mergesort") + .groupby(["environment", "igam_entitlement"], as_index=False)["subsource_id"] + .agg(lambda s: ",".join(s.unique())) + .rename(columns={"subsource_id": "subsource_id_list"}) + ) + + # List of tuples (IGAM_ENTITLEMENT, rar_subsource_id_list) — mirrors your log payload + accessType2bValidList = list(zip( + agg_df["igam_entitlement"].astype(str), + agg_df["subsource_id_list"].astype(str) + )) + logger.info(f"accessType2bValidList : {accessType2bValidList}") + + # --- Entitlements for policy generation (unique, non-empty) --- + igam_entitlements = ( + igamRoleDF["igam_entitlement"] + .dropna() + .map(str) + .str.strip() + .loc[lambda s: s.ne("")] + .drop_duplicates() + .tolist() + ) + logger.info(f"Collected IGAM entitlements ({len(igam_entitlements)}): {igam_entitlements}") + + # --- Row-level permissions (per your existing API) --- + params_row_level = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + igam_entitlements + ) + + # --- Table-level permissions, merging in full-access entitlements if provided --- + if ["full_access_entitlement_list"] != None: + combined_entitlements = igam_entitlements + config['full_access_entitlement_list'] + logger.info(f"Full-access entitlements provided ({len(config['full_access_entitlement_list'])}): {config['full_access_entitlement_list']}") + else: + combined_entitlements = igam_entitlements + + + # ---- Optional: append full-access YAML if list provided on config ---- + if config["full_access_entitlement_list"] != None: + # If your code already provides a list, use it directly; otherwise split string. + params_full = ranger.add_table_permission_groups( + config["corporate_store"], + config["target_table"], + config["access_type"], # keep same access type per your pattern + config["source_table"], + combined_entitlements + ) + ranger.generate_policy(params_full, env_config) + + + ranger.yaml_format_2b(params_row_level,env_config, config['full_access_entitlement_list']) # row-level policy + logger.info("Final YAML format emitted for 2B.") + + + + +def accessType_3(config,env_config, igamRoleDF): + """ + Python/pandas translation of the Scala accessType_3. + Expects igamRoleDF to have at least: ['igam_entitlement', 'rar_subsource_id']. + The `config` object should expose the attributes used below (names match your Scala/Python usage). + Uses a YAML formatter module `ranger` with: + - add_table_permission_groups(corporate_store, target_table, access_type, source_table, entitlements) + - yaml_format_3(params) + - yaml_format_1(params) + """ + + # --- 1) Filter entitlements where rar_subsource_id = 'TMS' --- + if not {"igam_entitlement", "subsource_id"}.issubset(igamRoleDF.columns): + missing = {"igam_entitlement", "subsource_id"} - set(igamRoleDF.columns) + raise KeyError(f"igamRoleDF missing required column(s): {sorted(missing)}") + + new_df = ( + igamRoleDF.loc[ + igamRoleDF["subsource_id"].astype(str).str.upper() == "TMS", + ["igam_entitlement"] + ].drop_duplicates() + ) + logger.info("new_df :\n%s", new_df.to_string(index=False)) + + accessType3ValidList = new_df["igam_entitlement"].astype(str).str.strip().tolist() + + # --- 2) Build params for row-level groups (type 3) --- + params_row_level = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + accessType3ValidList + ) + + corp = str(config['corporate_store']).lower() + src_tbl = str(config['source_table']).lower() + + # --- 3) Compose the filter expressions (match Scala strings) --- + sqlCreateView3NonRestrString_Ptree = ( + "(parent_fk in ( " + f"select portfolio_fk from {corp}.nh_portfolio_access " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + ")) AND (child_fk in ( " + f"select portfolio_fk from {corp}.nh_portfolio_access " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + "))" + ) + + sqlCreateView3NonRestrString_Pos = ( + "position_key in ( " + f"select position_key from {corp}.nh_portfolio_access a " + f"inner join {corp}.nh_position b on ( " + "(b.portfolio_fk = a.portfolio_fk and b.portfolio_fk is not NULL) or " + "(b.portfolio_compare_fk = a.portfolio_fk and b.portfolio_compare_fk is not NULL) " + ") " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + ")" + ) + + sqlCreateView3PortAccess = "lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%')" + sqlCreateView3LimAccess = "lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%')" + + # Standard case uses the configured key columns/table names + key_col = getattr(config, "type3SourceTableKeyColumn", None) + acc_col = getattr(config, "type3AccessTableKeyColumn", None) + acc_table= getattr(config, "type3AccessTable", None) + if not all([key_col, acc_col, acc_table]): + # Only needed for the default branch; keep None if your config doesn't use the default + key_col = key_col or "source_key_col" + acc_col = acc_col or "access_key_col" + acc_table = acc_table or "type3_access_table" + + sqlCreateView3NonRestrString_Stdrd = ( + f"{key_col} in (select {acc_col} from {corp}.{acc_table} " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp())" + ) + + # --- 4) Choose the filter by source table (matches Scala match/case) --- + if src_tbl == "nh_portfoliotree": + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Ptree + elif src_tbl == "nh_position": + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Pos + elif src_tbl == "nh_portfolio_access": + sqlCreateViewType3Filter = sqlCreateView3PortAccess + elif src_tbl == "nh_limit_access": + sqlCreateViewType3Filter = sqlCreateView3LimAccess + else: + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Stdrd + + # --- 5) Row filter YAML block (uses groups from params_row_level) --- + # Expecting params_row_level like {'igam_roles': '...'}; adjust key if your API differs. + igam_roles_lower = str(params_row_level.get("igam_roles", "")).lower() + rowFilter = ( + "- groups:\n" + f" {igam_roles_lower}\n" + " accesses:\n" + " - select\n" + f" filterExpr: \"{sqlCreateViewType3Filter}\"\n" + " " + ) + + # --- 6) Handle optional full access entitlements --- + + + if config['full_access_entitlement_list']: + paramsFullAccess = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + config['full_access_entitlement_list'] + ) + + full_groups_lower = str(paramsFullAccess.get("igam_roles", "")).lower() + """ + fullAccessFilter = ( + "- groups:\n" + f" {full_groups_lower}\n" + " accesses:\n" + " - select\n" + " filterExpr: \"1=1\"\n" + " " + ) + """ + params_table_level = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + accessType3ValidList + config['full_access_entitlement_list'] + ) + else: + fullAccessFilter = "" + params_table_level = ranger.add_table_permission_groups( + config['corporate_store'], + config['target_table'], + config['access_type'], + config['source_table'], + accessType3ValidList + ) + + # --- 7) Render YAML and merge like Scala --- + ranger.yaml_format_3(params_row_level,env_config,sqlCreateViewType3Filter,config['full_access_entitlement_list'] ) # base type 3 yaml + ranger.yaml_format_1(params_table_level,env_config) # table-level yaml + +######################################################################################################################################################### +####################################STARTING DAG######################################################################################################### + + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_table_generator_core', + default_args=default_args, + description='Core Devo table generator workflow for single table', + schedule=None, + catchup=False, + tags=['DevoTableGenerator'], + max_active_runs=10, + max_active_tasks=16, +) as dag: + + # Init - read config from context + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + store = conf.get("store") + if not store: + raise ValueError("store parameter is required") + store = store.lower() + + owner_table = conf.get("owner_table") + if not owner_table or '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst","acc","prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + if store not in {"mopdb", "rar", 'rqsd'}: + raise ValueError(f"Unsupported store '{store}'. Expected 'mopdb', 'rar', 'rqsd'.") + + if store.lower() == "mopdb": + p_service_name = "MOPDB" + elif store.lower() == "rar": + p_service_name = "RAR" + elif store.lower() == 'rqsd': + p_service_name = "RQSD" + + + if env == "dev" or env == "tst": + sentry_role_environment = "TEST/INTEGRATION" + elif env == "acc": + sentry_role_environment = "ACCEPTANCE" + elif env == 'prd': + sentry_role_environment = "PRODUCTION" + + + with open(ENV_CONFIG_PATH, "r") as f: + cfg = yaml.safe_load(f) + + + env_cfg = cfg[env] + store_cfg = cfg[store] + p_objectstore_uri = env_cfg["S3_LOCATION_URI"].replace("{0}", store.lower()) + p_run_id = str(ti.run_id) + logging.info("=== init_step begins === env=%s store=%s table=%s.%s run_id=%s sentry_role_environment=%s", + env, store, table_owner, table_name, p_run_id, sentry_role_environment) + + if store.lower()=='rqsd': + env_cfg["DEVO_SECRET"]=env_cfg["DEVO_SECRET_RQSD"] + env_cfg["DEVO_USERNAME"]=env_cfg["DEVO_USERNAME_RQSD"] + try: + devo_secret_name = env_cfg["DEVO_SECRET"] + env_cfg["DEVO_SECRET"]= get_secret(devo_secret_name) + except: + logger.error("Failed to retrieve credentials from secrets") + raise(Exception) + + logging.info("=== init_step getting table info=== env=%s store=%s table=%s.%s run_id=%s devo_user=%s sentry_role_environment=%s", + env, store, table_owner, table_name, p_run_id,env_cfg["DEVO_USERNAME"], sentry_role_environment) + try: + target_table=get_target_table(store_cfg['oracle_mgmt_table'],table_owner,table_name)['table_alias'][0] + except Exception as e: + logger.error("Table not found in oracle management table") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + raise + try: + access_type=get_type_ofAccess(store_cfg['oracle_metadata_table'],table_owner,table_name,p_service_name)['rar3_type_of_access'][0] + except Exception as e: + logger.error("Table not found in oracle metadata inventory") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + raise + + + xcom = { + 'db':store, + 'env': env, + 'run_id':p_run_id, + 'corporate_store':store_cfg['corporate_store'], + 'service_name': p_service_name, + 'source_schema':table_owner, + 'source_table':table_name, + 'oracle_metadata_table':store_cfg['oracle_metadata_table'], + 'oracle_igam_table':store_cfg['oracle_igam_table'], + 'oracle_mgmt_table': store_cfg['oracle_mgmt_table'], + 'target_table':target_table, + 'sentry_role_environment':sentry_role_environment, + 'target_s3_bucket': env_cfg["BUCKET_PREFIX"]+store_cfg['target_s3_bucket'] , + 'tech_meta_data_fields': store_cfg['tech_meta_data_fields'], + 'full_access_entitlement_list':env_cfg[f'FULL_ACCESS_LIST_{p_service_name.upper()}'].split(','), + 'access_type': access_type, + 'DEVO_USERNAME': env_cfg["DEVO_USERNAME"], + 'DEVO_SECRET': env_cfg["DEVO_SECRET"], + 'IMPALA_HOSTNAME': env_cfg["IMPALA_HOSTNAME"], + 'HIVE_HOSTNAME': env_cfg["HIVE_HOSTNAME"], + 'RANGER_HOSTNAME': env_cfg["RANGER_HOSTNAME"], + 'BUCKET_PREFIX': env_cfg['BUCKET_PREFIX'], + 'S3_LOCATION_URI': env_cfg["S3_LOCATION_URI"] + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Start log table + def start_log_table_task(**context): + ti = context["ti"] + p_run_id = ti.xcom_pull(task_ids='init_step', key='run_id') + p_service_name = ti.xcom_pull(task_ids='init_step', key='service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='table_name') + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.end_log_table_generator', + [p_run_id, p_service_name, p_table_owner, p_table_name,'success'] + ) + oracle_conn.commit() + logging.info("start_log_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in start_log_table: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='start_log_table', + python_callable=start_log_table_task, + ) + + + # Drop table + def drop_table_task(**context): + ti = context["ti"] + config={"corporate_store":ti.xcom_pull(task_ids='init_step', key='corporate_store'), + "target_table":ti.xcom_pull(task_ids='init_step', key='target_table'), + 'access_type': ti.xcom_pull(task_ids='init_step', key='access_type'), + 'env': ti.xcom_pull(task_ids='init_step', key='env') + + } + env_config={'DEVO_USERNAME': ti.xcom_pull(task_ids='init_step', key="DEVO_USERNAME"), + 'DEVO_SECRET': ti.xcom_pull(task_ids='init_step', key="DEVO_SECRET"), + 'IMPALA_HOSTNAME':ti.xcom_pull(task_ids='init_step', key="IMPALA_HOSTNAME"), + 'HIVE_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="HIVE_HOSTNAME"), + 'RANGER_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="RANGER_HOSTNAME"), + 'BUCKET_PREFIX': ti.xcom_pull(task_ids='init_step', key='BUCKET_PREFIX'), + 'S3_LOCATION_URI':ti.xcom_pull(task_ids='init_step', key="S3_LOCATION_URI")} + try: + deleteExternalTable(config,env_config) + logging.info("drop_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in drop_table: {e}") + raise + + t2 = PythonOperator( + task_id='drop_table', + python_callable=drop_table_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Devo Iextarnal table creation + def devo_table_creation_task(**context): + ti = context["ti"] + + config={"corporate_store":ti.xcom_pull(task_ids='init_step', key='corporate_store'), + "target_table":ti.xcom_pull(task_ids='init_step', key='target_table'), + 'access_type': ti.xcom_pull(task_ids='init_step', key='access_type'), + 'source_schema': ti.xcom_pull(task_ids='init_step', key='source_schema'), + 'source_table': ti.xcom_pull(task_ids='init_step', key='source_table'), + 'oracle_metadata_table': ti.xcom_pull(task_ids='init_step', key='oracle_metadata_table'), + 'target_s3_bucket': ti.xcom_pull(task_ids='init_step', key='target_s3_bucket'), + 'tech_meta_data_fields': ti.xcom_pull(task_ids='init_step', key='tech_meta_data_fields'), + 'env': ti.xcom_pull(task_ids='init_step', key='env') + + } + env_config={'DEVO_USERNAME': ti.xcom_pull(task_ids='init_step', key="DEVO_USERNAME"), + 'DEVO_SECRET': ti.xcom_pull(task_ids='init_step', key="DEVO_SECRET"), + 'IMPALA_HOSTNAME':ti.xcom_pull(task_ids='init_step', key="IMPALA_HOSTNAME"), + 'HIVE_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="HIVE_HOSTNAME"), + 'RANGER_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="RANGER_HOSTNAME"), + 'BUCKET_PREFIX': ti.xcom_pull(task_ids='init_step', key='BUCKET_PREFIX'), + 'S3_LOCATION_URI':ti.xcom_pull(task_ids='init_step', key="S3_LOCATION_URI")} + + tableFields=loadMetadataTable(config) + logging.info("Starting table creation on hive with env=%s store=%s corporate_store=%s table=%s devo_user=%s", + ti.xcom_pull(task_ids='init_step', key='env'), ti.xcom_pull(task_ids='init_step', key='db'), ti.xcom_pull(task_ids='init_step', key='corporate_store'), ti.xcom_pull(task_ids='init_step', key='target_table'),env_config["DEVO_USERNAME"]) + + if (config['target_table'][-4:].upper() == '_EXT'): + createExternalTables( config, tableFields,env_config ) + else: + createTableFromExternal( config, tableFields,env_config) + + try: + logging.info("Impyla (Devo) task finished successfully.") + except Exception as e: + logging.error(f"Error in devo_impyla_task: {e}") + raise + + t3 = PythonOperator( + task_id='devo_table_creation_task', + python_callable=devo_table_creation_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Devo + def ranger_policy_creation(**context): + ti = context["ti"] + + config={"corporate_store":ti.xcom_pull(task_ids='init_step', key='corporate_store'), + 'source_table': ti.xcom_pull(task_ids='init_step', key='source_table'), + "target_table":ti.xcom_pull(task_ids='init_step', key='target_table'), + 'access_type': ti.xcom_pull(task_ids='init_step', key='access_type'), + 'source_schema': ti.xcom_pull(task_ids='init_step', key='source_schema'), + 'service_name': ti.xcom_pull(task_ids='init_step', key='service_name'), + 'oracle_metadata_table': ti.xcom_pull(task_ids='init_step', key='oracle_metadata_table'), + 'target_s3_bucket': ti.xcom_pull(task_ids='init_step', key='target_s3_bucket'), + 'tech_meta_data_fields': ti.xcom_pull(task_ids='init_step', key='tech_meta_data_fields'), + 'sentry_role_environment': ti.xcom_pull(task_ids='init_step', key='sentry_role_environment'), + 'full_access_entitlement_list': ti.xcom_pull(task_ids='init_step', key='full_access_entitlement_list'), + 'oracle_igam_table':ti.xcom_pull(task_ids='init_step', key='oracle_igam_table'), + 'env': ti.xcom_pull(task_ids='init_step', key='env') + + } + env_config={'DEVO_USERNAME': ti.xcom_pull(task_ids='init_step', key="DEVO_USERNAME"), + 'DEVO_SECRET': ti.xcom_pull(task_ids='init_step', key="DEVO_SECRET"), + 'IMPALA_HOSTNAME':ti.xcom_pull(task_ids='init_step', key="IMPALA_HOSTNAME"), + 'HIVE_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="HIVE_HOSTNAME"), + 'RANGER_HOSTNAME': ti.xcom_pull(task_ids='init_step', key="RANGER_HOSTNAME"), + 'BUCKET_PREFIX': ti.xcom_pull(task_ids='init_step', key='BUCKET_PREFIX'), + 'S3_LOCATION_URI':ti.xcom_pull(task_ids='init_step', key="S3_LOCATION_URI")} + + + logging.info("Starting Policy creation with env=%s store=%s corporate_store=%s table=%s sentry_role_environment=%s", + ti.xcom_pull(task_ids='init_step', key='env'), ti.xcom_pull(task_ids='init_step', key='db'), ti.xcom_pull(task_ids='init_step', key='corporate_store'), ti.xcom_pull(task_ids='init_step', key='target_table'),ti.xcom_pull(task_ids='init_step', key='sentry_role_environment')) + + + if config['target_table'][-4:].upper() != '_EXT': + igamRoles=readIGAMRoles(config) + logger.info(accessTypeMapper(config,env_config,igamRoles)) + + try: + logging.info("Impyla (Devo) task finished successfully.") + except Exception as e: + logging.error(f"Error in devo_impyla_task: {e}") + raise + + t4 = PythonOperator( + task_id='ranger_policy_creation', + python_callable=ranger_policy_creation, + trigger_rule=TriggerRule.ALL_DONE, + ) + + + # End log table + def end_log_table_task(**context): + ti = context["ti"] + p_service_name = ti.xcom_pull(task_ids='init_step', key='p_service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='p_table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='p_table_name') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.end_log_table', + [p_service_name, p_table_owner, p_table_name] + ) + oracle_conn.commit() + logging.info("end_log_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in end_log_table: {e}") + logging.info("Continuing despite end_log_table error (cleanup task)") + finally: + if oracle_conn: + oracle_conn.close() + + t5 = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + + # Check status and fail if needed + def fail_if_any_failed(**context): + dag_run = context['dag_run'] + check_tasks = ['init_step','start_log_table', 'drop_table', 'devo_table_creation_task','ranger_policy_creation', 'end_log_table'] + failed = [] + + for tid in check_tasks: + ti_up = dag_run.get_task_instance(tid) + if ti_up and ti_up.state == 'failed': + failed.append(tid) + + if failed: + ti = context["ti"] + p_run_id = ti.xcom_pull(task_ids='init_step', key='run_id') + p_service_name = ti.xcom_pull(task_ids='init_step', key='service_name') + p_table_owner = ti.xcom_pull(task_ids='init_step', key='table_owner') + p_table_name = ti.xcom_pull(task_ids='init_step', key='table_name') + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + oraconn.run_proc( + oracle_conn, + 'MRDS_LOADER.DATA_REPLICATOR.end_log_table_generator', + [p_run_id, p_service_name, p_table_owner, p_table_name,f"The Following task failed: {failed[0]}"] + ) + oracle_conn.commit() + logging.info("start_log_table procedure executed successfully") + except Exception as e: + logging.error(f"Error in start_log_table: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + error_msg = f"Critical task(s) failed: {', '.join(failed)}. DAG execution failed." + logging.error(error_msg) + raise AirflowFailException(error_msg) + + logging.info("All critical tasks completed successfully: %s", check_tasks) + + t6 = PythonOperator( + task_id='fail_if_any_failed', + python_callable=fail_if_any_failed, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Dependencies + init >> t1 >> t2 >> t3>> t4 + [t1, t2, t3, t4] >> t5 + t5 >> t6 diff --git a/airflow/devo_replicator/devo_table_generator_trigger_mopdb.py b/airflow/devo_replicator/devo_table_generator_trigger_mopdb.py new file mode 100644 index 0000000..e1c0332 --- /dev/null +++ b/airflow/devo_replicator/devo_table_generator_trigger_mopdb.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import os +import sys +import logging +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get MOPDB table options for dropdown +def get_mopdb_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_mopdb ORDER BY OWNER, TABLE_NAME") + options = ["__ALL_EXCLUDE_COPY__"] + [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting MOPDB table options: {e}") + return ["__ALL_EXCLUDE_COPY__"] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, +} + +with DAG( + dag_id='devo_table_generator_trigger_mopdb', + default_args=default_args, + description='External trigger DAG for MOPDB tables', + schedule=None, + catchup=False, + tags=['DevoTableGenerator', 'DevoTableGeneratorTrigger'], + params={ + "owner_table": Param( + default="__ALL_EXCLUDE_COPY__", + type="string", + description="Select '__ALL_EXCLUDE_COPY__' to run all tables without _COPY, or select specific table in format OWNER.TABLE_NAME", + enum=get_mopdb_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "mopdb" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + + # Handle special "run all" case + run_all_exclude_copy = (owner_table == "__ALL_EXCLUDE_COPY__") + + if not run_all_exclude_copy and '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME' or '__ALL_EXCLUDE_COPY__'") + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s run_all_exclude_copy=%s", + env, store, owner_table, run_all_exclude_copy) + + xcom = { + "env": env, + "store": store, + "owner_table": owner_table, + "run_all_exclude_copy": run_all_exclude_copy, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + run_all_exclude_copy = ti.xcom_pull(task_ids='init_step', key='run_all_exclude_copy') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if run_all_exclude_copy: + # Get all tables excluding those with _COPY in the name + cursor = oracle_conn.cursor() + cursor.execute(""" + SELECT OWNER, TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_mopdb + WHERE TABLE_NAME NOT LIKE '%_COPY%' + ORDER BY OWNER, TABLE_NAME + """) + tables = cursor.fetchall() + cursor.close() + logging.info("Processing ALL tables excluding _COPY: %d tables found", len(tables)) + elif owner_table: + # Process specific table + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + # Fallback: get all tables + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_mopdb ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for MOPDB", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Trigger core DAG for each table + def trigger_tables(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + skipped_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + cursor = oracle_conn.cursor() + + # Check if table is already running + service_name = store.upper() + sql_query = f""" + SELECT COUNT(*) + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' + AND TABLE_NAME = '{table_name}' + AND LAST_STATUS = 'RUNNING' + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + table_running_val = result[0] or 0 + cursor.close() + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + skipped_count += 1 + continue + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_table_generator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + + logging.info("Summary: Total DAGs triggered: %d, Skipped (already running): %d", + triggered_count, skipped_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + ti.xcom_push(key='skipped_count', value=skipped_count) + + except Exception as e: + logging.error(f"Error in trigger_tables: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='trigger_tables', + python_callable=trigger_tables, + ) + + # Dependencies + init >> t1 >> t2 + +""" +MOPDB Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME or __ALL_EXCLUDE_COPY__) +- Sets store to "mopdb" (fixed for this DAG) +- Determines if running all tables excluding _COPY +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If __ALL_EXCLUDE_COPY__: queries all tables from CT_MRDS.a_devo_replica_mgmt_mopdb excluding _COPY tables +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_mopdb +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) trigger_tables +- Loops through each table from the table list +- For each table: + - Checks if table is already running + - If table already running: skips to next table + - If not running: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered and skipped +- No threshold checking or waiting logic +""" diff --git a/airflow/devo_replicator/devo_table_generator_trigger_rar.py b/airflow/devo_replicator/devo_table_generator_trigger_rar.py new file mode 100644 index 0000000..09e2766 --- /dev/null +++ b/airflow/devo_replicator/devo_table_generator_trigger_rar.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import os +import sys +import logging +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get RAR table options for dropdown +def get_rar_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rar ORDER BY OWNER, TABLE_NAME") + options = ["__ALL_EXCLUDE_COPY__"] + [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting RAR table options: {e}") + return ["__ALL_EXCLUDE_COPY__"] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_table_generator_trigger_rar', + default_args=default_args, + description='External trigger DAG for RAR tables', + schedule=None, + catchup=False, + tags=['DevoTableGenerator', 'DevoTableGeneratorTrigger'], + params={ + "owner_table": Param( + default="__ALL_EXCLUDE_COPY__", + type="string", + description="Select '__ALL_EXCLUDE_COPY__' to run all tables without _COPY, or select specific table in format OWNER.TABLE_NAME", + enum=get_rar_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "rar" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + + # Handle special "run all" case + run_all_exclude_copy = (owner_table == "__ALL_EXCLUDE_COPY__") + + if not run_all_exclude_copy and '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME' or '__ALL_EXCLUDE_COPY__'") + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s run_all_exclude_copy=%s", + env, store, owner_table, run_all_exclude_copy) + + xcom = { + "env": env, + "store": store, + "owner_table": owner_table, + "run_all_exclude_copy": run_all_exclude_copy, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + run_all_exclude_copy = ti.xcom_pull(task_ids='init_step', key='run_all_exclude_copy') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if run_all_exclude_copy: + # Get all tables excluding those with _COPY in the name + cursor = oracle_conn.cursor() + cursor.execute(""" + SELECT OWNER, TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_rar + WHERE TABLE_NAME NOT LIKE '%_COPY%' + ORDER BY OWNER, TABLE_NAME + """) + tables = cursor.fetchall() + cursor.close() + logging.info("Processing ALL tables excluding _COPY: %d tables found", len(tables)) + elif owner_table: + # Process specific table + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + # Fallback: get all tables + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rar ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for RAR", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Trigger core DAG for each table + def trigger_tables(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + skipped_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + cursor = oracle_conn.cursor() + + # Check if table is already running + service_name = store.upper() + sql_query = f""" + SELECT COUNT(*) + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' + AND TABLE_NAME = '{table_name}' + AND LAST_STATUS = 'RUNNING' + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + table_running_val = result[0] or 0 + cursor.close() + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + skipped_count += 1 + continue + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_table_generator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + + logging.info("Summary: Total DAGs triggered: %d, Skipped (already running): %d", + triggered_count, skipped_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + ti.xcom_push(key='skipped_count', value=skipped_count) + + except Exception as e: + logging.error(f"Error in trigger_tables: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='trigger_tables', + python_callable=trigger_tables, + ) + + # Dependencies + init >> t1 >> t2 + +""" +RAR Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME or __ALL_EXCLUDE_COPY__) +- Sets store to "rar" (fixed for this DAG) +- Determines if running all tables excluding _COPY +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If __ALL_EXCLUDE_COPY__: queries all tables from CT_MRDS.a_devo_replica_mgmt_rar excluding _COPY tables +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_rar +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) trigger_tables +- Loops through each table from the table list +- For each table: + - Checks if table is already running + - If table already running: skips to next table + - If not running: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered and skipped +- No threshold checking or waiting logic +""" diff --git a/airflow/devo_replicator/devo_table_generator_trigger_rqsd.py b/airflow/devo_replicator/devo_table_generator_trigger_rqsd.py new file mode 100644 index 0000000..4d1d1f1 --- /dev/null +++ b/airflow/devo_replicator/devo_table_generator_trigger_rqsd.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import os +import sys +import logging +from datetime import timedelta + +from airflow import DAG +from airflow.utils.dates import days_ago +from airflow.operators.python import PythonOperator +from airflow.models import Param + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') + +from mrds.utils import oraconn + +# Get RQSD table options for dropdown +def get_rqsd_table_options(): + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER || '.' || TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rqsd ORDER BY OWNER, TABLE_NAME") + options = [row[0] for row in cursor.fetchall()] + cursor.close() + return options + except Exception as e: + logging.error(f"Error getting RQSD table options: {e}") + return [] + finally: + if oracle_conn: + oracle_conn.close() + +default_args = { + 'owner': 'devo', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +with DAG( + dag_id='devo_table_generator_trigger_rqsd', + default_args=default_args, + description='External trigger DAG for RQSD tables', + schedule=None, + catchup=False, + tags=['DevoTableGenerator', 'DevoTableGeneratorTrigger'], + params={ + "owner_table": Param( + default=None, + type="string", + description="Select table in format OWNER.TABLE_NAME", + enum=get_rqsd_table_options() + ) + } +) as dag: + + # Init + def init_step(**context): + dag_run = context.get("dag_run") + ti = context["ti"] + conf = (dag_run.conf or {}) if dag_run else {} + + env = os.getenv("MRDS_ENV") + if not env: + raise ValueError("MRDS_ENV environment variable is required") + env = env.lower() + + store = "rqsd" + owner_table = conf.get("owner_table") + + if not owner_table: + raise ValueError("owner_table parameter is required") + if '.' not in owner_table: + raise ValueError("owner_table must be in format 'OWNER.TABLE_NAME'") + + table_owner, table_name = owner_table.split('.', 1) + + if env not in {"dev", "tst", "acc", "prd"}: + raise ValueError(f"Unsupported env '{env}'. Expected 'dev', 'tst', 'acc' or 'prd'.") + + logging.info("=== init_step === env=%s store=%s owner_table=%s", + env, store, owner_table) + + xcom = { + "env": env, + "store": store, + "table_owner": table_owner, + "table_name": table_name, + "owner_table": owner_table, + } + + for k, v in xcom.items(): + ti.xcom_push(key=k, value=v) + + init = PythonOperator( + task_id='init_step', + python_callable=init_step, + ) + + # Get table list + def get_table_list(**context): + ti = context["ti"] + store = ti.xcom_pull(task_ids='init_step', key='store') + owner_table = ti.xcom_pull(task_ids='init_step', key='owner_table') + + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + if owner_table: + table_owner, table_name = owner_table.split('.', 1) + tables = [(table_owner, table_name)] + logging.info("Processing specific table: %s", owner_table) + else: + cursor = oracle_conn.cursor() + cursor.execute("SELECT OWNER, TABLE_NAME FROM CT_MRDS.a_devo_replica_mgmt_rqsd ORDER BY OWNER, TABLE_NAME") + tables = cursor.fetchall() + cursor.close() + logging.info("Found %d tables for RQSD", len(tables)) + + ti.xcom_push(key='tables_to_process', value=tables) + return tables + except Exception as e: + logging.error(f"Error in get_table_list: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t1 = PythonOperator( + task_id='get_table_list', + python_callable=get_table_list, + ) + + # Trigger core DAG for each table + def trigger_tables(**context): + ti = context["ti"] + env = ti.xcom_pull(task_ids='init_step', key='env') + store = ti.xcom_pull(task_ids='init_step', key='store') + tables = ti.xcom_pull(task_ids='get_table_list', key='tables_to_process') + + oracle_conn = None + triggered_count = 0 + skipped_count = 0 + + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + + for table_owner, table_name in tables: + logging.info("Processing table: %s.%s", table_owner, table_name) + + cursor = oracle_conn.cursor() + + # Check if table is already running + service_name = store.upper() + sql_query = f""" + SELECT COUNT(*) + FROM CT_MRDS.A_DEVO_REPLICA_MGMT_{service_name} + WHERE OWNER = '{table_owner}' + AND TABLE_NAME = '{table_name}' + AND LAST_STATUS = 'RUNNING' + """ + + cursor.execute(sql_query) + result = cursor.fetchone() + table_running_val = result[0] or 0 + cursor.close() + + if table_running_val >= 1: + logging.info("Table %s.%s is already running. Skipping.", table_owner, table_name) + skipped_count += 1 + continue + + # Trigger core DAG + from airflow.api.common.trigger_dag import trigger_dag + + conf = { + "store": store, + "owner_table": f"{table_owner}.{table_name}" + } + + trigger_dag( + dag_id='devo_table_generator_core', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info("Triggered core DAG for table %s.%s", table_owner, table_name) + + logging.info("Summary: Total DAGs triggered: %d, Skipped (already running): %d", + triggered_count, skipped_count) + ti.xcom_push(key='triggered_count', value=triggered_count) + ti.xcom_push(key='skipped_count', value=skipped_count) + + except Exception as e: + logging.error(f"Error in trigger_tables: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + + t2 = PythonOperator( + task_id='trigger_tables', + python_callable=trigger_tables, + ) + + # Dependencies + init >> t1 >> t2 + +""" +RQSD Trigger DAG +1) init_step +- Gets environment from MRDS_ENV environment variable +- Reads owner_table parameter from DAG configuration +- Validates owner_table format (must be OWNER.TABLE_NAME) +- Sets store to "rqsd" (fixed for this DAG) +- Pushes parameters to XCom +2) get_table_list +- Connects to Oracle database (MRDS_LOADER) +- If specific owner_table provided: creates single table list +- If no owner_table: queries all tables from CT_MRDS.a_devo_replica_mgmt_rqsd +- Returns list of (owner, table_name) tuples to process +- Pushes table list to XCom +3) trigger_tables +- Loops through each table from the table list +- For each table: + - Checks if table is already running + - If table already running: skips to next table + - If not running: triggers core DAG with table parameters +- Counts and logs total number of DAGs triggered and skipped +- No threshold checking or waiting logic +""" \ No newline at end of file diff --git a/airflow/mopdb/CSDB/DEBT/w_MOPDB_CSDB_DEBT.py b/airflow/mopdb/CSDB/DEBT/w_MOPDB_CSDB_DEBT.py new file mode 100644 index 0000000..cbdf79e --- /dev/null +++ b/airflow/mopdb/CSDB/DEBT/w_MOPDB_CSDB_DEBT.py @@ -0,0 +1,146 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT = DbtTaskGroup( + group_id="m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() diff --git a/airflow/mopdb/LM/ADH_ADJUSTMENTS/w_MOPDB_LM_ADHOC_ADJUSTMENT.py b/airflow/mopdb/LM/ADH_ADJUSTMENTS/w_MOPDB_LM_ADHOC_ADJUSTMENT.py new file mode 100644 index 0000000..f58529e --- /dev/null +++ b/airflow/mopdb/LM/ADH_ADJUSTMENTS/w_MOPDB_LM_ADHOC_ADJUSTMENT.py @@ -0,0 +1,156 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT = DbtTaskGroup( + group_id="m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + + + + + diff --git a/airflow/mopdb/LM/BALANCE_SHEET/w_MOPDB_LM_BALANCESHEET.py b/airflow/mopdb/LM/BALANCE_SHEET/w_MOPDB_LM_BALANCESHEET.py new file mode 100644 index 0000000..e8d7390 --- /dev/null +++ b/airflow/mopdb/LM/BALANCE_SHEET/w_MOPDB_LM_BALANCESHEET.py @@ -0,0 +1,152 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_BALANCE_SHEET_OU_LM_BALANCE_SHEET = DbtTaskGroup( + group_id="m_MOPDB_LM_T_BALANCE_SHEET_OU_LM_BALANCE_SHEET", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_BALANCE_SHEET_OU_LM_BALANCE_SHEET", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_BALANCE_SHEET_OU_LM_BALANCE_SHEET >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + diff --git a/airflow/mopdb/LM/CSM_ADJUSTMENTS/w_MOPDB_LM_CSM_ADJUSTMENT.py b/airflow/mopdb/LM/CSM_ADJUSTMENTS/w_MOPDB_LM_CSM_ADJUSTMENT.py new file mode 100644 index 0000000..3b7066a --- /dev/null +++ b/airflow/mopdb/LM/CSM_ADJUSTMENTS/w_MOPDB_LM_CSM_ADJUSTMENT.py @@ -0,0 +1,147 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT = DbtTaskGroup( + group_id="m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + diff --git a/airflow/mopdb/LM/CURRENT_ACCOUNTS/w_MOPDB_LM_CURRENT_ACCOUNTS.py b/airflow/mopdb/LM/CURRENT_ACCOUNTS/w_MOPDB_LM_CURRENT_ACCOUNTS.py new file mode 100644 index 0000000..d1f7824 --- /dev/null +++ b/airflow/mopdb/LM/CURRENT_ACCOUNTS/w_MOPDB_LM_CURRENT_ACCOUNTS.py @@ -0,0 +1,157 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS = DbtTaskGroup( + group_id="m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + + + + + + diff --git a/airflow/mopdb/LM/FORECAST/w_MOPDB_LM_FORECAST.py b/airflow/mopdb/LM/FORECAST/w_MOPDB_LM_FORECAST.py new file mode 100644 index 0000000..6f6ceba --- /dev/null +++ b/airflow/mopdb/LM/FORECAST/w_MOPDB_LM_FORECAST.py @@ -0,0 +1,149 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST = DbtTaskGroup( + group_id="m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + diff --git a/airflow/mopdb/LM/QRE_ADJUSTMENTS/w_MOPDB_LM_QRE_ADJUSTMENT.py b/airflow/mopdb/LM/QRE_ADJUSTMENTS/w_MOPDB_LM_QRE_ADJUSTMENT.py new file mode 100644 index 0000000..47c9da5 --- /dev/null +++ b/airflow/mopdb/LM/QRE_ADJUSTMENTS/w_MOPDB_LM_QRE_ADJUSTMENT.py @@ -0,0 +1,155 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT = DbtTaskGroup( + group_id="m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + + + + diff --git a/airflow/mopdb/LM/STANDING_FACILITY/w_MOPDB_LM_STANDING_FACILITY.py b/airflow/mopdb/LM/STANDING_FACILITY/w_MOPDB_LM_STANDING_FACILITY.py new file mode 100644 index 0000000..9d4c4aa --- /dev/null +++ b/airflow/mopdb/LM/STANDING_FACILITY/w_MOPDB_LM_STANDING_FACILITY.py @@ -0,0 +1,175 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES = DbtTaskGroup( + group_id="m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig( + select=["tag:m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES"], + exclude=["m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ"], + ), + operator_args=common_operator_args, + ) + + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + def trigger_devo_replicator(**context): + from airflow.api.common.trigger_dag import trigger_dag + + + target_dag_id = 'devo_replicator_trigger_mopdb' + + # Create configuration dictionary + trigger_conf = { + "owner_table": "LM.T_STANDING_FACILITIES" + } + try: + dag_run = trigger_dag( + dag_id=target_dag_id, + conf=trigger_conf, + execution_date=None, + replace_microseconds=False + ) + + return dag_run.run_id + except Exception as e: + raise + trigger_replicator = PythonOperator( + task_id='trigger_devo_replicator', + python_callable=trigger_devo_replicator, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES >> control_external_run_end >>trigger_replicator >> dag_status + +globals()[DAG_NAME] = run_dag() diff --git a/airflow/mopdb/RQSD/.gitkeep b/airflow/mopdb/RQSD/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/mopdb/RQSD/RQSD_PROCESS/.gitkeep b/airflow/mopdb/RQSD/RQSD_PROCESS/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS.py b/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS.py new file mode 100644 index 0000000..5ffe2cc --- /dev/null +++ b/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS.py @@ -0,0 +1,316 @@ +import os +###### +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.api.common.trigger_dag import trigger_dag +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +from mrds.utils.security_utils import get_verified_run_id, verify_run_id +from mrds.utils import oraconn + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + # Get verified run_id using security utilities + run_id = get_verified_run_id(kwargs) + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +def get_rqsd_tables_to_replicate(**kwargs): + """ + Get list of RQSD tables from a_devo_replica_mgmt_rqsd, + excluding tables ending with _COPY + """ + import logging + oracle_conn = None + try: + oracle_conn = oraconn.connect('MRDS_LOADER') + cursor = oracle_conn.cursor() + + # Query to get all tables excluding _COPY versions + sql = """ + SELECT OWNER, TABLE_NAME + FROM CT_MRDS.a_devo_replica_mgmt_rqsd + WHERE TABLE_NAME NOT LIKE '%_COPY' + ORDER BY OWNER, TABLE_NAME + """ + + cursor.execute(sql) + tables = cursor.fetchall() + cursor.close() + + logging.info(f"Found {len(tables)} RQSD tables to replicate (excluding _COPY versions)") + + # Convert to list of owner.table_name format + table_list = [f"{owner}.{table_name}" for owner, table_name in tables] + + # Push to XCom for next task + kwargs["ti"].xcom_push(key="rqsd_tables", value=table_list) + + return table_list + + except Exception as e: + logging.error(f"Error getting RQSD tables: {e}") + raise + finally: + if oracle_conn: + oracle_conn.close() + +def trigger_rqsd_replication(**kwargs): + """ + Trigger devo_replicator_trigger_rqsd for each table in the list + """ + import logging + from datetime import datetime + + ti = kwargs["ti"] + table_list = ti.xcom_pull(task_ids="get_rqsd_tables", key="rqsd_tables") + + if not table_list: + logging.warning("No RQSD tables found to replicate") + return + + logging.info(f"Triggering replication for {len(table_list)} tables") + + triggered_count = 0 + failed_triggers = [] + + for owner_table in table_list: + try: + conf = { + "owner_table": owner_table + } + + trigger_dag( + dag_id='devo_replicator_trigger_rqsd', + conf=conf, + execution_date=None, + replace_microseconds=False + ) + + triggered_count += 1 + logging.info(f"Successfully triggered replication for {owner_table}") + + except Exception as e: + logging.error(f"Failed to trigger replication for {owner_table}: {e}") + failed_triggers.append(owner_table) + + logging.info(f"Replication triggered for {triggered_count}/{len(table_list)} tables") + + if failed_triggers: + logging.warning(f"Failed to trigger replication for: {', '.join(failed_triggers)}") + + # Push results to XCom + ti.xcom_push(key="triggered_count", value=triggered_count) + ti.xcom_push(key="failed_triggers", value=failed_triggers) + + return { + "triggered_count": triggered_count, + "total_tables": len(table_list), + "failed_triggers": failed_triggers + } + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA = DbtTaskGroup( + group_id="m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA", + ]), + operator_args=common_operator_args, + ) + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # Get list of RQSD tables to replicate + get_rqsd_tables = PythonOperator( + task_id="get_rqsd_tables", + python_callable=get_rqsd_tables_to_replicate, + provide_context=True, + ) + + # Trigger replication for all RQSD tables + trigger_rqsd_replication_task = PythonOperator( + task_id="trigger_rqsd_replication", + python_callable=trigger_rqsd_replication, + provide_context=True, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # dependency chain + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> [ + m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS, + ] >> m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA >> control_external_run_end >> get_rqsd_tables >> trigger_rqsd_replication_task >> dag_status + +globals()[DAG_NAME] = run_dag() \ No newline at end of file diff --git a/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS_MANUAL.py b/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS_MANUAL.py new file mode 100644 index 0000000..4c682bc --- /dev/null +++ b/airflow/mopdb/RQSD/RQSD_PROCESS/w_MOPDB_RQSD_PROCESS_MANUAL.py @@ -0,0 +1,216 @@ +""" +NEW CHANGES FOR MANUAL: +Changed DAG_NAME from dynamic file-based to hardcoded "w_MOPDB_RQSD_PROCESS_MANUAL" +Changed the last task group from m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA to m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL +Updated the tag in the RenderConfig to "tag:m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL" +Added tags=["MOPDB", "RQSD", "DBT", "MANUAL"] to the DAG decorator + +KEEP IDENTICAL : +All other task groups remain the same as the original MOPDB DAG +Flow structure remains identical: +read_vars → retrieve_run_id → control_external_run_start +→ 5 parallel ANNEX tasks → MANUAL output task → control_external_run_end → dag_status +""" +import os + +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +from mrds.utils.security_utils import get_verified_run_id, verify_run_id + +DAG_NAME = "w_MOPDB_RQSD_PROCESS_MANUAL" +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = get_verified_run_id(kwargs) + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, + tags=["MOPDB", "RQSD", "DBT", "MANUAL"] +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS = DbtTaskGroup( + group_id="m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS", + ]), + operator_args=common_operator_args, + ) + + m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL = DbtTaskGroup( + group_id="m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL", + ]), + operator_args=common_operator_args, + ) + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> [ + m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS, + m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS, + ] >> m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() diff --git a/airflow/mopdb/TOP/.gitkeep b/airflow/mopdb/TOP/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/mopdb/TOP/AGGREGATED_ALLOTMENT/w_MOPDB_TOP_AGGREGATED_ALLOTMENT.py b/airflow/mopdb/TOP/AGGREGATED_ALLOTMENT/w_MOPDB_TOP_AGGREGATED_ALLOTMENT.py new file mode 100644 index 0000000..c4b40f8 --- /dev/null +++ b/airflow/mopdb/TOP/AGGREGATED_ALLOTMENT/w_MOPDB_TOP_AGGREGATED_ALLOTMENT.py @@ -0,0 +1,156 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT = DbtTaskGroup( + group_id="m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + + + + + diff --git a/airflow/mopdb/TOP/ALLOTMENT/w_MOPDB_TOP_ALLOTMENT.py b/airflow/mopdb/TOP/ALLOTMENT/w_MOPDB_TOP_ALLOTMENT.py new file mode 100644 index 0000000..74f9248 --- /dev/null +++ b/airflow/mopdb/TOP/ALLOTMENT/w_MOPDB_TOP_ALLOTMENT.py @@ -0,0 +1,156 @@ +import os +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +DAG_NAME = os.path.splitext(os.path.basename(__file__))[0] +ENV_NAME = os.getenv("MRDS_ENV", "").lower() + +DATABASE_NAME_MAP = { + "dev": "MOPDB", + "test": "MOPDB_TEST", +} +DATABASE_NAME = DATABASE_NAME_MAP.get(ENV_NAME, "MOPDB") + +# check cron 2 + +dbt_root_path = "/opt/dbt" +dbt_profiles_dir = "/opt/dbt/profiles.yml" +dbt_profiles_dir_parent = "/opt/dbt" + +dbt_env = { + "DBT_PROFILES_DIR": dbt_profiles_dir_parent, + "DBT_TARGET": ENV_NAME, + "MRDS_LOADER_DB_USER": os.getenv("MRDS_LOADER_DB_USER"), + "MRDS_LOADER_DB_PASS": os.getenv("MRDS_LOADER_DB_PASS"), + "MRDS_LOADER_DB_TNS": os.getenv("MRDS_LOADER_DB_TNS", "XE"), + "MRDS_SCHEMA": os.getenv("MRDS_SCHEMA", "CT_MRDS"), + "MRDS_PROTOCOL": os.getenv("MRDS_PROTOCOL", "tcps"), + "MRDS_THREADS": os.getenv("MRDS_THREADS", "4"), + + "DBT_LOG_PATH": "/opt/dbt/logs", + "DBT_TARGET_PATH": "/opt/dbt/target", + "PYTHONUNBUFFERED": "1", +} + +def retrieve_run_id(**kwargs): + run_id = kwargs["run_id"] + kwargs["ti"].xcom_push(key="run_id", value=run_id) + return run_id + +def check_dag_status(**kwargs): + for ti in kwargs["dag_run"].get_task_instances(): + if ti.state == "failed" and ti.task_id != kwargs["task_instance"].task_id: + raise Exception(f"Task {ti.task_id} failed. Failing this DAG run") + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False, +) +def run_dag(): + def read_vars(**context): + BUCKET = os.getenv("INBOX_BUCKET") + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE") + print("========= DBT ENV =========") + print(f"BUCKET_NAMESPACE: {BUCKET_NAMESPACE}, BUCKET : {BUCKET}") + return 1 + + read_vars_task = PythonOperator( + task_id="read_vars", + python_callable=read_vars, + provide_context=True, + ) + + retrieve_run_id_task = PythonOperator( + task_id="retrieve_run_id", + python_callable=retrieve_run_id, + provide_context=True, + ) + + control_external_run_start = BashOperator( + task_id="control_external_run_start", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_start \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + ) + + common_profile = ProfileConfig( + profiles_yml_filepath=dbt_profiles_dir, + profile_name="mrds", + target_name=ENV_NAME, + ) + common_project = ProjectConfig(dbt_project_path=dbt_root_path) + common_vars = { + "orchestration_run_id": "{{ ti.xcom_pull(task_ids='retrieve_run_id', key='run_id') }}", + "input_service_name": DATABASE_NAME, + "workflow_name": DAG_NAME, + } + common_operator_args = { + "vars": common_vars, + "env": dbt_env, + } + + m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT = DbtTaskGroup( + group_id="m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT", + project_config=common_project, + profile_config=common_profile, + render_config=RenderConfig(select=[ + "tag:m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT", + ]), + operator_args=common_operator_args, + ) + + + control_external_run_end = BashOperator( + task_id="control_external_run_end", + params={"db": DATABASE_NAME, "wf": DAG_NAME}, + env=dbt_env, + bash_command=""" + set -euxo pipefail + cd /opt/dbt + dbt --log-format json --log-level debug --debug --log-path /opt/dbt/logs \ + run-operation control_external_run_end \ + --vars '{{ { + "orchestration_run_id": ti.xcom_pull(task_ids="retrieve_run_id", key="run_id"), + "input_service_name": params.db, + "workflow_name": params.wf + } | tojson }}' + """, + trigger_rule=TriggerRule.ALL_DONE, + ) + + dag_status = PythonOperator( + task_id="dag_status", + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, + ) + + read_vars_task >> retrieve_run_id_task >> control_external_run_start >> m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT >> control_external_run_end >> dag_status + +globals()[DAG_NAME] = run_dag() + + + + + + + + + + diff --git a/airflow/ods/c2d/uc/config/common_ucdb.xsd b/airflow/ods/c2d/uc/config/common_ucdb.xsd new file mode 100644 index 0000000..655c278 --- /dev/null +++ b/airflow/ods/c2d/uc/config/common_ucdb.xsd @@ -0,0 +1,1534 @@ + + + + + + + + + + + + + Defines all meta information. + + + + + Marketable Assets + + + + + The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the asset. + + + + + Registration code. To be provided either by ISINCode or OtherRegistrationNumber, or both if both exist. + + + + + Nominal amount submitted before market valuation and application of the haircut. + + + + + Market value of the nominal amount submitted before application of the haircut. + + + + + Market value of the nominal amount submitted after application of the haircut. + + + + + Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + + Defines in what kind of operation the asset was used. Mandatory if system_type=earm + + + + + Defines if the asset was used on a domestic basis or on across border basis. + + + + + Potentially eligible asset information. To be completed only for potentially eligible assets (status H in EADB). + + + + + Free text field in which the NCB may explain any irregularities in the data. + + + + + + + + Non-Marketable Assets + + + + + The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the asset. + + + + + Unique code that can identify an asset. + + + + + Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + + Defines in what kind of operation the asset was used. Mandatory if system_type=earm. + + + + + Defines if the asset was used on a domestic basis or on across border basis. + + + + + The type of asset for non marketable assets. + + + + + The actual maturity date for individual collateral and the average or mid-value for aggregated records. + + + + + Type of interest payment can be fixed for fixed interest rates or floating for floating interest rates. + + + + + Defines the cap information of a credit claim. + + + + + Defines the reference rate of a credit claim from a set of codelist controlled values. + + + + + Free text field where NCBs can explain irregularities with regards reference rates of credit claims. + + + + + Market value of the nominal amount submitted before application of the haircut, expressed in EUR millions. + + + + + Market value of the nominal amount submitted after application of the haircut, expressed in EUR millions. + + + + + The number of debtors that have been aggregated together. Individual debtors are identified with a 1. + + + + + Indication whether the asset is eligible only via the guarantor or via the debtor. +Y=asset is only eligible via the guarantor. All guarantor and debtor fields should be completed. +N=asset is eligible via the debtor. All debtor fields should be completed. + + + + + Debtor information. To be provided for debtors with at least one asset above the threshold. + + + + + Guarantor information. To be provided only when eligible via guarantor. + + + + + The number of assets which have been aggregated together. + + + + + ISO4217 currency codes + + + + + Indicates if the asset is secured. +Possible values Y=Yes or N=No. +Mandatory if Nonmkt_asset_type=ACC + + + + + An interval of years to maturity expressed in years. + + + + + A range of outstanding amounts expressed in EUR millions. + + + + + Free text field in which the NCB may explain any irregularities in the data. + + + + + Defines whether the NCB assigned the price based on a theoretical price or the outstanding amount. + + + + + Nominal amount submitted before market valuation and application of the haircut. + + + + + Identifies when a floating rate claim has a resetting period for more than 1 year and needs to be treated as a fixed rate credit claim for the application of the haircut. + + + + + + Observed agent identifier + + + + + Contract identifier + + + + + Instrument identifier + + + + + + + Data on nonmarketable DECC + + + + + The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the intraday credit. + + + + + The International Securities Identification Number (ISIN) uniquely identifies an asset. + + + + + Nominal amount submitted before market valuation and application of the haircut. + + + + + Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + + Defines in what kind of operation the asset was used. Mandatory if system_type=earm. + + + + + Defines if the asset was used on a domestic basis or on across border basis. + + + + + Free text field in which the NCB may explain any irregularities in the data. + + + + + + The type of asset for non marketable assets. + + + + + + + Block for Debtor information. + + + + + Name of the debtor + + + + + Identifier of the debtor. + + + + + The class indicating how to consider the debtor regarding its rating. + + + + + Two letter ISO country code or three letter code AGG for pooled additional credit claims. For fixed term deposits the value is "EU". + + + + + Type of credit assessment source for the debtor. + + + + + Name of the actual organisation providing the credit assessment source for the debtor. + + + + + • Where Debtor_cred_assess_source = ECAI: enter the rating code as shown in the code list of the Eligible Assets Database (e.g. LIMY06 for a Moody’s long-term issuer rating of A2). +• Where Debtor_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Debtor_cred_assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Debtor_cred_assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: For records with Aggregated_no_of_debtors > 1, please enter the mid-point of the range of probabilities of default in percentage points (decimal format) with the amount of decimal digits needed, e.g. a range for a probability of default is 0.10%-0.05%, the reported figure would be the 0.00075. Use as many decimal places as deemed necessary . + + + + + + + + Central Bank + + + + + + + + Central Government + + + + + + + + Non Financial Corporation + + + + + + + + Credit institution (excluding Agency credit institution) + + + + + + + + Regional/local government + + + + + + + + Supranational + + + + + + + + Agency-non credit institution + + + + + + + + Agency-credit institution + + + + + + + + Financial Corporation + + + + + + + + Private Household + + + + + + + + Public Corporation + + + + + + + + Aggregated credit claim with multiple credit assessment sources + + + + + + + + Block for Guarantor information. + + + + + Name of the guarantor + + + + + Filled in with the identification number assigned by the NCB to the guarantor. + + + + + The class indicating how to consider the guarantor regarding its rating. + + + + + Two letter ISO country code or three letter code AGG for pooled additional credit claims. For fixed term deposits the value is "EU". + + + + + Type of credit assessment source for the guarantor. + + + + + Name of the actual organisation providing the credit assessment source for the guarantor. + + + + + • Where Guarantor_cred_assess_source = ECAI: enter the rating code as shown in the code list of the Eligible Assets Database (e.g. LIMY06 for a Moody’s long-term issuer rating of A2). +• Where Guarantor_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Guarantor_cred _assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Guarantor_cred _assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: For records with Aggregated_no_of_assets > 1, please enter the mid-point of the range of probabilities of default in percentage points (decimal format) with the amount of decimal digits needed, e.g. a range for a probability of default is 0.10%-0.05%, the reported figure would be the 0.00075. Use as many decimal places as deemed necessary . + + + + + + + + Type of the debtor/guarantor identifier + + + + + + + + Financial Vehicle Corporation + + + + + + + + + + + + + + + Monetary Financial Institute + + + + + + + + + + + + + + + ID from AVOX + + + + + + + + + + + + + + + + + National Central Bank + + + + + + + + + + + + European Central Bank + + + + + + + + + + + + Aggregated credit claim + + + + + + + + + + + + RIAD + + + + + + + + + + + + Central Bank + + + + + + + + Central Government + + + + + + + + Non Financial Corporation + + + + + + + + Credit institution (excluding Agency credit institution) + + + + + + + + Regional/local government + + + + + + + + Supranational + + + + + + + + Agency-non credit institution + + + + + + + + Agency-credit institution + + + + + + + + Financial Corporation + + + + + + + + Private Household + + + + + + + + Public Corporation + + + + + + + + Aggregated credit claim with multiple credit assessment sources + + + + + + + + + + For unrated assets (EADB status H) it gives the source of the credit assessment. + + + + + For unrated assets (EADB status H) it gives the name of the provider of the credit assessment. + + + + + For unrated assets (EADB status H) it gives the class of the issuer. + + + + + For unrated assets (EADB status H) : +• Where Issuer_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Issuer_cred _assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Issuer_cred _assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: In case only a range is available for a probability of default instead of a point estimate, please report the mid-point of the range (e.g. if a range for a probability of default is 0.0005-0.001, please report the 0.00075). Use as many decimal places as deemed necessary. + + + + + + + + + + + + + The International Securities Identification Number (ISIN) uniquely identifies an asset. + + + + + Unique code that can identify an asset. + + + + + + + + + + + + + + + + + + Defines all valuation methodologies. + + + + + price based on theoretical price + + + + + price based on outstanding amount + + + + + + + Defines a decimal number of 13 digits long, with max. 8 decimals. Caters for amounts between 0 million euro and 99999.99999999 million euro + + + + + + + + + + + Defines a decimal number of 14 digits long, with max. 9 decimals. Caters for amounts between 0 million euro and 999999.99999999 million euro + + + + + + + + + + + Defines a decimal number of 14-digit long, with max. 8 decimals. Caters for amounts between 0 million and 99999.99999999 million euro + + + + + + + + + + + Defines allowed values for outstanding amount size in EUR. + + + + + Outstanding amount smaller than or equal to EUR 100,000 + + + + + Outstanding amount bigger than EUR 100,000 + + + + + + + ISO4217 currency codes + + + + + + + + + + yes, the asset is secured. + + + + + no, the asset is not secured. + + + + + + + The class indicating how to consider the debtor regarding its rating. + + + + + PSE class1 (assessed as being equivalent to central government) + + + + + PSE class2 (assessed as being one credit quality step below the central government) + + + + + Private household + + + + + any other class + + + + + PSE class 3 or non-financial corporations (treated as a private sector issuer or debtor) + + + + + Pooled additional credit claim + + + + + + + Defines Guarantor class string type + + + + + PSE class1 (assessed as being equivalent to central government) + + + + + PSE class2 (assessed as being one credit quality step below the central government + + + + + any other class + + + + + PSE class 3 or non-financial corporations (treated as a private sector issuer or debtor) + + + + + Private household + + + + + Aggregated credit claim with multiple credit assessment sources + + + + + + + Defines Guarantor/Debtor credit assessment source string type + + + + + External credit assessment institution source + + + + + NCB in-house credit assessment system source + + + + + Counterparties internal ratings based system source + + + + + Implicit credit assessments for public sector entities + + + + + Third party rating tool source + + + + + Aggregated credit claim with multiple credit assessment sources + + + + + + + Type of interest payment can be fixed for fixed interest rates or floating for floating interest rates. + + + + + Fixed interest payment + + + + + Floating interest payment + + + + + + + Defines Issuer class string type + + + + + PSE class 1 + + + + + PSE class 2 + + + + + PSE class 3 and NFC (non-financial corporations) + + + + + + + Defines Issuer credit assessment source string type + + + + + NCB in-house credit assessment system source + + + + + Counterparties internal ratings-based system source + + + + + Implicit credit assessments for public sector entities + + + + + Third-party rating tool source + + + + + + + Defines a string pattern for ISIN code. Must be exactly 12 characters long, where two first characters should be A-Z, followed by 10 characters fo A-Z or 0-9, or 2). + + + + + + + + Defines max. 200 characters long string. + + + + + + + + + Defines max. 255 characters long string. + + + + + + + + + Defines max. 500 characters long string. + + + + + + + + + Defines a string pattern for MFI Id. Restricted to max 256 characters long. + - The rule for this simpleType has been changed in June 2017. + + + + + + + + Defines a string pattern for Riad Code. Restricted to max 256 characters long. + - The rule for this simpleType has been changed in June 2017. + + + + + + + + The type of asset for non marketable assets. + + + + + Credit claim or bank loan + + + + + Retail mortgage-backed debt instrument. Includes Irish MBPN + + + + + Fixed term deposit + + + + + Additional Credit Claim + + + + + Cash deposit + + + + + Nonmarketable Debt instruments backed by Eligible Credit Claims + + + + + + + An interval of years to maturity expressed in years. + + + + + Residual maturity not longer than one year + + + + + Residual maturity greater than one year up to three years + + + + + Residual maturity greater than three years up to five years + + + + + Residual maturity greater than five years up to seven years + + + + + Residual maturity greater than seven years up to ten years + + + + + Residual maturity greater than ten years up to fifteen years + + + + + Residual maturity greater than fifteen years up to thirty years + + + + + Residual maturity greater than thirty years + + + + + + + Defines type of operation + + + + + Main refinancing operation + + + + + Longer term refinancing operation + + + + + Fine-tuning operation + + + + + Marginal lending facility + + + + + Intra-day credit + + + + + Monetary Policy Operation + + + + + Other liquidity providing operations, e.g. in US dollars + + + + + + + Defines type of system with the following codes: earm, pool + + + + + Asset was used in an earmarking system + + + + + Asset was used in a pooling system + + + + + + + Defines type of transaction with the following codes: domestic, cross-border + + + + + Asset was used on a domestic basis + + + + + Asset was used on a cross-border basis + + + + + + + System facilitating the counterparty's subscription to IDC + + + + + Proprietary Home Accoung + + + + + Target2 + + + + + Securities Settlement System + + + + + + + Defines an integer number greater than 0 (i.e minimum value =1) and smaller than/equal to 9999999999 + + + + + + + + Defines the referenced filetype. + + + + + Initial file containing normal use of collateral + + + + + Final initial file filled in by NCBs to update suspect records for normal use of collateral file + + + + + Revision file containing suspect records for normal use of collateral file + + + + + Dissemination file containing normal use of collateral + + + + + ELA file with outstanding credit and use of collateral + + + + + + + + + + + + + + + Denmark + + + + + + + Defines a string pattern for the residence. + + + + + + + + Defines max. 4 characters long string. + + + + + + + + + Defines max. 30 characters long string. + + + + + + + + + Defines max. 50 characters long string + + + + + + + + + Defines max. 60 characters long string + + + + + + + + + Defines max. 100 characters long string. + + + + + + + + + Defines max. 256 characters long string. + + + + + + + + + Defines the format of ORN string for nonmarketable assets. + + + + + + + + + + + + + Addon for initial file records + + + + + A short code describing the type of suspect record found. + + + + + A message describing the suspect record found. + + + + + Shows which row and column in the initial/revision file has caused a suspect record. + + + + + + + + + + + + + + + The type of channel used to mobilise collateral. + + + + + Correspondent Central Bank. + + + + + The CSD (central securities depository) for the investor. + + + + + The CSD (central securities depository) for the intermediary. + + + + + The CSD (central securities depository) for the issuer. + + + + + + + + + + + The agent used for mobilisation. + + + + + + + + + + + + + + + Defines a decimal number of 9 digits long, with max. 8 decimals. Caters for values between 0 and 0.99999999 + + + + + + + + + + + + + + + + + + Cap exists for the credit claim. + + + + + Cap does not exist for the credit claim. + + + + + + + + Defines the format of OaId string for non-marketable assets. + + + + + + + + + Defines the format of ContractId string for non-marketable assets. + + + + + + + + + Defines the format of InstrmntId string for non-marketable assets. + + + + + + + diff --git a/airflow/ods/c2d/uc/config/disseminationFile.xsd b/airflow/ods/c2d/uc/config/disseminationFile.xsd new file mode 100644 index 0000000..a88774a --- /dev/null +++ b/airflow/ods/c2d/uc/config/disseminationFile.xsd @@ -0,0 +1,124 @@ + + + + + + + + + Use of Collateral message. Dissemination files hold all reported usages of a snapshot and a specific NCB. + + + + + + + + + + + + + + + + + + The two letter code identifying the NCB contained in the dissemination file. + + + + + The date the snapshot of the initial file data was taken. + + + + + The date when the dissemination file has been created. + + + + + The number of records in the dissemination file still in status suspect. + + + + + + + + + Holds all marketable assets. + + + + + + + + Holds all non marketable assets. + + + + + + + + Holds all non marketable DECCs. + + + + + + + + Marketable Assets with suspect addon + + + + + + + + + + + + Non Marketable Assets with suspect addon + + + + + + + + + + + + Non Marketable DECCs with suspect addon + + + + + + + + + + + + Holds all suspect information of a reported usage. + + + + + An ID created by the UCDB system identifying each single record that is stored in the UCDB system as a suspect record. + + + + + + diff --git a/airflow/ods/c2d/uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml b/airflow/ods/c2d/uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml new file mode 100644 index 0000000..2ad1c62 --- /dev/null +++ b/airflow/ods/c2d/uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml @@ -0,0 +1,45 @@ +# Global configurations +inbox_prefix: INBOX/C2D/CollateralDataDissemination +archive_prefix: ARCHIVE/C2D/CollateralDataDissemination +workflow_name: w_ODS_C2D_UC_DISSEMI +validation_schema_path: 'disseminationFile.xsd' +file_type: xml + +# List of tasks + +tasks: + + # Task 1 + - task_name: m_ODS_C2D_UC_DISSEM_METADATA_PARSE + ods_prefix: INBOX/C2D/CollateralDataDissemination/C2D_A_UC_DISSEM_METADATA_LOADS + output_table: C2D_A_UC_DISSEM_METADATA_LOADS + namespaces: + ns: 'http://c2d.escb.eu/UseOfCollateralMessage' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns:DisseminationFile/@version' + column_header: 'C2D_VERSION' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:DateCreated' + column_header: 'FILE_CREATION_DATE' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:NumberOfSuspectRecords' + column_header: 'NO_OF_SUSPECT_RECORDS' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:ReportingNCB' + column_header: 'REPORTING_NCB' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:SnapshotDate' + column_header: 'SNAPSHOT_DATE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'PROCESSED_TO_DWH' \ No newline at end of file diff --git a/airflow/ods/c2d/uc/config/ucdb_cl.xsd b/airflow/ods/c2d/uc/config/ucdb_cl.xsd new file mode 100644 index 0000000..bd7d038 --- /dev/null +++ b/airflow/ods/c2d/uc/config/ucdb_cl.xsd @@ -0,0 +1,968 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd b/airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd new file mode 100644 index 0000000..f93cd7c --- /dev/null +++ b/airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd @@ -0,0 +1,2412 @@ + + + + +Aggregated credit claim with multiple credit assessment sources + + + + + + + + + + + +Use of Collateral message. Dissemination files hold all reported usages of a snapshot and a specific NCB. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +For unrated assets (EADB status H) it gives the source of the credit assessment. + + + + +For unrated assets (EADB status H) it gives the name of the provider of the credit assessment. + + + + +For unrated assets (EADB status H) it gives the class of the issuer. + + + + +For unrated assets (EADB status H) : +• Where Issuer_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Issuer_cred _assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Issuer_cred _assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: In case only a range is available for a probability of default instead of a point estimate, please report the mid-point of the range (e.g. if a range for a probability of default is 0.0005-0.001, please report the 0.00075). Use as many decimal places as deemed necessary. + + + + + + + + +Holds all marketable assets. + + + + + + + + +Holds all suspect information of a reported usage. + + + + +An ID created by the UCDB system identifying each single record that is stored in the UCDB system as a suspect record. + + + + +Possible values are S=suspect, C=confirm, U=update. + + + + + + + + +Block for Debtor information. + + + + +Name of the debtor + + + + +Identifier of the debtor. + + + + +The class indicating how to consider the debtor regarding its rating. + + + + +Two letter ISO country code or three letter code AGG for pooled additional credit claims. For fixed term deposits the value is "EU". + + + + +Type of credit assessment source for the debtor. + + + + +Name of the actual organisation providing the credit assessment source for the debtor. + + + + +• Where Debtor_cred_assess_source = ECAI: enter the rating code as shown in the code list of the Eligible Assets Database (e.g. LIMY06 for a Moody’s long-term issuer rating of A2). +• Where Debtor_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Debtor_cred_assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Debtor_cred_assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: For records with Aggregated_no_of_debtors > 1, please enter the mid-point of the range of probabilities of default in percentage points (decimal format) with the amount of decimal digits needed, e.g. a range for a probability of default is 0.10%-0.05%, the reported figure would be the 0.00075. Use as many decimal places as deemed necessary . + + + + + + + + + + + + + + + + + +Block for Guarantor information. + + + + +Name of the guarantor + + + + +Filled in with the identification number assigned by the NCB to the guarantor. + + + + +The class indicating how to consider the guarantor regarding its rating. + + + + +Two letter ISO country code or three letter code AGG for pooled additional credit claims. For fixed term deposits the value is "EU". + + + + +Type of credit assessment source for the guarantor. + + + + +Name of the actual organisation providing the credit assessment source for the guarantor. + + + + +• Where Guarantor_cred_assess_source = ECAI: enter the rating code as shown in the code list of the Eligible Assets Database (e.g. LIMY06 for a Moody’s long-term issuer rating of A2). +• Where Guarantor_cred_assess_source = ICAS: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. Alternatively, enter the rating code used internally. In this case, a code list that translates the code into a probability of default must also be provided. +• Where Guarantor_cred _assess_source = IRB: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. +• Where Guarantor_cred _assess_source = RT: enter the probability of default in percentage points (decimal format) with the amount of decimal digits needed, i.e. 1.23% would be reported as 0.0123. + +NOTE: For records with Aggregated_no_of_assets > 1, please enter the mid-point of the range of probabilities of default in percentage points (decimal format) with the amount of decimal digits needed, e.g. a range for a probability of default is 0.10%-0.05%, the reported figure would be the 0.00075. Use as many decimal places as deemed necessary . + + + + + + + + +Supranational + + + + + + + + + + + + + + + + + +Defines all meta information. + + + + + + + +Regional/local government + + + + + + + + + + + +Non-Marketable Assets + + + + +The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the asset. + + + + +Unique code that can identify an asset. + + + + +Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + +Defines in what kind of operation the asset was used. Mandatory if system_type=earm. + + + + +Defines if the asset was used on a domestic basis or on across border basis. + + + + +The type of asset for non marketable assets. + + + + +The actual maturity date for individual collateral and the average or mid-value for aggregated records. + + + + +Type of interest payment can be fixed for fixed interest rates or floating for floating interest rates. + + + + +Defines the cap information of a credit claim. + + + + +Defines the reference rate of a credit claim from a set of codelist controlled values. + + + + +Free text field where NCBs can explain irregularities with regards reference rates of credit claims. + + + + +Market value of the nominal amount submitted before application of the haircut, expressed in EUR millions. + + + + +Market value of the nominal amount submitted after application of the haircut, expressed in EUR millions. + + + + +The number of debtors that have been aggregated together. Individual debtors are identified with a 1. + + + + +Indication whether the asset is eligible only via the guarantor or via the debtor. +Y=asset is only eligible via the guarantor. All guarantor and debtor fields should be completed. +N=asset is eligible via the debtor. All debtor fields should be completed. + + + + +Debtor information. To be provided for debtors with at least one asset above the threshold. + + + + +Guarantor information. To be provided only when eligible via guarantor. + + + + +The number of assets which have been aggregated together. + + + + +ISO4217 currency codes + + + + +Indicates if the asset is secured. +Possible values Y=Yes or N=No. +Mandatory if Nonmkt_asset_type=ACC + + + + +An interval of years to maturity expressed in years. + + + + +A range of outstanding amounts expressed in EUR millions. + + + + +Free text field in which the NCB may explain any irregularities in the data. + + + + +Defines whether the NCB assigned the price based on a theoretical price or the outstanding amount. + + + + +Nominal amount submitted before market valuation and application of the haircut. + + + + +Identifies when a floating rate claim has a resetting period for more than 1 year and needs to be treated as a fixed rate credit claim for the application of the haircut. + + + + + +Observed agent identifier + + + + +Contract identifier + + + + +Instrument identifier + + + + + + + +Central Bank + + + + + + + + + + + +Marketable Assets with suspect addon + + + + + + + + + + + + +Marketable Assets + + + + +The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the asset. + + + + +Registration code. To be provided either by ISINCode or OtherRegistrationNumber, or both if both exist. + + + + +Nominal amount submitted before market valuation and application of the haircut. + + + + +Market value of the nominal amount submitted before application of the haircut. + + + + +Market value of the nominal amount submitted after application of the haircut. + + + + +Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + +Defines in what kind of operation the asset was used. Mandatory if system_type=earm + + + + +Defines if the asset was used on a domestic basis or on across border basis. + + + + +Potentially eligible asset information. To be completed only for potentially eligible assets (status H in EADB). + + + + +Free text field in which the NCB may explain any irregularities in the data. + + + + + + + + +Type of the debtor/guarantor identifier + + + + + + + + + +Central Government + + + + + + + + + + + +Public Corporation + + + + + + + + + + + +Financial Vehicle Corporation + + + + + + + + + + + + + +Credit institution (excluding Agency credit institution) + + + + + + + + + + + +Financial Corporation + + + + + + + + + + + +Private Household + + + + + + + + + + + + + + + + +Agency-credit institution + + + + + + + + + + + +Non Financial Corporation + + + + + + + + + + + +Central Government + + + + + + + + + + + +Data on intraday credit at counterparty level + + + + +The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the intraday credit. + + + + +Peak intraday credit used by the participant on the specified snapshot date through a specific IDC source in EUR million. + + + + +Credit line of the participant for IDC on the specified snapshot date through a specific IDC source in EUR million. + + + + +The source indicating from where the intraday credit came, either PHA=Proprietary Home Account, SSS=Securities Settlement System, T2=Target2. + + + + +Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + +Free text field in which the NCB may explain any irregularities in the data. + + + + + + + +Data on nonmarketable DECC + + + + +The MFI ID is a unique identifier of the institution (Monetary Financial Institution) that is using the intraday credit. + + + + +The International Securities Identification Number (ISIN) uniquely identifies an asset. + + + + +Nominal amount submitted before market valuation and application of the haircut. + + + + +Type of system that the NCB is using to report the asset. Possible values are pool (for pooling system) or earm (for earmark system). + + + + +Defines in what kind of operation the asset was used. Mandatory if system_type=earm. + + + + +Defines if the asset was used on a domestic basis or on across border basis. + + + + +Free text field in which the NCB may explain any irregularities in the data. + + + + + +The type of asset for non marketable assets. + + + + + + + + + + + +The International Securities Identification Number (ISIN) uniquely identifies an asset. + + + + +Unique code that can identify an asset. + + + + + + + + + +Agency-non credit institution + + + + + + + + + + + +Addon for initial file records + + + + +A short code describing the type of suspect record found. + + + + +A message describing the suspect record found. + + + + +Shows which row and column in the initial/revision file has caused a suspect record. + + + + + + + + + + + + +Credit institution (excluding Agency credit institution) + + + + + + + + + + + +Agency-credit institution + + + + + + + + + + + +Non Marketable Assets with suspect addon + + + + + + + + + + + + +ID from AVOX + + + + + + + + + + + + + +Non Financial Corporation + + + + + + + + + + + +Non Marketable Assets with suspect addon + + + + + + + + + + + + +Agency-non credit institution + + + + + + + + + + + +Financial Corporation + + + + + + + + + + + +Holds all intraday credits. + + + + + + + + +Private Household + + + + + + + + + + + +Holds all non marketable DECCs. + + + + + + + + +Supranational + + + + + + + + + + + +National Central Bank + + + + + + + + + + + + + + + + + +The two letter code identifying the NCB contained in the dissemination file. + + + + +The date the snapshot of the initial file data was taken. + + + + +The date when the dissemination file has been created. + + + + +The number of records in the dissemination file still in status suspect. + + + + + + + + + +Aggregated credit claim + + + + + + + + + + + + + +Regional/local government + + + + + + + + + + + +Monetary Financial Institute + + + + + + + + + + + + + +RIAD + + + + + + + + + + + + + +Central Bank + + + + + + + + + + + +Aggregated credit claim with multiple credit assessment sources + + + + + + + + + + + + + + +The agent used for mobilisation. + + + + + + + +Public Corporation + + + + + + + + + + + + + +The type of channel used to mobilise collateral. + + + + +Correspondent Central Bank. + + + + +The CSD (central securities depository) for the investor. + + + + +The CSD (central securities depository) for the intermediary. + + + + +The CSD (central securities depository) for the issuer. + + + + + + + +Holds all non marketable assets. + + + + + + + + +Non Marketable DECCs with suspect addon + + + + + + + + + + + + +European Central Bank + + + + + + + + + + + + + + + + + + + + + +Defines a decimal number of 9 digits long, with max. 8 decimals. Caters for values between 0 and 0.99999999 + + + + + + + + + + +Defines a decimal number of 13 digits long, with max. 8 decimals. Caters for amounts between 0 million euro and 99999.99999999 million euro + + + + + + + + + + +Defines a decimal number of 14 digits long, with max. 9 decimals. Caters for amounts between 0 million euro and 999999.99999999 million euro + + + + + + + + + + +Defines a decimal number of 14-digit long, with max. 8 decimals. Caters for amounts between 0 million and 99999.99999999 million euro + + + + + + + + + + +Defines an integer number greater than 0 (i.e minimum value =1) and smaller than/equal to 9999999999 + + + + + + + +Defines max. 256 characters long string. + + + + + + + + + + + + + + + + + + + + + +Defines the referenced filetype. + + + + + + + + + + + + + + +Defines a string pattern for Riad Code. Restricted to max 256 characters long.- The rule for this simpleType has been changed in June 2017. + + + + + + + +Defines max. 4 characters long string. + + + + + + + + +Possible values are S=suspect, C=confirm, U=update + + + + + + + + + + + + + + + +Defines Guarantor class string type + + + + + + + + + + + + +Type of interest payment can be fixed for fixed interest rates or floating for floating interest rates. + + + + + + + + +Defines the format of ORN string for nonmarketable assets. + + + + + + + + + + + + + + + + + +Defines max. 100 characters long string. + + + + + + + + + + + + + +Defines a string pattern for ISIN code. Must be exactly 12 characters long, where two first characters should be A-Z, followed by 10 characters fo A-Z or 0-9, or 2). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines type of system with the following codes: earm, pool + + + + + + + + +The type of asset for non marketable assets. + + + + + + + + + + + + +Defines type of operation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines a string pattern for MFI Id. Restricted to max 256 characters long.- The rule for this simpleType has been changed in June 2017. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines Issuer class string type + + + + + + + + + +Defines Guarantor/Debtor credit assessment source string type + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines type of transaction with the following codes: domestic, cross-border + + + + + + + + +Defines max. 50 characters long string + + + + + + + + +The class indicating how to consider the debtor regarding its rating. + + + + + + + + + + + + +Defines the format of OaId string for non-marketable assets. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines Issuer credit assessment source string type + + + + + + + + + + +Defines a string pattern for the residence. + + + + + + + +Defines max. 500 characters long string. + + + + + + + + +Defines max. 30 characters long string. + + + + + + + + +Defines max. 200 characters long string. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Defines max. 60 characters long string + + + + + + + + +Defines the format of ContractId string for non-marketable assets. + + + + + + + +Defines the format of InstrmntId string for non-marketable assets. + + + + + + + +Defines allowed values for outstanding amount size in EUR. + + + + + + + + +Defines all valuation methodologies. + + + + + + + + +An interval of years to maturity expressed in years. + + + + + + + + + + + + + + +ISO4217 currency codes + + + + + + + +System facilitating the counterparty's subscription to IDC + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml b/airflow/ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml new file mode 100644 index 0000000..9757588 --- /dev/null +++ b/airflow/ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml @@ -0,0 +1,386 @@ +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/C2D/UC_DISSEM +workflow_name: w_OU_C2D_UC_DISSEM +validation_schema_path: '/opt/airflow/src/airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd' +bucket: mrds_inbox_tst +file_type: xml + +# List of tasks +tasks: + - task_name: t_ODS_C2D_UC_DISSEM_create_metadata_file + ods_prefix: INBOX/C2D/UC_DISSEM/A_UC_DISSEM_METADATA_LOADS + output_table: A_UC_DISSEM_METADATA_LOADS + namespaces: + ns: 'http://c2d.escb.eu/UseOfCollateralMessage' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns:DisseminationFile/@version' + column_header: 'C2D_VERSION' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:DateCreated' + column_header: 'FILE_CREATION_DATE' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:NumberOfSuspectRecords' + column_header: 'NO_OF_SUSPECT_RECORDS' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:ReportingNCB' + column_header: 'REPORTING_NCB' + is_key: 'N' + - type: 'xpath' + value: '//ns:MetaInformation/ns:SnapshotDate' + column_header: 'SNAPSHOT_DATE' + is_key: 'N' + - type: 'static' + value: 'N' + column_header: 'PROCESSED_TO_DWH' + + - task_name: ou_C2D_UC_DISSEM_create_marketable_assets_file + ods_prefix: INBOX/C2D/UC_DISSEM/UC_MA_DISSEM + output_table: UC_MA_DISSEM + namespaces: + ns: 'http://c2d.escb.eu/UseOfCollateralMessage' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns:MetaInformation/ns:ReportingNCB' + is_key: 'Y' + column_header: 'REPORTING_NCB' + - type: 'xpath' + value: '//ns:MetaInformation/ns:SnapshotDate' + is_key: 'Y' + column_header: 'SNAPSHOT_DATE' + - type: 'xpath' + value: '//ns:MetaInformation/ns:DateCreated' + column_header: 'FILE_CREATION_DATE' + is_key: 'Y' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:MFIId' + column_header: 'MFI_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:RegistrationCode/ns:ISINCode' + column_header: 'ISIN_CODE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'OTHER_REG_NO' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:NominalAmountSubmitted' + column_header: 'NOM_AMT_SUBMITTED' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CollateralValueBeforeHaircuts' + column_header: 'COLL_BEFORE_HAIRCUTS' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CollateralValueAfterHaircuts' + column_header: 'COLL_AFTER_HAIRCUTS' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:TypeOfSystem' + column_header: 'TYPE_OF_SYSTEM' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'TYPE_OF_OPERATION' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:DomesticOrXborder' + column_header: 'DOM_OR_XBORDER' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'ISSUER_CAS' + - type: 'static' + value: '' + column_header: 'ISSUER_CRED_PROVIDER' + - type: 'static' + value: '' + column_header: 'ISSUER_CLASS' + - type: 'static' + value: '' + column_header: 'ISSUER_RATING_ENUM_VALUE' + - type: 'static' + value: '' + column_header: 'ISSUER_RATING_NUMBER_VALUE' + - type: 'static' + value: '' + column_header: 'NCB_COMMENT' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:MobilisationChannel' + column_header: 'MOBILISATION_CHANNEL' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CCB' + column_header: 'CCB' + is_key: 'N' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:InvestorSSS' + column_header: 'INVESTOR_SSS' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'INTERMEDIARY_SSS' + - type: 'xpath' + value: '//ns:MarketableAssets/ns:MarketableAsset/ns:IssuerSSS' + column_header: 'ISSUER_SSS' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'TRIPARTY_AGENT' + - type: 'static' + value: '' + column_header: 'SUSPECT_ID' + - type: 'static' + value: '' + column_header: 'QUALITY_CHECK_STATUS' + - type: 'static' + value: '' + column_header: 'ERROR_CODE' + - type: 'static' + value: '' + column_header: 'ERROR_MESSAGE' + - type: 'static' + value: '' + column_header: 'ERROR_POSITION_IN_FILE' + + - task_name: ou_C2D_UC_DISSEM_create_nonmarketable_assets_file + ods_prefix: INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM + output_table: UC_NMA_DISSEM + namespaces: + ns: 'http://c2d.escb.eu/UseOfCollateralMessage' + xsi: 'http://www.w3.org/2001/XMLSchema-instance' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns:MetaInformation/ns:ReportingNCB' + is_key: 'Y' + column_header: 'REPORTING_NCB' + - type: 'xpath' + value: '//ns:MetaInformation/ns:SnapshotDate' + is_key: 'Y' + column_header: 'SNAPSHOT_DATE' + - type: 'xpath' + value: '//ns:MetaInformation/ns:DateCreated' + column_header: 'FILE_CREATION_DATE' + is_key: 'Y' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:MFIId' + column_header: 'MFI_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:OtherRegistrationNumber' + column_header: 'OTHER_REG_NO' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:TypeOfSystem' + column_header: 'TYPE_OF_SYSTEM' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'TYPE_OF_OPERATION' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:DomesticOrXborder' + column_header: 'DOM_OR_XBORDER' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NonMktAssetType' + column_header: 'NON_MKT_ASSET_TYPE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:DateOfMaturity' + column_header: 'MATURITY_DATE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:InterestPaymentType' + column_header: 'INTEREST_PAYMENT_TYPE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Cap' + column_header: 'CAP' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ReferenceRate' + column_header: 'REFERENCE_RATE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'REFERENCE_RATE_COMMENT' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CollateralValueBeforeHaircuts' + column_header: 'COLL_BEFORE_HAIRCUTS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CollateralValueAfterHaircuts' + column_header: 'COLL_AFTER_HAIRCUTS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NumberOfAggregatedDebtors' + column_header: 'NO_AGGR_DEBTORS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:EligibleViaGuarantor' + column_header: 'ELIGIBLE_VIA_GUAR' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/@xsi:type' + column_header: 'DEBTOR_TYPE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Name' + column_header: 'DEBTOR_NAME' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:ID/@xsi:type' + column_header: 'DEBTOR_ID_TYPE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:ID/ns:value' + column_header: 'DEBTOR_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Class' + column_header: 'DEBTOR_CLASS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Residence' + column_header: 'DEBTOR_RESIDENCE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:CreditAssessmentSource' + column_header: 'DEBTOR_CAS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:CredAssessSysProvider' + column_header: 'DEBTOR_CRED_PROV' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'DEBTOR_RATING_ENUM_VALUE' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Rating/ns:NumberValue' + column_header: 'DEBTOR_RATING_NUMBER_VALUE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'GUAR_TYPE' + - type: 'static' + value: '' + column_header: 'GUAR_NAME' + - type: 'static' + value: '' + column_header: 'GUAR_ID_TYPE' + - type: 'static' + value: '' + column_header: 'GUAR_ID' + - type: 'static' + value: '' + column_header: 'GUAR_CLASS' + - type: 'static' + value: '' + column_header: 'GUAR_RESIDENCE' + - type: 'static' + value: '' + column_header: 'GUAR_CRED_CAS' + - type: 'static' + value: '' + column_header: 'GUAR_CRED_PROV' + - type: 'static' + value: '' + column_header: 'GUAR_RATING_ENUM_VALUE' + - type: 'static' + value: '' + column_header: 'GUAR_RATING_NUMBER_VALUE' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NumberOfAggregatedAssets' + column_header: 'NO_AGGR_ASSETS' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Denomination' + column_header: 'DENOMINATION' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Secured' + column_header: 'SECURED_FLAG' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ResidualMaturity' + column_header: 'RESIDUAL_MATURITY' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:BucketSize' + column_header: 'BUCKET_SIZE' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NCBComment' + column_header: 'NCB_COMMENT' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ValuationMethodology' + column_header: 'VALUATION_METHODOLOGY' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NominalAmountSubmitted' + column_header: 'NOM_AMT_SUBMITTED' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ResettingPeriodMoreThanOneYear' + column_header: 'RESET_PERIOD_MORE_ONE_YEAR' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:MobilisationChannel' + column_header: 'MOBILISATION_CHANNEL' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CCB' + column_header: 'CCB' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'INVESTOR_SSS' + - type: 'static' + value: '' + column_header: 'INTERMEDIARY_SSS' + - type: 'static' + value: '' + column_header: 'ISSUER_SSS' + - type: 'static' + value: '' + column_header: 'SUSPECT_ID' + - type: 'static' + value: '' + column_header: 'QUALITY_CHECK_STATUS' + - type: 'static' + value: '' + column_header: 'ERROR_CODE' + - type: 'static' + value: '' + column_header: 'ERROR_MESSAGE' + - type: 'static' + value: '' + column_header: 'ERROR_POSITION_IN_FILE' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:OaId' + column_header: 'OA_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ContractId' + column_header: 'CONTRACT_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:InstrmntId' + column_header: 'INSTRMNT_ID' + is_key: 'N' \ No newline at end of file diff --git a/airflow/ods/c2d/uc/test_data/MOC-CollateralDataDissemination-UCDB_dissemination_DISS-FR-2023-02-16-34270_example.xml b/airflow/ods/c2d/uc/test_data/MOC-CollateralDataDissemination-UCDB_dissemination_DISS-FR-2023-02-16-34270_example.xml new file mode 100644 index 0000000..ca031c8 --- /dev/null +++ b/airflow/ods/c2d/uc/test_data/MOC-CollateralDataDissemination-UCDB_dissemination_DISS-FR-2023-02-16-34270_example.xml @@ -0,0 +1,179 @@ + + + + FR + 2023-02-16 + 2023-02-23T11:00:35 + 0 + + + + FR10107 + + BE6302866973 + + 40 + 41.92566012 + 40.75174164 + pool + cross-border + CCBM mkt + BE + CLBE01 + CLBE01 + + + FR10107 + + DE000A1RQCP0 + + 10 + 10.2664863 + 10.06115657 + pool + cross-border + CCBM mkt + DE + CLDE01 + CLDE01 + + + FR10107 + + ES0305248009 + + 7.5753425 + 7.31191527 + 6.78545737 + pool + cross-border + CCBM mkt + ES + CLES01 + CLES01 + + + + + FR10107 + FRCPACAGGREGATE + pool + domestic + Credit claim + 2023-07-08 + Fixed + 100 + 88.5 + 2 + N + + Aggregate + + Aggregate + + NFC-PSE3 + FR + ICAS + Banque de France + + 0.0003 + + + 2 + EUR + N + 1-3 + 0_to_100 + 10107 + Outstanding + 10 + Local dom nonmkt + FR + 10107 + 1549493 + 1549493 + + + FR11188 + FRC000748968616 + pool + domestic + ACC + 2023-09-30 + Floating + N + 3MEUBOR + 200 + 160 + 1 + N + + SAS UNIP SAINT MALO AUTOMOBILES DISTRIBUTION + + FR895780419 + + Other + FR + ICAS + Banque de France + + 0.0051 + + + 1 + EUR + N + 10-15 + 11188 + Outstanding + 999999.99999999 + N + Local dom nonmkt + FR + 11188 + R05020ETC + 202095459110 + + + FR11188 + FRC000748968732 + pool + domestic + ACC + 2023-09-30 + Floating + N + 3MEUBOR + 300 + 201 + 1 + N + + ALLIANCE E.S.D.B + + FR347861981 + + Other + FR + ICAS + Banque de France + + 0.0051 + + + 1 + EUR + N + +30 + 11188 + Outstanding + 0 + N + Local dom nonmkt + FR + 11188 + R05320ETC + 202095459010 + + + diff --git a/airflow/ods/csdb/debt/.gitkeep b/airflow/ods/csdb/debt/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/csdb/debt/config/.gitkeep b/airflow/ods/csdb/debt/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/csdb/debt/config/m_ODS_CSDB_DEBT_PARSE.yaml b/airflow/ods/csdb/debt/config/m_ODS_CSDB_DEBT_PARSE.yaml new file mode 100644 index 0000000..a66f9d2 --- /dev/null +++ b/airflow/ods/csdb/debt/config/m_ODS_CSDB_DEBT_PARSE.yaml @@ -0,0 +1,398 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/CentralizedSecuritiesDissemination +archive_prefix: ARCHIVE/CSDB/CentralizedSecuritiesDissemination +workflow_name: w_ODS_CSDB_DEBT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_DEBT_PARSE + ods_prefix: INBOX/CSDB/CentralizedSecuritiesDissemination/CSDB_DEBT + output_table: CSDB_DEBT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'Date last modified' + column_header: 'NEWUPDATED' + - type: 'csv_header' + value: 'Extraction date' + column_header: 'IDLOADDATE_DIM' + - type: 'csv_header' + value: 'ISIN code' + column_header: 'EXTERNALCODE_ISIN' + - type: 'csv_header' + value: 'National instrument code type' + column_header: 'EXTERNALCODETYPE_NC' + - type: 'csv_header' + value: 'National instrument code' + column_header: 'EXTERNALCODE_NATIONAL' + - type: 'csv_header' + value: 'Internal instrument code' + column_header: 'IDIRINSTRUMENT' + - type: 'csv_header' + value: 'Short name' + column_header: 'SHORTNAME' + - type: 'csv_header' + value: 'Bond duration' + column_header: 'VA_BONDDURATION' + - type: 'csv_header' + value: 'Debt type' + column_header: 'IDIRDEBTTYPE' + - type: 'csv_header' + value: 'Asset securitisation type' + column_header: 'IDIRASSETSECTYPE' + - type: 'csv_header' + value: 'CFI classification' + column_header: 'IDIRCLASSIFICATIONCODE_CFI' + - type: 'csv_header' + value: 'Instrument ESA 95 class' + column_header: 'IDIRCLASSIFICATIONCODE_ESAI' + - type: 'csv_header' + value: 'Instrument ESA 95 class - value type' + column_header: 'IDIRCLASSIFICATIONCODE_ESAI_DM' + - type: 'csv_header' + value: 'Nominal currency' + column_header: 'IDIRCURRENCY_NOMINAL' + - type: 'csv_header' + value: 'Amount issued' + column_header: 'AMOUNTISSUED' + - type: 'csv_header' + value: 'Amount outstanding' + column_header: 'AMOUNTOUTSTANDING' + - type: 'csv_header' + value: 'Amount outstanding in EUR' + column_header: 'AMOUNTOUTSTANDING_EUR' + - type: 'csv_header' + value: 'Pool factor' + column_header: 'POOLFACTOR' + - type: 'csv_header' + value: 'Issue price' + column_header: 'ISSUEPRICE' + - type: 'csv_header' + value: 'Issue date' + column_header: 'IDISSUEDATE' + - type: 'csv_header' + value: 'Coupon type' + column_header: 'IDIRCOUPONTYPE' + - type: 'csv_header' + value: 'Last Coupon frequency' + column_header: 'IDIRCOUPONFREQUENCY' + - type: 'csv_header' + value: 'Coupon currency' + column_header: 'IDIRCURRENCY_COUPON' + - type: 'csv_header' + value: 'Last Coupon rate' + column_header: 'COUPONRATE' + - type: 'csv_header' + value: 'Last Coupon date' + column_header: 'COUPONDATE' + - type: 'csv_header' + value: 'Redemption type' + column_header: 'IDIRREDEMPTIONTYPE' + - type: 'csv_header' + value: 'Redemption frequency' + column_header: 'IDIRREDEMPTIONFREQUENCY' + - type: 'csv_header' + value: 'Redemption currency' + column_header: 'IDIRCURRENCY_REDEMPTION' + - type: 'csv_header' + value: 'Redemption price' + column_header: 'REDEMPTIONPRICE' + - type: 'csv_header' + value: 'Maturity date' + column_header: 'IDMATURITYDATE' + - type: 'csv_header' + value: 'Issuer organisation alias type' + column_header: 'IDIRORGANISATIONALIASTYPE_IS' + - type: 'csv_header' + value: 'Issuer organisation alias code' + column_header: 'ISSUERSOURCECODE' + - type: 'csv_header' + value: 'Issuer MFI code' + column_header: 'ISSUEREXTERNALCODE_MFI' + - type: 'csv_header' + value: 'Issuer BIC code' + column_header: 'ISSUEREXTERNALCODE_BIC' + - type: 'csv_header' + value: 'Issuer BEI code' + column_header: 'ISSUEREXTERNALCODE_BEI' + - type: 'csv_header' + value: 'Internal organisation code' + column_header: 'IDIRORGANISATION_ISSUER' + - type: 'csv_header' + value: 'Issuer name' + column_header: 'ISSUERNAME' + - type: 'csv_header' + value: 'Issuer domicile country' + column_header: 'IDIRCOUNTRY' + - type: 'csv_header' + value: 'Issuer domicile country - value type' + column_header: 'IDIRCOUNTRY_DM' + - type: 'csv_header' + value: 'Issuer ESA 95 sector' + column_header: 'IDIRCLASSIFICATIONCODE_ESAO' + - type: 'csv_header' + value: 'Issuer ESA 95 sector - value type' + column_header: 'IDIRCLASSIFICATIONCODE_ESAO_DM' + - type: 'csv_header' + value: 'Issuer NACE sector' + column_header: 'IDIRCLASSIFICATIONCODE_NACE' + - type: 'csv_header' + value: 'Price date' + column_header: 'PUBLICATIONPRICEDATE' + - type: 'csv_header' + value: 'Price value' + column_header: 'PUBLICATIONPRICE' + - type: 'csv_header' + value: 'Price value - type' + column_header: 'PUBLICATIONPRICETYPE' + - type: 'csv_header' + value: 'Quotation basis' + column_header: 'PUBLICATIONPRICEQUOTATIONBASIS' + - type: 'csv_header' + value: 'Monthly average price' + column_header: 'MONTHLYAVERAGEPRICE' + - type: 'csv_header' + value: 'Accrual start date' + column_header: 'ACCRUALSTARTDATE' + - type: 'csv_header' + value: 'Accrued income factor' + column_header: 'DEBTACCRUALDEBTOR' + - type: 'csv_header' + value: 'Accrued income factor - value type' + column_header: 'DEBTACCRUALDEBTOR_DM' + - type: 'csv_header' + value: 'Accrued income (Creditor)' + column_header: 'DEBTACCRUALCREDITOR' + - type: 'csv_header' + value: 'Accrued income (Creditor) - value type' + column_header: 'DEBTACCRUALCREDITOR_TYP' + - type: 'csv_header' + value: 'Accrued interest' + column_header: 'ACCRUEDINTEREST' + - type: 'csv_header' + value: 'Yield to maturity' + column_header: 'YTMNONOPTIONADJUSTED' + - type: 'csv_header' + value: 'ESCB issuer identifier' + column_header: 'ESCB_ISSUER_IDENT' + - type: 'csv_header' + value: 'ESCB issuer identifier type' + column_header: 'VA_ESCBCODETYPE' + - type: 'csv_header' + value: 'Issuer compound ID' + column_header: 'IDUDCMPPARTY' + - type: 'csv_header' + value: 'Amount Oustanding type' + column_header: 'AMOUNTOUTSTANDINGTYPE' + - type: 'csv_header' + value: 'Market Capitalisation' + column_header: 'MARKETCAPITALISATION' + - type: 'csv_header' + value: 'Market Capitalisation in euro' + column_header: 'MARKETCAPITALISATION_EUR' + - type: 'csv_header' + value: 'Security Status' + column_header: 'VA_SECURITYSTATUS' + - type: 'csv_header' + value: 'Instrument suppl class' + column_header: 'VA_INSTRSUPPLEMENTARYCLASS' + - type: 'csv_header' + value: 'Residual maturity class' + column_header: 'VA_RESIDUALMATURITYCLASS' + - type: 'csv_header' + value: 'Is In SEC' + column_header: 'VA_ISINSEC' + - type: 'csv_header' + value: 'Is In EADB' + column_header: 'VA_ISELIGIBLEFOREADB' + - type: 'csv_header' + value: 'Instrument ESA 2010 class' + column_header: 'IDIRCLASSIFICATIONCODE_ESAI10' + - type: 'csv_header' + value: 'Issuer ESA 2010 sector' + column_header: 'IDIRCLASSIFICATIONCODE_ESAO10' + - type: 'csv_header' + value: 'Primary asset classification 2' + column_header: 'IDIRDEBTTYPE_N' + - type: 'csv_header' + value: 'Instruments seniority type' + column_header: 'SENIORITY' + - type: 'csv_header' + value: 'Issuer LEI code' + column_header: 'ISSUEREXTERNALCODE_LEI' + - type: 'csv_header' + value: 'Instrument ESA 2010 class - value type' + column_header: 'INSTR_ESA2010_CLASS_VALUETYPE' + - type: 'csv_header' + value: 'Issuer ESA 2010 class - value type' + column_header: 'ISS_ESA2010_CLASS_VALUETYPE' + - type: 'csv_header' + value: 'Security status date' + column_header: 'VA_SECURITYSTATUSDATE' + - type: 'csv_header' + value: 'Group type' + column_header: 'GROUP_TYPE' + - type: 'csv_header' + value: 'Has embedded option' + column_header: 'HASEMBEDDEDOPTION' + - type: 'csv_header' + value: 'Volume traded' + column_header: 'VOLUMETRADED' + - type: 'csv_header' + value: 'Primary listing name' + column_header: 'PRIMARYLISTINGNAME' + - type: 'csv_header' + value: 'Primary listing residency country' + column_header: 'PRIMARYLISTINGCOUNTRY' + - type: 'csv_header' + value: 'Instrument portfolio flags' + column_header: 'VA_INSTRPORTFLAGS' + - type: 'csv_header' + value: 'Residual maturity' + column_header: 'RESIDUALMATURITY' + - type: 'csv_header' + value: 'Original maturity' + column_header: 'ORIGINAL_MATURITY' + - type: 'csv_header' + value: 'CFIN classification' + column_header: 'IDIRCLASSIFICATIONCODE_CFIN' + - type: 'csv_header' + value: 'First scheduled Coupon date' + column_header: 'COUPONFIRSTPAYMENTDATE' + - type: 'csv_header' + value: 'Last scheduled Coupon date' + column_header: 'COUPONLASTPAYMENTDATE' + - type: 'csv_header' + value: 'Coupon rate underlying ISIN' + column_header: 'COUPONRATEUNDERLYINGCODE_ISIN' + - type: 'csv_header' + value: 'Coupon rate spread' + column_header: 'COUPONRATESPREAD' + - type: 'csv_header' + value: 'Coupon rate multiplier' + column_header: 'COUPONRATEMULTIPLIER' + - type: 'csv_header' + value: 'Coupon rate cap' + column_header: 'COUPONRATECAP' + - type: 'csv_header' + value: 'Coupon rate floor' + column_header: 'COUPONRATEFLOOR' + - type: 'csv_header' + value: 'Issue date tranche' + column_header: 'IDISSUEDATE_TRANCHE' + - type: 'csv_header' + value: 'Issue price tranche' + column_header: 'ISSUEPRICE_TRANCHE' + - type: 'csv_header' + value: 'Is private placement' + column_header: 'VA_ISPRIVATEPLACEMENT' + - type: 'csv_header' + value: 'RIAD code' + column_header: 'RIAD_CODE' + - type: 'csv_header' + value: 'RIAD OUID' + column_header: 'RIAD_OUID' + - type: 'csv_header' + value: 'ESG Flag 1' + column_header: 'ESG1' + - type: 'csv_header' + value: 'ESG Flag 2' + column_header: 'ESG2' + - type: 'csv_header' + value: 'ESG Flag 3' + column_header: 'ESG3' + - type: 'csv_header' + value: 'Strip' + column_header: 'STRIP' + - type: 'csv_header' + value: 'Depository receipt' + column_header: 'DEPOSITORY_RECEIPT' + - type: 'csv_header' + value: 'Rule 144A' + column_header: 'RULE_144A' + - type: 'csv_header' + value: 'Reg S' + column_header: 'REG_S' + - type: 'csv_header' + value: 'Warrant' + column_header: 'WARRANT' + - type: 'csv_header' + value: 'CSEC Relevance - stock' + column_header: 'CSEC_RELEVANCE_STOCK' + - type: 'csv_header' + value: 'CSEC relevance - gross issuance' + column_header: 'CSEC_RELEVANCE_GROSS_ISSUANCE' + - type: 'csv_header' + value: 'CSEC relevance - redemption' + column_header: 'CSEC_RELEVANCE_REDEMPTION' + - type: 'csv_header' + value: 'Accruing coupon' + column_header: 'ACCRUING_COUPON' + - type: 'csv_header' + value: 'Accruing discount' + column_header: 'ACCRUING_DISCOUNT' + - type: 'csv_header' + value: 'STEP Id' + column_header: 'STEPID' + - type: 'csv_header' + value: 'Program Name' + column_header: 'PROGRAMNAME' + - type: 'csv_header' + value: 'Program Ceiling' + column_header: 'PROGRAMCEILING' + - type: 'csv_header' + value: 'Program Status' + column_header: 'PROGRAMSTATUS' + - type: 'csv_header' + value: 'Issuer NACE21 sector' + column_header: 'ISSUERNACE21SECTOR' + - type: 'csv_header' + value: 'Instrument quotation basis' + column_header: 'INSTRUMENTQUOTATIONBASIS' + - type: 'csv_header' + value: 'placeholder 38' + column_header: 'PLACEHOLDER38' + - type: 'csv_header' + value: 'placeholder 39' + column_header: 'PLACEHOLDER39' + - type: 'csv_header' + value: 'placeholder 40' + column_header: 'PLACEHOLDER40' + - type: 'csv_header' + value: 'placeholder 41' + column_header: 'PLACEHOLDER41' + - type: 'csv_header' + value: 'placeholder 42' + column_header: 'PLACEHOLDER42' + - type: 'csv_header' + value: 'placeholder 43' + column_header: 'PLACEHOLDER43' + - type: 'csv_header' + value: 'placeholder 44' + column_header: 'PLACEHOLDER44' + - type: 'csv_header' + value: 'placeholder 45' + column_header: 'PLACEHOLDER45' + - type: 'csv_header' + value: 'placeholder 46' + column_header: 'PLACEHOLDER46' + - type: 'csv_header' + value: 'placeholder 47' + column_header: 'PLACEHOLDER47' + - type: 'csv_header' + value: 'placeholder 48' + column_header: 'PLACEHOLDER48' + - type: 'csv_header' + value: 'placeholder 49' + column_header: 'PLACEHOLDER49' + - type: 'csv_header' + value: 'placeholder 50' + column_header: 'PLACEHOLDER50' diff --git a/airflow/ods/csdb/debt_daily/config/yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml b/airflow/ods/csdb/debt_daily/config/yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml new file mode 100644 index 0000000..646fdbc --- /dev/null +++ b/airflow/ods/csdb/debt_daily/config/yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml @@ -0,0 +1,400 @@ +# App configurations +encoding_type: latin1 +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination +archive_prefix: ARCHIVE/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination +workflow_name: w_ODS_CSDB_DEBT_DAILY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_DEBT_DAILY_PARSE + ods_prefix: INBOX/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination/CSDB_DEBT_DAILY + output_table: CSDB_DEBT_DAILY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'Date last modified' + column_header: 'Date last modified' + - type: 'csv_header' + value: 'Extraction date' + column_header: 'Extraction date' + - type: 'csv_header' + value: 'ISIN code' + column_header: 'ISIN code' + - type: 'csv_header' + value: 'National instrument code type' + column_header: 'National instrument code type' + - type: 'csv_header' + value: 'National instrument code' + column_header: 'National instrument code' + - type: 'csv_header' + value: 'Internal instrument code' + column_header: 'Internal instrument code' + - type: 'csv_header' + value: 'Short name' + column_header: 'Short name' + - type: 'csv_header' + value: 'Bond duration' + column_header: 'Bond duration' + - type: 'csv_header' + value: 'Debt type' + column_header: 'Debt type' + - type: 'csv_header' + value: 'Asset securitisation type' + column_header: 'Asset securitisation type' + - type: 'csv_header' + value: 'CFI classification' + column_header: 'CFI classification' + - type: 'csv_header' + value: 'Instrument ESA 95 class' + column_header: 'Instrument ESA 95 class' + - type: 'csv_header' + value: 'Instrument ESA 95 class - value type' + column_header: 'Instrument ESA 95 class - value type' + - type: 'csv_header' + value: 'Nominal currency' + column_header: 'Nominal currency' + - type: 'csv_header' + value: 'Amount issued' + column_header: 'Amount issued' + - type: 'csv_header' + value: 'Amount outstanding' + column_header: 'Amount outstanding' + - type: 'csv_header' + value: 'Amount outstanding in EUR' + column_header: 'Amount outstanding in EUR' + - type: 'csv_header' + value: 'Pool factor' + column_header: 'Pool factor' + - type: 'csv_header' + value: 'Issue price' + column_header: 'Issue price' + - type: 'csv_header' + value: 'Issue date' + column_header: 'Issue date' + - type: 'csv_header' + value: 'Coupon type' + column_header: 'Coupon type' + - type: 'csv_header' + value: 'Last Coupon frequency' + column_header: 'Last Coupon frequency' + - type: 'csv_header' + value: 'Coupon currency' + column_header: 'Coupon currency' + - type: 'csv_header' + value: 'Last Coupon rate' + column_header: 'Last Coupon rate' + - type: 'csv_header' + value: 'Last Coupon date' + column_header: 'Last Coupon date' + - type: 'csv_header' + value: 'Redemption type' + column_header: 'Redemption type' + - type: 'csv_header' + value: 'Redemption frequency' + column_header: 'Redemption frequency' + - type: 'csv_header' + value: 'Redemption currency' + column_header: 'Redemption currency' + - type: 'csv_header' + value: 'Redemption price' + column_header: 'Redemption price' + - type: 'csv_header' + value: 'Maturity date' + column_header: 'Maturity date' + - type: 'csv_header' + value: 'Issuer organisation alias type' + column_header: 'Issuer organisation alias type' + - type: 'csv_header' + value: 'Issuer organisation alias code' + column_header: 'Issuer organisation alias code' + - type: 'csv_header' + value: 'Issuer MFI code' + column_header: 'Issuer MFI code' + - type: 'csv_header' + value: 'Issuer BIC code' + column_header: 'Issuer BIC code' + - type: 'csv_header' + value: 'Issuer BEI code' + column_header: 'Issuer BEI code' + - type: 'csv_header' + value: 'Internal organisation code' + column_header: 'Internal organisation code' + - type: 'csv_header' + value: 'Issuer name' + column_header: 'Issuer name' + - type: 'csv_header' + value: 'Issuer domicile country' + column_header: 'Issuer domicile country' + - type: 'csv_header' + value: 'Issuer domicile country - value type' + column_header: 'Issuer domicile country - value type' + - type: 'csv_header' + value: 'Issuer ESA 95 sector' + column_header: 'Issuer ESA 95 sector' + - type: 'csv_header' + value: 'Issuer ESA 95 sector - value type' + column_header: 'Issuer ESA 95 sector - value type' + - type: 'csv_header' + value: 'Issuer NACE sector' + column_header: 'Issuer NACE sector' + - type: 'csv_header' + value: 'Price date' + column_header: 'Price date' + - type: 'csv_header' + value: 'Price value' + column_header: 'Price value' + - type: 'csv_header' + value: 'Price value - type' + column_header: 'Price value - type' + - type: 'csv_header' + value: 'Quotation basis' + column_header: 'Quotation basis' + - type: 'csv_header' + value: 'Monthly average price' + column_header: 'Monthly average price' + - type: 'csv_header' + value: 'Accrual start date' + column_header: 'Accrual start date' + - type: 'csv_header' + value: 'Accrued income factor' + column_header: 'Accrued income factor' + - type: 'csv_header' + value: 'Accrued income factor - value type' + column_header: 'Accrued income factor - value type' + - type: 'csv_header' + value: 'Accrued income (Creditor)' + column_header: 'Accrued income (Creditor)' + - type: 'csv_header' + value: 'Accrued income (Creditor) - value type' + column_header: 'Accrued income (Creditor) - value type' + - type: 'csv_header' + value: 'Accrued interest' + column_header: 'Accrued interest' + - type: 'csv_header' + value: 'Yield to maturity' + column_header: 'Yield to maturity' + - type: 'csv_header' + value: 'ESCB issuer identifier' + column_header: 'ESCB issuer identifier' + - type: 'csv_header' + value: 'ESCB issuer identifier type' + column_header: 'ESCB issuer identifier type' + - type: 'csv_header' + value: 'Issuer compound ID' + column_header: 'Issuer compound ID' + - type: 'csv_header' + value: 'Amount Oustanding type' + column_header: 'Amount Oustanding type' + - type: 'csv_header' + value: 'Market Capitalisation' + column_header: 'Market Capitalisation' + - type: 'csv_header' + value: 'Market Capitalisation in euro' + column_header: 'Market Capitalisation in euro' + - type: 'csv_header' + value: 'Security Status' + column_header: 'Security Status' + - type: 'csv_header' + value: 'Instrument suppl class' + column_header: 'Instrument suppl class' + - type: 'csv_header' + value: 'Residual maturity class' + column_header: 'Residual maturity class' + - type: 'csv_header' + value: 'Is In SEC' + column_header: 'Is In SEC' + - type: 'csv_header' + value: 'Is In EADB' + column_header: 'Is In EADB' + - type: 'csv_header' + value: 'Instrument ESA 2010 class' + column_header: 'Instrument ESA 2010 class' + - type: 'csv_header' + value: 'Issuer ESA 2010 sector' + column_header: 'Issuer ESA 2010 sector' + - type: 'csv_header' + value: 'Primary asset classification 2' + column_header: 'Primary asset classification 2' + - type: 'csv_header' + value: 'Instruments seniority type' + column_header: 'Instruments seniority type' + - type: 'csv_header' + value: 'Issuer LEI code' + column_header: 'Issuer LEI code' + - type: 'csv_header' + value: 'Instrument ESA 2010 class - value type' + column_header: 'Instrument ESA 2010 class - value type' + - type: 'csv_header' + value: 'Issuer ESA 2010 class - value type' + column_header: 'Issuer ESA 2010 class - value type' + - type: 'csv_header' + value: 'Security status date' + column_header: 'Security status date' + - type: 'csv_header' + value: 'Group type' + column_header: 'Group type' + - type: 'csv_header' + value: 'Has embedded option' + column_header: 'Has embedded option' + - type: 'csv_header' + value: 'Volume traded' + column_header: 'Volume traded' + - type: 'csv_header' + value: 'Primary listing name' + column_header: 'Primary listing name' + - type: 'csv_header' + value: 'Primary listing residency country' + column_header: 'Primary listing residency country' + - type: 'csv_header' + value: 'Instrument portfolio flags' + column_header: 'Instrument portfolio flags' + - type: 'static' + value: '' + column_header: 'BOND_DURATION' + - type: 'csv_header' + value: 'Residual maturity' + column_header: 'Residual maturity' + - type: 'csv_header' + value: 'Original maturity' + column_header: 'Original maturity' + - type: 'csv_header' + value: 'CFIN classification' + column_header: 'CFIN classification' + - type: 'csv_header' + value: 'First scheduled Coupon date' + column_header: 'First scheduled Coupon date' + - type: 'csv_header' + value: 'Last scheduled Coupon date' + column_header: 'Last scheduled Coupon date' + - type: 'csv_header' + value: 'Coupon rate underlying ISIN' + column_header: 'Coupon rate underlying ISIN' + - type: 'csv_header' + value: 'Coupon rate spread' + column_header: 'Coupon rate spread' + - type: 'csv_header' + value: 'Coupon rate multiplier' + column_header: 'Coupon rate multiplier' + - type: 'csv_header' + value: 'Coupon rate cap' + column_header: 'Coupon rate cap' + - type: 'csv_header' + value: 'Coupon rate floor' + column_header: 'Coupon rate floor' + - type: 'csv_header' + value: 'Issue date tranche' + column_header: 'Issue date tranche' + - type: 'csv_header' + value: 'Issue price tranche' + column_header: 'Issue price tranche' + - type: 'csv_header' + value: 'Is private placement' + column_header: 'Is private placement' + - type: 'csv_header' + value: 'RIAD code' + column_header: 'RIAD code' + - type: 'csv_header' + value: 'RIAD OUID' + column_header: 'RIAD OUID' + - type: 'csv_header' + value: 'ESG Flag 1' + column_header: 'ESG Flag 1' + - type: 'csv_header' + value: 'ESG Flag 2' + column_header: 'ESG Flag 2' + - type: 'csv_header' + value: 'ESG Flag 3' + column_header: 'ESG Flag 3' + - type: 'csv_header' + value: 'Strip' + column_header: 'Strip' + - type: 'csv_header' + value: 'Depository receipt' + column_header: 'Depository receipt' + - type: 'csv_header' + value: 'Rule 144A' + column_header: 'Rule 144A' + - type: 'csv_header' + value: 'Reg S' + column_header: 'Reg S' + - type: 'csv_header' + value: 'Warrant' + column_header: 'Warrant' + - type: 'csv_header' + value: 'CSEC Relevance - stock' + column_header: 'CSEC Relevance - stock' + - type: 'csv_header' + value: 'CSEC relevance - gross issuance' + column_header: 'CSEC relevance - gross issuance' + - type: 'csv_header' + value: 'CSEC relevance - redemption' + column_header: 'CSEC relevance - redemption' + - type: 'csv_header' + value: 'Accruing coupon' + column_header: 'Accruing coupon' + - type: 'csv_header' + value: 'Accruing discount' + column_header: 'Accruing discount' + - type: 'csv_header' + value: 'STEP Id' + column_header: 'STEP Id' + - type: 'csv_header' + value: 'Program Name' + column_header: 'Program Name' + - type: 'csv_header' + value: 'Program Ceiling' + column_header: 'Program Ceiling' + - type: 'csv_header' + value: 'Program Status' + column_header: 'Program Status' + - type: 'csv_header' + value: 'Issuer NACE21 sector' + column_header: 'Issuer NACE21 sector' + - type: 'csv_header' + value: 'Instrument quotation basis' + column_header: 'Instrument quotation basis' + - type: 'csv_header' + value: 'placeholder 38' + column_header: 'placeholder 38' + - type: 'csv_header' + value: 'placeholder 39' + column_header: 'placeholder 39' + - type: 'csv_header' + value: 'placeholder 40' + column_header: 'placeholder 40' + - type: 'csv_header' + value: 'placeholder 41' + column_header: 'placeholder 41' + - type: 'csv_header' + value: 'placeholder 42' + column_header: 'placeholder 42' + - type: 'csv_header' + value: 'placeholder 43' + column_header: 'placeholder 43' + - type: 'csv_header' + value: 'placeholder 44' + column_header: 'placeholder 44' + - type: 'csv_header' + value: 'placeholder 45' + column_header: 'placeholder 45' + - type: 'csv_header' + value: 'placeholder 46' + column_header: 'placeholder 46' + - type: 'csv_header' + value: 'placeholder 47' + column_header: 'placeholder 47' + - type: 'csv_header' + value: 'placeholder 48' + column_header: 'placeholder 48' + - type: 'csv_header' + value: 'placeholder 49' + column_header: 'placeholder 49' + - type: 'csv_header' + value: 'placeholder 50' + column_header: 'placeholder 50' \ No newline at end of file diff --git a/airflow/ods/csdb/full_ratings/.gitkeep b/airflow/ods/csdb/full_ratings/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/csdb/full_ratings/config/.gitkeep b/airflow/ods/csdb/full_ratings/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml new file mode 100644 index 0000000..f1207b8 --- /dev/null +++ b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml @@ -0,0 +1,103 @@ +encoding_type: latin1 +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/FullRatingsDissemination +archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination +workflow_name: w_ODS_CSDB_RATINGS_FULL +##file format +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_INSTR_DESC_FULL_PARSE + ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_INSTR_DESC_FULL + output_table: CSDB_INSTR_DESC_FULL + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'RDB_INSTR_ID' + column_header: 'IDIRINSTRUMENT' + - type: 'csv_header' + value: 'ISIN' + column_header: 'ISIN' + - type: 'csv_header' + value: 'MOO_INSTR_ID' + column_header: 'MOO_INSTR_ID' + - type: 'csv_header' + value: 'SNP_INSTR_ID' + column_header: 'SNP_INSTR_ID' + - type: 'csv_header' + value: 'FTC_INSTR_ID' + column_header: 'FITCH_IDENTIFIER' + - type: 'csv_header' + value: 'DBR_INSTR_ID' + column_header: 'DBRS_IDENTIFIER' + - type: 'csv_header' + value: 'EA_STATUS' + column_header: 'EA_STATUS' + - type: 'csv_header' + value: 'IS_TMS' + column_header: 'IS_TMS' + - type: 'csv_header' + value: 'DBRS_COVERED_BOND_PROGRAM_ID' + column_header: 'DBRS_COVERED_BOND_PROGRAM' + - type: 'csv_header' + value: 'FITCH_PROGRAM_ID' + column_header: 'FITCH_PRG_IDENTIFIER' + - type: 'csv_header' + value: 'MOO_DEAL_NUMBER' + column_header: 'MOO_DEAL_NUMBER' + - type: 'csv_header' + value: 'SNP_PROGRAM_ID' + column_header: 'SNP_PROGRAM_ID' + - type: 'csv_header' + value: 'DBRS_DEBT_TYPE' + column_header: 'IDIRDEBTTYPE' + - type: 'csv_header' + value: 'SNP_DEBT_TYPE' + column_header: 'SNP_DEBT_TYPE' + - type: 'csv_header' + value: 'MOODY_SENIORITY' + column_header: 'MOODY_SENIORITY' + - type: 'csv_header' + value: 'FITCH_DEBT_LEVEL_CODE' + column_header: 'FITCH_DEBT_LEVEL_CODE' + - type: 'csv_header' + value: 'DBRS_RANK_TYPE' + column_header: 'DBRS_RANK_TYPE' + - type: 'csv_header' + value: 'DBRS_SECURITY_TYPE' + column_header: 'DBRS_SECURITY_TYPE' + - type: 'csv_header' + value: 'SCO_DEBT_TYPE' + column_header: 'SCO_DEBT_TYPE' + - type: 'csv_header' + value: 'SCO_INSTR_ID' + column_header: 'SCO_INSTR_ID' + - type: 'csv_header' + value: 'SCO_COVERED_BOND_PROGRAM' + column_header: 'SCO_COVERED_BOND_PROGRAM' + - type: 'csv_header' + value: 'SCO_CATEGORY' + column_header: 'SCO_CATEGORY' + - type: 'csv_header' + value: 'PLACEHOLDER15' + column_header: 'PLACEHOLDER15' + - type: 'csv_header' + value: 'PLACEHOLDER16' + column_header: 'PLACEHOLDER16' + - type: 'csv_header' + value: 'PLACEHOLDER17' + column_header: 'PLACEHOLDER17' + - type: 'csv_header' + value: 'PLACEHOLDER18' + column_header: 'PLACEHOLDER18' + - type: 'csv_header' + value: 'PLACEHOLDER19' + column_header: 'PLACEHOLDER19' + - type: 'csv_header' + value: 'PLACEHOLDER20' + column_header: 'PLACEHOLDER20' + diff --git a/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml new file mode 100644 index 0000000..f7ae5ec --- /dev/null +++ b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml @@ -0,0 +1,130 @@ +encoding_type: latin1 +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/FullRatingsDissemination +archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination +workflow_name: w_ODS_CSDB_RATINGS_FULL +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_INSTR_RAT_FULL_PARSE + ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_INSTR_RAT_FULL + output_table: CSDB_INSTR_RAT_FULL + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'RDB_INSTR_ID' + column_header: 'RDB_INSTR_ID' + - type: 'csv_header' + value: 'SOURCE' + column_header: 'SOURCE' + - type: 'csv_header' + value: 'RATING_SCHEME' + column_header: 'RATING_SCHEME' + - type: 'csv_header' + value: 'RATING' + column_header: 'RATING' + - type: 'csv_header' + value: 'RATING_DATE' + column_header: 'RATING_DATE' + - type: 'csv_header' + value: 'TIME_HORIZON' + column_header: 'TIME_HORIZON' + - type: 'csv_header' + value: 'CURRENCY_TYPE' + column_header: 'CURRENCY_TYPE' + - type: 'csv_header' + value: 'NOTES' + column_header: 'NOTES' + - type: 'csv_header' + value: 'VALID_FROM' + column_header: 'VALID_FROM' + - type: 'csv_header' + value: 'VALID_UNTIL' + column_header: 'VALID_UNTIL' + - type: 'csv_header' + value: 'RDB_RATINGS_ID' + column_header: 'RDB_RATINGS_ID' + - type: 'csv_header' + value: 'OUTLOOK' + column_header: 'WATCHLIST' + - type: 'csv_header' + value: 'OUTLOOK_DATE' + column_header: 'WATCHLIST_DATE' + - type: 'csv_header' + value: 'WATCHLIST' + column_header: 'OUTLOOK' + - type: 'csv_header' + value: 'WATCHLIST_DATE' + column_header: 'OUTLOOK_DATE' + - type: 'csv_header' + value: 'RATING_ACTION' + column_header: 'RATING_ACTION' + - type: 'csv_header' + value: 'RATING_ACTION_DATE' + column_header: 'RATING_ACTION_DATE' + - type: 'csv_header' + value: 'IS_PRELIMINARY' + column_header: 'IS_PRELIMINARY' + - type: 'csv_header' + value: 'RATING_RAW' + column_header: 'RATING_RAW' + - type: 'csv_header' + value: 'RATING_TYPE' + column_header: 'RATING_TYPE' + - type: 'csv_header' + value: 'ENDORSEMENT_INDICATOR' + column_header: 'ENDORSEMENT_INDICATOR' + - type: 'csv_header' + value: 'LAST_REVIEW_DATE' + column_header: 'LAST_REVIEW_DATE' + - type: 'csv_header' + value: 'PLACEHOLDER6' + column_header: 'PLACEHOLDER6' + - type: 'csv_header' + value: 'PLACEHOLDER7' + column_header: 'PLACEHOLDER7' + - type: 'csv_header' + value: 'PLACEHOLDER8' + column_header: 'PLACEHOLDER8' + - type: 'csv_header' + value: 'PLACEHOLDER9' + column_header: 'PLACEHOLDER9' + - type: 'csv_header' + value: 'PLACEHOLDER10' + column_header: 'PLACEHOLDER10' + - type: 'csv_header' + value: 'PLACEHOLDER11' + column_header: 'PLACEHOLDER11' + - type: 'csv_header' + value: 'PLACEHOLDER12' + column_header: 'PLACEHOLDER12' + - type: 'csv_header' + value: 'PLACEHOLDER13' + column_header: 'PLACEHOLDER13' + - type: 'csv_header' + value: 'PLACEHOLDER14' + column_header: 'PLACEHOLDER14' + - type: 'csv_header' + value: 'PLACEHOLDER15' + column_header: 'PLACEHOLDER15' + - type: 'csv_header' + value: 'PLACEHOLDER16' + column_header: 'PLACEHOLDER16' + - type: 'csv_header' + value: 'PLACEHOLDER17' + column_header: 'PLACEHOLDER17' + - type: 'csv_header' + value: 'PLACEHOLDER18' + column_header: 'PLACEHOLDER18' + - type: 'csv_header' + value: 'PLACEHOLDER19' + column_header: 'PLACEHOLDER19' + - type: 'csv_header' + value: 'PLACEHOLDER20' + column_header: 'PLACEHOLDER20' + diff --git a/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml new file mode 100644 index 0000000..d8b4117 --- /dev/null +++ b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml @@ -0,0 +1,106 @@ +encoding_type: latin1 +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/FullRatingsDissemination +archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination +workflow_name: w_ODS_CSDB_RATINGS_FULL +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_ISSUER_DESC_FULL_PARSE + ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_ISSUER_DESC_FULL + output_table: CSDB_ISSUER_DESC_FULL + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'RDB_ISSUER_ID' + column_header: 'RDB_ISSUER_ID' + - type: 'csv_header' + value: 'ISSUER_NAME' + column_header: 'ISSUERNAME' + - type: 'csv_header' + value: 'COUNTRY_DOMICILE' + column_header: 'COUNTRY_DOMICILE' + - type: 'csv_header' + value: 'IS_SOVEREIGN' + column_header: 'IS_SOVEREIGN' + - type: 'csv_header' + value: 'MOO_ISSUER_ID' + column_header: 'MOODY_IDENTIFIER' + - type: 'csv_header' + value: 'SNP_ISSUER_ID' + column_header: 'SNP_ISSUER_ID' + - type: 'csv_header' + value: 'FTC_ISSUER_ID' + column_header: 'FITCH_IDENTIFIER' + - type: 'csv_header' + value: 'DBR_ISSUER_ID' + column_header: 'DBRS_IDENTIFIER' + - type: 'csv_header' + value: 'LEI_ISSUER_ID' + column_header: 'LEI_ISSUER_ID' + - type: 'csv_header' + value: 'RIAD_CODE' + column_header: 'RIAD_CODE' + - type: 'csv_header' + value: 'RIAD_OUID' + column_header: 'RIAD_OUID' + - type: 'csv_header' + value: 'CLASH_GROUP_STATUS' + column_header: 'CLASH_GROUP_STATUS' + - type: 'csv_header' + value: 'SCO_ISSUER_ID' + column_header: 'SCO_ISSUER_ID' + - type: 'csv_header' + value: 'PLACEHOLDER5' + column_header: 'PLACEHOLDER5' + - type: 'csv_header' + value: 'PLACEHOLDER6' + column_header: 'PLACEHOLDER6' + - type: 'csv_header' + value: 'PLACEHOLDER7' + column_header: 'PLACEHOLDER7' + - type: 'csv_header' + value: 'PLACEHOLDER8' + column_header: 'PLACEHOLDER8' + - type: 'csv_header' + value: 'PLACEHOLDER9' + column_header: 'PLACEHOLDER9' + - type: 'csv_header' + value: 'PLACEHOLDER10' + column_header: 'PLACEHOLDER10' + - type: 'csv_header' + value: 'PLACEHOLDER11' + column_header: 'PLACEHOLDER11' + - type: 'csv_header' + value: 'PLACEHOLDER12' + column_header: 'PLACEHOLDER12' + - type: 'csv_header' + value: 'PLACEHOLDER13' + column_header: 'PLACEHOLDER13' + - type: 'csv_header' + value: 'PLACEHOLDER14' + column_header: 'PLACEHOLDER14' + - type: 'csv_header' + value: 'PLACEHOLDER15' + column_header: 'PLACEHOLDER15' + - type: 'csv_header' + value: 'PLACEHOLDER16' + column_header: 'PLACEHOLDER16' + - type: 'csv_header' + value: 'PLACEHOLDER17' + column_header: 'PLACEHOLDER17' + - type: 'csv_header' + value: 'PLACEHOLDER18' + column_header: 'PLACEHOLDER18' + - type: 'csv_header' + value: 'PLACEHOLDER19' + column_header: 'PLACEHOLDER19' + - type: 'csv_header' + value: 'PLACEHOLDER20' + column_header: 'PLACEHOLDER20' + diff --git a/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml new file mode 100644 index 0000000..57f1cdc --- /dev/null +++ b/airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml @@ -0,0 +1,131 @@ +encoding_type: latin1 +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/CSDB/FullRatingsDissemination +archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination +workflow_name: w_ODS_CSDB_RATINGS_FULL +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_CSDB_ISSUER_RAT_FULL_PARSE + ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_ISSUER_RAT_FULL + output_table: CSDB_ISSUER_RAT_FULL + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'RDB_ISSUER_ID' + column_header: 'RDB_ISSUER_ID' + - type: 'csv_header' + value: 'SOURCE' + column_header: 'SOURCE' + - type: 'csv_header' + value: 'RATING_SCHEME' + column_header: 'RATING_SCHEME' + - type: 'csv_header' + value: 'RATING' + column_header: 'RATING' + - type: 'csv_header' + value: 'RATING_DATE' + column_header: 'RATING_DATE' + - type: 'csv_header' + value: 'TIME_HORIZON' + column_header: 'TIME_HORIZON' + - type: 'csv_header' + value: 'CURRENCY_TYPE' + column_header: 'CURRENCY_TYPE' + - type: 'csv_header' + value: 'NOTES' + column_header: 'NOTES' + - type: 'csv_header' + value: 'VALID_FROM' + column_header: 'VALID_FROM' + - type: 'csv_header' + value: 'VALID_UNTIL' + column_header: 'VALID_UNTIL' + - type: 'csv_header' + value: 'RDB_RATINGS_ID' + column_header: 'RDB_RATINGS_ID' + - type: 'csv_header' + value: 'OUTLOOK' + column_header: 'OUTLOOK' + - type: 'csv_header' + value: 'OUTLOOK_DATE' + column_header: 'OUTLOOK_DATE' + - type: 'csv_header' + value: 'WATCHLIST' + column_header: 'WATCHLIST' + - type: 'csv_header' + value: 'WATCHLIST_DATE' + column_header: 'WATCHLIST_DATE' + - type: 'csv_header' + value: 'RATING_ACTION' + column_header: 'RATING_ACTION' + - type: 'csv_header' + value: 'RATING_ACTION_DATE' + column_header: 'RATING_ACTION_DATE' + - type: 'csv_header' + value: 'IS_PRELIMINARY' + column_header: 'IS_PRELIMINARY' + - type: 'csv_header' + value: 'RATING_RAW' + column_header: 'RATING_RAW' + - type: 'csv_header' + value: 'RATING_TYPE' + column_header: 'RATING_TYPE' + - type: 'csv_header' + value: 'ENDORSEMENT_INDICATOR' + column_header: 'ENDORSEMENT_INDICATOR' + - type: 'csv_header' + value: 'LAST_REVIEW_DATE' + column_header: 'LAST_REVIEW_DATE' + - type: 'csv_header' + value: 'PLACEHOLDER6' + column_header: 'PLACEHOLDER6' + - type: 'csv_header' + value: 'PLACEHOLDER7' + column_header: 'PLACEHOLDER7' + - type: 'csv_header' + value: 'PLACEHOLDER8' + column_header: 'PLACEHOLDER8' + - type: 'csv_header' + value: 'PLACEHOLDER9' + column_header: 'PLACEHOLDER9' + - type: 'csv_header' + value: 'PLACEHOLDER10' + column_header: 'PLACEHOLDER10' + - type: 'csv_header' + value: 'PLACEHOLDER11' + column_header: 'PLACEHOLDER11' + - type: 'csv_header' + value: 'PLACEHOLDER12' + column_header: 'PLACEHOLDER12' + - type: 'csv_header' + value: 'PLACEHOLDER13' + column_header: 'PLACEHOLDER13' + - type: 'csv_header' + value: 'PLACEHOLDER14' + column_header: 'PLACEHOLDER14' + - type: 'csv_header' + value: 'PLACEHOLDER15' + column_header: 'PLACEHOLDER15' + - type: 'csv_header' + value: 'PLACEHOLDER16' + column_header: 'PLACEHOLDER16' + - type: 'csv_header' + value: 'PLACEHOLDER17' + column_header: 'PLACEHOLDER17' + - type: 'csv_header' + value: 'PLACEHOLDER18' + column_header: 'PLACEHOLDER18' + - type: 'csv_header' + value: 'PLACEHOLDER19' + column_header: 'PLACEHOLDER19' + - type: 'csv_header' + value: 'PLACEHOLDER20' + column_header: 'PLACEHOLDER20' + + diff --git a/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py b/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py new file mode 100644 index 0000000..1b653d5 --- /dev/null +++ b/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py @@ -0,0 +1,420 @@ +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.python import get_current_context + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +dag_id = "w_ODS_CSDB_RATINGS_FULL_COORDINATOR" + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") +OBJECT_PREFIX = os.getenv("OBJECT_PREFIX", "csdb/ratings/full/") +REPROCESS = (os.getenv("CSDB_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" + + +def _oci_client(): + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + + +def _load_processed_map() -> dict[str, float]: + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + return {} + + +def _list_all_zip_objects(include_processed: bool = False) -> list[dict]: + """List all zip files in the bucket""" + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() if not include_processed else {} + + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=OBJECT_PREFIX) + all_items: list[dict] = [] + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + + if not name or name.endswith('/') or not base: + continue + + if not ("STC-FullRatingsDissemination" in base and base.lower().endswith(".zip")): + continue + + # Get timestamp + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + + # Check if already processed + last_proc_ts = float(processed_map.get(name, 0.0)) + is_processed = (ts <= last_proc_ts) if processed_map else False + + item = { + "name": name, + "base": base, + "mtime": ts, + "is_processed": is_processed + } + all_items.append(item) + + # Sort by timestamp (oldest first) + all_items.sort(key=lambda x: x["mtime"]) + + return all_items + + +def _list_new_zip_objects() -> list[dict]: + """List only new/unprocessed zip files""" + all_items = _list_all_zip_objects(include_processed=False) + + # Filter out processed items + new_items = [item for item in all_items if not item.get("is_processed", False)] + + logging.info("Found %d new STC-FullRatingsDissemination zip file(s) (sorted oldest to newest)", len(new_items)) + return new_items + + +def _find_specific_zip(filename_pattern: str) -> dict: + """Find a specific zip file by name pattern""" + all_items = _list_all_zip_objects(include_processed=True) + + # Try exact match first + for item in all_items: + if item["base"] == filename_pattern or item["name"] == filename_pattern: + logging.info("Found exact match: %s", item["base"]) + return item + + # Try partial match + for item in all_items: + if filename_pattern.lower() in item["base"].lower(): + logging.info("Found partial match: %s", item["base"]) + return item + + raise AirflowFailException(f"No zip file found matching pattern: {filename_pattern}") + + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='CSDB Ratings Full Coordinator: Lists and triggers processing for zip files', + schedule_interval="0 */6 * * *", # Every 6 hours, adjust as needed + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["CSDB", "COORDINATOR", "ODS", "OCI", "RATINGS"], +) as dag: + + @af_task(task_id="determine_processing_mode") + def determine_processing_mode(**context): + """ + Determine what to process based on dag_run configuration. + + Configuration options: + 1. No config or mode='all': Process all new zip files + 2. mode='specific' + filename='xxx': Process specific zip file + 3. mode='reprocess_all': Reprocess all zip files (including already processed) + 4. mode='list_only': Just list available files without processing + 5. filenames=['file1.zip', 'file2.zip']: Process specific list of files + """ + conf = context.get('dag_run').conf or {} + + mode = conf.get('mode', 'all') + filename = conf.get('filename') + filenames = conf.get('filenames', []) + force_reprocess = conf.get('force_reprocess', False) + limit = conf.get('limit') # Limit number of files to process + + logging.info("Processing mode: %s", mode) + logging.info("Configuration: %s", json.dumps(conf, indent=2)) + + result = { + "mode": mode, + "filename": filename, + "filenames": filenames, + "force_reprocess": force_reprocess, + "limit": limit + } + + return result + + @af_task(task_id="list_zip_files") + def list_zip_files(mode_config: dict): + """List zip files based on the processing mode""" + mode = mode_config.get("mode", "all") + filename = mode_config.get("filename") + filenames = mode_config.get("filenames", []) + force_reprocess = mode_config.get("force_reprocess", False) + limit = mode_config.get("limit") + + zip_files = [] + + if mode == "list_only": + # Just list all files for information + all_files = _list_all_zip_objects(include_processed=True) + logging.info("=== Available ZIP Files ===") + for idx, f in enumerate(all_files, 1): + status = "PROCESSED" if f.get("is_processed") else "NEW" + logging.info("%d. [%s] %s (mtime: %s)", + idx, status, f["base"], + datetime.fromtimestamp(f["mtime"]).isoformat()) + raise AirflowSkipException("List only mode - no processing triggered") + + elif mode == "specific": + # Process a specific file + if not filename: + raise AirflowFailException("mode='specific' requires 'filename' parameter") + + zip_file = _find_specific_zip(filename) + zip_files = [zip_file] + logging.info("Processing specific file: %s", zip_file["base"]) + + elif mode == "specific_list": + # Process a list of specific files + if not filenames: + raise AirflowFailException("mode='specific_list' requires 'filenames' parameter") + + for fn in filenames: + try: + zip_file = _find_specific_zip(fn) + zip_files.append(zip_file) + except Exception as e: + logging.warning("Could not find file %s: %s", fn, e) + + if not zip_files: + raise AirflowFailException("None of the specified files were found") + + logging.info("Processing %d specific files", len(zip_files)) + + elif mode == "reprocess_all": + # Reprocess all files (including already processed) + all_files = _list_all_zip_objects(include_processed=True) + zip_files = all_files + logging.info("Reprocessing all %d files", len(zip_files)) + + elif mode == "date_range": + # Process files within a date range + start_date = mode_config.get("start_date") + end_date = mode_config.get("end_date") + + if not start_date or not end_date: + raise AirflowFailException("mode='date_range' requires 'start_date' and 'end_date'") + + start_ts = datetime.fromisoformat(start_date).timestamp() + end_ts = datetime.fromisoformat(end_date).timestamp() + + all_files = _list_all_zip_objects(include_processed=True) + zip_files = [f for f in all_files if start_ts <= f["mtime"] <= end_ts] + + logging.info("Found %d files in date range %s to %s", + len(zip_files), start_date, end_date) + + else: # mode == "all" or default + # Process all new files + zip_files = _list_new_zip_objects() + + if not zip_files: + logging.info("No new zip files to process") + raise AirflowSkipException("No new zip files found") + + # Apply limit if specified + if limit and isinstance(limit, int) and limit > 0: + original_count = len(zip_files) + zip_files = zip_files[:limit] + logging.info("Limited processing from %d to %d files", original_count, len(zip_files)) + + # Sort by timestamp (oldest first) + zip_files.sort(key=lambda x: x["mtime"]) + + logging.info("Selected %d zip file(s) for processing:", len(zip_files)) + for idx, f in enumerate(zip_files, 1): + logging.info("%d. %s (mtime: %s)", + idx, f["base"], + datetime.fromtimestamp(f["mtime"]).isoformat()) + + return { + "zip_files": zip_files, + "mode": mode, + "force_reprocess": force_reprocess + } + + @af_task(task_id="trigger_processing_dags") + def trigger_processing_dags(list_result: dict): + """Trigger the processing DAG for each zip file sequentially""" + from airflow.api.common.trigger_dag import trigger_dag + from time import sleep + + zip_files = list_result.get("zip_files", []) + mode = list_result.get("mode", "all") + force_reprocess = list_result.get("force_reprocess", False) + + if not zip_files: + logging.info("No zip files to process") + return [] + + triggered_runs = [] + + for idx, zip_file in enumerate(zip_files): + conf = { + "zip_object_name": zip_file["name"], + "zip_base_name": zip_file["base"], + "zip_mtime": zip_file["mtime"], + "sequence_number": idx + 1, + "total_files": len(zip_files), + "processing_mode": mode, + "force_reprocess": force_reprocess, + "is_processed": zip_file.get("is_processed", False) + } + + logging.info(f"Triggering processing DAG for file {idx + 1}/{len(zip_files)}: {zip_file['base']}") + + try: + run_id = trigger_dag( + dag_id="w_ODS_CSDB_RATINGS_FULL_CORE", + run_id=f"coordinator__{datetime.now().strftime('%Y%m%d_%H%M%S')}__{idx}", + conf=conf, + execution_date=None, + replace_microseconds=False, + ) + + triggered_runs.append({ + "run_id": str(run_id), + "zip_file": zip_file["base"], + "sequence": idx + 1, + "status": "triggered" + }) + + logging.info(f"Successfully triggered run: {run_id}") + + except Exception as e: + logging.error(f"Failed to trigger processing for {zip_file['base']}: {e}") + triggered_runs.append({ + "zip_file": zip_file["base"], + "sequence": idx + 1, + "status": "failed", + "error": str(e) + }) + + # Small delay between triggers to avoid overwhelming the system + sleep(2) + + logging.info(f"Triggered {len([r for r in triggered_runs if r.get('status') == 'triggered'])} processing DAG runs") + logging.info(f"Failed to trigger {len([r for r in triggered_runs if r.get('status') == 'failed'])} runs") + + return triggered_runs + + @af_task(task_id="summary_report") + def summary_report(trigger_result: list): + """Generate a summary report of triggered runs""" + if not trigger_result: + logging.info("No runs were triggered") + return + + successful = [r for r in trigger_result if r.get("status") == "triggered"] + failed = [r for r in trigger_result if r.get("status") == "failed"] + + logging.info("=" * 80) + logging.info("PROCESSING SUMMARY") + logging.info("=" * 80) + logging.info(f"Total files: {len(trigger_result)}") + logging.info(f"Successfully triggered: {len(successful)}") + logging.info(f"Failed to trigger: {len(failed)}") + + if successful: + logging.info("\nSuccessfully triggered:") + for r in successful: + logging.info(f" - {r['zip_file']} (run_id: {r['run_id']})") + + if failed: + logging.info("\nFailed to trigger:") + for r in failed: + logging.info(f" - {r['zip_file']} (error: {r.get('error', 'unknown')})") + + logging.info("=" * 80) + + return { + "total": len(trigger_result), + "successful": len(successful), + "failed": len(failed) + } + + # Build DAG structure + mode_task = determine_processing_mode() + list_task = list_zip_files(mode_task) + trigger_task = trigger_processing_dags(list_task) + summary_task = summary_report(trigger_task) + + mode_task >> list_task >> trigger_task >> summary_task + +logging.info("CSDB Ratings Full Coordinator DAG ready") diff --git a/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_CORE.py b/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_CORE.py new file mode 100644 index 0000000..a395769 --- /dev/null +++ b/airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_CORE.py @@ -0,0 +1,388 @@ +import sys +import os +import json +import logging +import zipfile +from pathlib import Path +from datetime import timedelta, datetime, timezone + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator +from airflow.operators.python import get_current_context + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/csdb') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = "w_ODS_CSDB_RATINGS_FULL_CORE" + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") +OBJECT_PREFIX = os.getenv("OBJECT_PREFIX", "csdb/ratings/full/") +TEMP_DIR = "/tmp/csdb_ratings" +PROCESSED_TS_VAR = "w_ODS_CSDB_RATINGS_FULL_COORDINATOR__processed_objects_ts" + +# CSV configurations +CSV_CONFIGS = [ + { + "source_filename": "FULL_INSTRUMENT_DESCRIPTION.csv", + "config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml", + "task_name": "m_ODS_CSDB_RATINGS_FULL_INSTRUMENT_DESCRIPTION" + }, + { + "source_filename": "FULL_INSTRUMENT_RATINGS.csv", + "config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml", + "task_name": "m_ODS_CSDB_RATINGS_FULL_INSTRUMENT_RATINGS" + }, + { + "source_filename": "FULL_ISSUER_DESCRIPTION.csv", + "config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml", + "task_name": "m_ODS_CSDB_RATINGS_FULL_ISSUER_DESCRIPTION" + }, + { + "source_filename": "FULL_ISSUER_RATINGS.csv", + "config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml", + "task_name": "m_ODS_CSDB_RATINGS_FULL_ISSUER_RATINGS" + } +] + + +def _oci_client(): + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + + +def _load_processed_map() -> dict[str, float]: + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + return {} + + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + + +def _mark_processed(zip_key: str, zip_mtime: float): + m = _load_processed_map() + m[zip_key] = float(zip_mtime) + _save_processed_map(m) + logging.info("Marked as processed: %s (mtime=%s)", zip_key, zip_mtime) + + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='CSDB Ratings Full Processor: Processes one zip file with 4 CSV files in parallel', + schedule_interval=None, # Triggered by coordinator + catchup=False, + max_active_runs=3, # Allow some parallelism but controlled + render_template_as_native_obj=True, + tags=["CSDB", "PROCESSOR", "MRDS", "ODS", "OCI", "RATINGS"], +) as dag: + + @af_task(task_id="get_zip_config") + def get_zip_config(**context): + """Get the zip file configuration from dag_run conf""" + conf = context['dag_run'].conf or {} + + zip_object_name = conf.get('zip_object_name') + zip_base_name = conf.get('zip_base_name') + zip_mtime = conf.get('zip_mtime') + sequence_number = conf.get('sequence_number', 0) + total_files = conf.get('total_files', 0) + + if not all([zip_object_name, zip_base_name, zip_mtime]): + raise AirflowFailException("Missing required configuration: zip_object_name, zip_base_name, or zip_mtime") + + logging.info(f"Processing zip file {sequence_number}/{total_files}: {zip_base_name}") + + return { + "zip_object_name": zip_object_name, + "zip_base_name": zip_base_name, + "zip_mtime": zip_mtime, + "sequence_number": sequence_number, + "total_files": total_files + } + + @af_task(task_id="download_and_unzip") + def download_and_unzip(config: dict): + """Download and unzip the specific zip file""" + zip_key = config["zip_object_name"] + zip_base = config["zip_base_name"] + + client = _oci_client() + os.makedirs(TEMP_DIR, exist_ok=True) + + # Create unique temp directory for this run + run_temp_dir = os.path.join(TEMP_DIR, f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}") + os.makedirs(run_temp_dir, exist_ok=True) + + local_zip = os.path.join(run_temp_dir, zip_base) + + logging.info("Downloading %s to %s", zip_key, local_zip) + get_obj = client.get_object(OCI_NAMESPACE, OCI_BUCKET, zip_key) + with open(local_zip, 'wb') as f: + for chunk in get_obj.data.raw.stream(1024 * 1024, decode_content=False): + f.write(chunk) + + logging.info("Unzipping %s", local_zip) + with zipfile.ZipFile(local_zip, 'r') as zip_ref: + zip_ref.extractall(run_temp_dir) + + extracted_files = [] + for root, dirs, files in os.walk(run_temp_dir): + for file in files: + if file.endswith('.csv'): + full_path = os.path.join(root, file) + extracted_files.append({"filename": file, "path": full_path}) + logging.info("Extracted CSV: %s", file) + + logging.info("Total CSV files extracted: %d", len(extracted_files)) + + return { + "extracted_files": extracted_files, + "zip_config": config, + "temp_dir": run_temp_dir + } + + @af_task(task_id="init_workflow") + def init_workflow(unzipped: dict): + """Initialize MRDS workflow""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + extracted_files = unzipped.get("extracted_files", []) + zip_config = unzipped.get("zip_config", {}) + temp_dir = unzipped.get("temp_dir") + + task_configs = [] + for csv_config in CSV_CONFIGS: + matching_file = next( + (ef for ef in extracted_files if ef["filename"] == csv_config["source_filename"]), + None + ) + if matching_file: + task_configs.append({ + "task_name": csv_config["task_name"], + "source_filename": csv_config["source_filename"], + "source_path": matching_file["path"], + "config_file": csv_config["config_yaml"], + }) + logging.info("Prepared task config for %s", csv_config["source_filename"]) + else: + logging.warning("CSV file %s not found in extracted files", csv_config["source_filename"]) + + return { + "workflow_history_key": a_workflow_history_key, + "task_configs": task_configs, + "zip_config": zip_config, + "temp_dir": temp_dir + } + + def run_mrds_task(task_config: dict, **context): + """Run MRDS processing for a single CSV file""" + ti = context['ti'] + + task_name = task_config["task_name"] + source_path = task_config["source_path"] + config_file = task_config["config_file"] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + if not os.path.exists(source_path): + raise FileNotFoundError(f"Source CSV file not found: {source_path}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_history_key = init_bundle.get('workflow_history_key') + + if not workflow_history_key: + raise AirflowFailException("No workflow_history_key from init_workflow") + + try: + logging.info(f"{task_name}: Starting MRDS processing for {source_path}") + mrds_main(workflow_history_key, source_path, config_file, generate_workflow_context=False) + logging.info(f"{task_name}: MRDS processing completed successfully") + except Exception as e: + logging.exception(f"{task_name}: MRDS failed on {source_path}") + raise + + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize the workflow and mark zip as processed""" + ti = context['ti'] + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + + a_workflow_history_key = init_bundle.get('workflow_history_key') + zip_config = init_bundle.get('zip_config', {}) + + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + # Check if any CSV task failed + csv_task_ids = [cfg["task_name"] for cfg in CSV_CONFIGS] + dag_run = context['dag_run'] + tis = [t for t in dag_run.get_task_instances() if t.task_id in csv_task_ids] + + from airflow.utils.state import State + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + + if not any_failed: + # Mark zip as processed + zip_key = zip_config.get("zip_object_name") + zip_mtime = zip_config.get("zip_mtime") + if zip_key and zip_mtime: + _mark_processed(zip_key, zip_mtime) + + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + else: + failed_tasks = [ti_i.task_id for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed tasks=%s)", + a_workflow_history_key, failed_tasks) + raise AirflowFailException(f"Workflow failed for tasks: {failed_tasks}") + + @af_task(task_id="cleanup_temp_files") + def cleanup_temp_files(**context): + """Clean up temporary files for this run""" + import shutil + ti = context['ti'] + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + temp_dir = init_bundle.get('temp_dir') + + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + logging.info("Cleaned up temp directory: %s", temp_dir) + + @af_task(task_id="move_zip_to_archive") + def move_zip_to_archive(**context): + """Move processed zip file to archive""" + ti = context['ti'] + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + zip_config = init_bundle.get('zip_config', {}) + + zip_key = zip_config.get("zip_object_name") + if not zip_key: + logging.warning("No zip key found, skipping archive") + return + + client = _oci_client() + archive_key = zip_key.replace(OBJECT_PREFIX, f"{OBJECT_PREFIX}archive/", 1) + + try: + client.copy_object( + OCI_NAMESPACE, + OCI_BUCKET, + { + "sourceObjectName": zip_key, + "destinationRegion": os.getenv("OCI_REGION", "eu-frankfurt-1"), + "destinationNamespace": OCI_NAMESPACE, + "destinationBucket": OCI_BUCKET, + "destinationObjectName": archive_key + } + ) + logging.info("Copied to archive: %s -> %s", zip_key, archive_key) + + client.delete_object(OCI_NAMESPACE, OCI_BUCKET, zip_key) + logging.info("Deleted from inbox: %s", zip_key) + except Exception as e: + logging.error("Failed to archive zip file %s: %s", zip_key, e) + raise + + # Build the DAG structure + config_task = get_zip_config() + unzip_task = download_and_unzip(config_task) + init_task = init_workflow(unzip_task) + + # Create CSV processing tasks dynamically + csv_tasks = [] + for csv_config in CSV_CONFIGS: + task = PythonOperator( + task_id=csv_config["task_name"], + python_callable=run_mrds_task, + op_kwargs={ + "task_config": { + "task_name": csv_config["task_name"], + "source_filename": csv_config["source_filename"], + "source_path": "{{ ti.xcom_pull(task_ids='init_workflow')['task_configs'] | selectattr('task_name', 'equalto', '" + csv_config["task_name"] + "') | map(attribute='source_path') | first }}", + "config_file": csv_config["config_yaml"], + } + }, + provide_context=True, + ) + csv_tasks.append(task) + + finalize_task = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + cleanup_task = cleanup_temp_files() + archive_task = move_zip_to_archive() + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_SUCCESS, + ) + + # Define task dependencies + config_task >> unzip_task >> init_task >> csv_tasks >> finalize_task >> [cleanup_task, archive_task] >> all_good + +logging.info("CSDB Ratings Full Processor DAG ready") diff --git a/airflow/ods/fxcd/BRANCH/config/m_ODS_FXCD_F_BRANCH_PARSE.yml b/airflow/ods/fxcd/BRANCH/config/m_ODS_FXCD_F_BRANCH_PARSE.yml new file mode 100644 index 0000000..1560de1 --- /dev/null +++ b/airflow/ods/fxcd/BRANCH/config/m_ODS_FXCD_F_BRANCH_PARSE.yml @@ -0,0 +1,29 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/BRANCH +archive_prefix: ARCHIVE/FXCD/BRANCH +workflow_name: w_ODS_FXCD_F_BRANCH +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_BRANCH_PARSE + ods_prefix: INBOX/FXCD/BRANCH/FXCD_F_BRANCH + output_table: FXCD_F_BRANCH + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'BRANCH_ID' + column_header: 'BRANCH_ID' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'CTP_ID' + column_header: 'CTP_ID' diff --git a/airflow/ods/fxcd/CLEARER/config/m_ODS_FXCD_F_CLEARER_PARSE.yml b/airflow/ods/fxcd/CLEARER/config/m_ODS_FXCD_F_CLEARER_PARSE.yml new file mode 100644 index 0000000..1fa3039 --- /dev/null +++ b/airflow/ods/fxcd/CLEARER/config/m_ODS_FXCD_F_CLEARER_PARSE.yml @@ -0,0 +1,53 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CLEARER +archive_prefix: ARCHIVE/FXCD/CLEARER +workflow_name: w_ODS_FXCD_F_CLEARER +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CLEARER_PARSE + ods_prefix: INBOX/FXCD/CLEARER/FXCD_F_CLEARER + output_table: FXCD_F_CLEARER + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CLEARER_ID' + column_header: 'CLEARER_ID' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_OF_FLAG' + column_header: 'ELIGIBILITY_OF_FLAG' + - type: 'csv_header' + value: 'ELIGIBILITY_FR_FLAG' + column_header: 'ELIGIBILITY_FR_FLAG' + - type: 'csv_header' + value: 'ACTIVE_FLAG' + column_header: 'ACTIVE_FLAG' + - type: 'csv_header' + value: 'OVERALL_OF_LIMIT_AMT' + column_header: 'OVERALL_OF_LIMIT_AMT' + - type: 'csv_header' + value: 'CASH_OF_LIMIT_AMT' + column_header: 'CASH_OF_LIMIT_AMT' + - type: 'csv_header' + value: 'SECURITIES_OF_LIMIT_AMT' + column_header: 'SECURITIES_OF_LIMIT_AMT' + - type: 'csv_header' + value: 'OVERALL_FR_LIMIT_AMT' + column_header: 'OVERALL_FR_LIMIT_AMT' + - type: 'csv_header' + value: 'CASH_FR_LIMIT_AMT' + column_header: 'CASH_FR_LIMIT_AMT' + - type: 'csv_header' + value: 'SECURITIES_FR_LIMIT_AMT' + column_header: 'SECURITIES_FR_LIMIT_AMT' diff --git a/airflow/ods/fxcd/CLEARER_NCB_LIMIT/config/m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/CLEARER_NCB_LIMIT/config/m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..33cd2f0 --- /dev/null +++ b/airflow/ods/fxcd/CLEARER_NCB_LIMIT/config/m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml @@ -0,0 +1,35 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CLEARER_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/CLEARER_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_CLEARER_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/CLEARER_NCB_LIMIT/FXCD_F_CLEARER_NCB_LIMIT + output_table: FXCD_F_CLEARER_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CLEARER_ID' + column_header: 'CLEARER_ID' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'OVERALL_LIMIT_AMT' + column_header: 'OVERALL_LIMIT_AMT' + - type: 'csv_header' + value: 'CASH_LIMIT_AMT' + column_header: 'CASH_LIMIT_AMT' + - type: 'csv_header' + value: 'SECURITIES_LIMIT_AMT' + column_header: 'SECURITIES_LIMIT_AMT' diff --git a/airflow/ods/fxcd/CONSTANT/config/m_ODS_FXCD_F_CONSTANT_PARSE.yml b/airflow/ods/fxcd/CONSTANT/config/m_ODS_FXCD_F_CONSTANT_PARSE.yml new file mode 100644 index 0000000..a0c8b1b --- /dev/null +++ b/airflow/ods/fxcd/CONSTANT/config/m_ODS_FXCD_F_CONSTANT_PARSE.yml @@ -0,0 +1,29 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CONSTANT +archive_prefix: ARCHIVE/FXCD/CONSTANT +workflow_name: w_ODS_FXCD_F_CONSTANT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CONSTANT_PARSE + ods_prefix: INBOX/FXCD/CONSTANT/FXCD_F_CONSTANT + output_table: FXCD_F_CONSTANT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CONSTANT_NAME' + column_header: 'CONSTANT_NAME' + - type: 'csv_header' + value: 'CONSTANT_VALUE' + column_header: 'CONSTANT_VALUE' + - type: 'csv_header' + value: 'CONSTANT_TYPE' + column_header: 'CONSTANT_TYPE' diff --git a/airflow/ods/fxcd/COUNTERPARTY/config/m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml b/airflow/ods/fxcd/COUNTERPARTY/config/m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml new file mode 100644 index 0000000..6b41f83 --- /dev/null +++ b/airflow/ods/fxcd/COUNTERPARTY/config/m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml @@ -0,0 +1,70 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/COUNTERPARTY +archive_prefix: ARCHIVE/FXCD/COUNTERPARTY +workflow_name: w_ODS_FXCD_F_COUNTERPARTY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_COUNTERPARTY_PARSE + ods_prefix: INBOX/FXCD/COUNTERPARTY/FXCD_F_COUNTERPARTY + output_table: FXCD_F_COUNTERPARTY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CTP_ID' + column_header: 'CTP_ID' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'CTP_GROUP_FKIT_CODE' + column_header: 'CTP_GROUP_FKIT_CODE' + - type: 'csv_header' + value: 'ACTIVE_FLAG' + column_header: 'ACTIVE_FLAG' + - type: 'csv_header' + value: 'CTP_COMMENT' + column_header: 'CTP_COMMENT' + - type: 'csv_header' + value: 'GUARANTOR_ID' + column_header: 'GUARANTOR_ID' + - type: 'csv_header' + value: 'OF_OVERALL_LMT_AMT' + column_header: 'OF_OVERALL_LMT_AMT' + - type: 'csv_header' + value: 'OF_MANUAL_FLAG' + column_header: 'OF_MANUAL_FLAG' + - type: 'csv_header' + value: 'FR_OVERALL_LMT_AMT' + column_header: 'FR_OVERALL_LMT_AMT' + - type: 'csv_header' + value: 'FR_MANUAL_FLAG' + column_header: 'FR_MANUAL_FLAG' + - type: 'csv_header' + value: 'MP_OVERALL_LMT_AMT' + column_header: 'MP_OVERALL_LMT_AMT' + - type: 'csv_header' + value: 'MP_MANUAL_FLAG' + column_header: 'MP_MANUAL_FLAG' + - type: 'csv_header' + value: 'EOI_CTP_GROUP' + column_header: 'EOI_CTP_GROUP' + - type: 'csv_header' + value: 'ART_101_FLA' + column_header: 'ART_101_FLA' + - type: 'csv_header' + value: 'MEDIAN_CAPITAL_FLAG' + column_header: 'MEDIAN_CAPITAL_FLAG' + - type: 'csv_header' + value: 'CHANGE_DESCRIPTION' + column_header: 'CHANGE_DESCRIPTION' + + diff --git a/airflow/ods/fxcd/COUNTRY/config/m_ODS_FXCD_F_COUNTRY_PARSE.yml b/airflow/ods/fxcd/COUNTRY/config/m_ODS_FXCD_F_COUNTRY_PARSE.yml new file mode 100644 index 0000000..fbeb91b --- /dev/null +++ b/airflow/ods/fxcd/COUNTRY/config/m_ODS_FXCD_F_COUNTRY_PARSE.yml @@ -0,0 +1,85 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/COUNTRY +archive_prefix: ARCHIVE/FXCD/COUNTRY +workflow_name: w_ODS_FXCD_F_COUNTRY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_COUNTRY_PARSE + ods_prefix: INBOX/FXCD/COUNTRY/FXCD_F_COUNTRY + output_table: FXCD_F_COUNTRY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'COUNTRY_GDP' + column_header: 'COUNTRY_GDP' + - type: 'csv_header' + value: 'NCB_USD_LAMBDA' + column_header: 'NCB_USD_LAMBDA' + - type: 'csv_header' + value: 'OF_FLAG' + column_header: 'OF_FLAG' + - type: 'csv_header' + value: 'FR_FLAG' + column_header: 'FR_FLAG' + - type: 'csv_header' + value: 'EU_FLAG' + column_header: 'EU_FLAG' + - type: 'csv_header' + value: 'EUROSYSTEM_FLAG' + column_header: 'EUROSYSTEM_FLAG' + - type: 'csv_header' + value: 'FR_PORTF_SHARE_OPTOUT' + column_header: 'FR_PORTF_SHARE_OPTOUT' + - type: 'csv_header' + value: 'FR_LIMIT_CALC_OPTOUT' + column_header: 'FR_LIMIT_CALC_OPTOUT' + - type: 'csv_header' + value: 'COUNTRY_COMMENT' + column_header: 'COUNTRY_COMMENT' + - type: 'csv_header' + value: 'OF_LMT_AMNT' + column_header: 'OF_LMT_AMNT' + - type: 'csv_header' + value: 'OF_MANUAL_FLAG' + column_header: 'OF_MANUAL_FLAG' + - type: 'csv_header' + value: 'FR_LMT_AMNT' + column_header: 'FR_LMT_AMNT' + - type: 'csv_header' + value: 'FR_MANUAL_FLAG' + column_header: 'FR_MANUAL_FLAG' + - type: 'csv_header' + value: 'USD_PORTFOLIO_EUR_SIZE' + column_header: 'USD_PORTFOLIO_EUR_SIZE' + - type: 'csv_header' + value: 'JPY_PORTFOLIO_EUR_SIZE' + column_header: 'JPY_PORTFOLIO_EUR_SIZE' + - type: 'csv_header' + value: 'CAPITAL_KEY_AMNT' + column_header: 'CAPITAL_KEY_AMNT' + - type: 'csv_header' + value: 'LAMBDA_MANUAL_FLAG' + column_header: 'LAMBDA_MANUAL_FLAG' + - type: 'csv_header' + value: 'CNY_PORTFOLIO_EUR_SIZE' + column_header: 'CNY_PORTFOLIO_EUR_SIZE' + - type: 'csv_header' + value: 'CHANGE_DESCRIPTION' + column_header: 'CHANGE_DESCRIPTION' + + diff --git a/airflow/ods/fxcd/COUNTRY_NCB_LIMIT/config/m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/COUNTRY_NCB_LIMIT/config/m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..11ee897 --- /dev/null +++ b/airflow/ods/fxcd/COUNTRY_NCB_LIMIT/config/m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml @@ -0,0 +1,35 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/COUNTRY_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/COUNTRY_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_COUNTRY_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/COUNTRY_NCB_LIMIT/FXCD_F_COUNTRY_NCB_LIMIT + output_table: FXCD_F_COUNTRY_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'NCB_COUNTRY_ID' + column_header: 'NCB_COUNTRY_ID' + - type: 'csv_header' + value: 'USD_LIMIT_AMT' + column_header: 'USD_LIMIT_AMT' + - type: 'csv_header' + value: 'JPY_LIMIT_AMT' + column_header: 'JPY_LIMIT_AMT' + - type: 'csv_header' + value: 'TOTAL_LIMIT_AMT' + column_header: 'TOTAL_LIMIT_AMT' diff --git a/airflow/ods/fxcd/CTP_GROUP/config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml b/airflow/ods/fxcd/CTP_GROUP/config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml new file mode 100644 index 0000000..90a13ee --- /dev/null +++ b/airflow/ods/fxcd/CTP_GROUP/config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml @@ -0,0 +1,35 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CTP_GROUP +archive_prefix: ARCHIVE/FXCD/CTP_GROUP +workflow_name: w_ODS_FXCD_F_CTP_GROUP +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CTP_GROUP_PARSE + ods_prefix: INBOX/FXCD/CTP_GROUP/FXCD_F_CTP_GROUP + output_table: FXCD_F_CTP_GROUP + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CTP_GROUP_FKIT_CODE' + column_header: 'CTP_GROUP_FKIT_CODE' + - type: 'csv_header' + value: 'CTP_GROUP_NAME' + column_header: 'CTP_GROUP_NAME' + - type: 'csv_header' + value: 'OF_MAX_LIMIT_AMT' + column_header: 'OF_MAX_LIMIT_AMT' + - type: 'csv_header' + value: 'FR_MP_MAX_LIMIT_AMT' + column_header: 'FR_MP_MAX_LIMIT_AMT' + - type: 'csv_header' + value: 'ACTIVE' + column_header: 'ACTIVE' diff --git a/airflow/ods/fxcd/CTP_GROUP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/CTP_GROUP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..c446ba4 --- /dev/null +++ b/airflow/ods/fxcd/CTP_GROUP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml @@ -0,0 +1,32 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CTP_GROUP_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/CTP_GROUP_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/CTP_GROUP_NCB_LIMIT/FXCD_F_CTP_GROUP_NCB_LIMIT + output_table: FXCD_F_CTP_GROUP_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'CTP_GROUP_FKIT_CODE' + column_header: 'CTP_GROUP_FKIT_CODE' + - type: 'csv_header' + value: 'ELIGIBILITY_ID' + column_header: 'ELIGIBILITY_ID' + - type: 'csv_header' + value: 'LIMIT_AMT' + column_header: 'LIMIT_AMT' diff --git a/airflow/ods/fxcd/CTP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/CTP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..0e2076e --- /dev/null +++ b/airflow/ods/fxcd/CTP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml @@ -0,0 +1,32 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/CTP_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/CTP_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_CTP_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/CTP_NCB_LIMIT/FXCD_F_CTP_NCB_LIMIT + output_table: FXCD_F_CTP_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_ID' + column_header: 'ELIGIBILITY_ID' + - type: 'csv_header' + value: 'CTP_ID' + column_header: 'CTP_ID' + - type: 'csv_header' + value: 'LIMIT_AMT' + column_header: 'LIMIT_AMT' diff --git a/airflow/ods/fxcd/ELIGIBILITY/config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml b/airflow/ods/fxcd/ELIGIBILITY/config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml new file mode 100644 index 0000000..23300f6 --- /dev/null +++ b/airflow/ods/fxcd/ELIGIBILITY/config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml @@ -0,0 +1,78 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ELIGIBILITY +archive_prefix: ARCHIVE/FXCD/ELIGIBILITY +workflow_name: w_ODS_FXCD_F_ELIGIBILITY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ELIGIBILITY_PARSE + ods_prefix: INBOX/FXCD/ELIGIBILITY/FXCD_F_ELIGIBILITY + output_table: FXCD_F_ELIGIBILITY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ELIGIBILITY_ID' + column_header: 'ELIGIBILITY_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_TYPE' + column_header: 'ELIGIBILITY_TYPE' + - type: 'csv_header' + value: 'ELIGIBILITY_NAME' + column_header: 'ELIGIBILITY_NAME' + - type: 'csv_header' + value: 'LIMIT_PERCENTAGE' + column_header: 'LIMIT_PERCENTAGE' + - type: 'csv_header' + value: 'LIMIT_AMT_MANUAL_FLAG' + column_header: 'LIMIT_AMT_MANUAL_FLAG' + - type: 'csv_header' + value: 'LIMIT_AMT' + column_header: 'LIMIT_AMT' + - type: 'csv_header' + value: 'NCB_LMT_AMT_FLOOR' + column_header: 'NCB_LMT_AMT_FLOOR' + - type: 'csv_header' + value: 'RATING_THRESHOLD' + column_header: 'RATING_THRESHOLD' + - type: 'csv_header' + value: 'FKIT_SUBLIMIT_NAME' + column_header: 'FKIT_SUBLIMIT_NAME' + - type: 'csv_header' + value: 'ELIGIBILITY_ORDER' + column_header: 'ELIGIBILITY_ORDER' + - type: 'csv_header' + value: 'LIMIT_DISTRIBUTION' + column_header: 'LIMIT_DISTRIBUTION' + - type: 'csv_header' + value: 'CTP_MIN_REQ_RATINGS' + column_header: 'CTP_MIN_REQ_RATINGS' + - type: 'csv_header' + value: 'LIMITS_CALCULATION' + column_header: 'LIMITS_CALCULATION' + - type: 'csv_header' + value: 'ART_101_FLAG' + column_header: 'ART_101_FLAG' + - type: 'csv_header' + value: 'DEFINITION' + column_header: 'DEFINITION' + - type: 'csv_header' + value: 'FOR_CP_FLAG' + column_header: 'FOR_CP_FLAG' + - type: 'csv_header' + value: 'FOR_CPG_FLAG' + column_header: 'FOR_CPG_FLAG' + - type: 'csv_header' + value: 'OVERALL_LIMIT_FLAG' + column_header: 'OVERALL_LIMIT_FLAG' + - type: 'csv_header' + value: 'IDENTIFIER' + column_header: 'IDENTIFIER' + diff --git a/airflow/ods/fxcd/ELIGIBILITY_GROUP_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml b/airflow/ods/fxcd/ELIGIBILITY_GROUP_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml new file mode 100644 index 0000000..29226c7 --- /dev/null +++ b/airflow/ods/fxcd/ELIGIBILITY_GROUP_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml @@ -0,0 +1,35 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ELIGIBILITY_GROUP_MAP +archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_GROUP_MAP +workflow_name: w_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE + ods_prefix: INBOX/FXCD/ELIGIBILITY_GROUP_MAP/FXCD_F_ELIGIBILITY_GROUP_MAP + output_table: FXCD_F_ELIGIBILITY_GROUP_MAP + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CTP_GROUP_FKIT_CODE' + column_header: 'CTP_GROUP_FKIT_CODE' + - type: 'csv_header' + value: 'ELIGIBILITY_ID' + column_header: 'ELIGIBILITY_ID' + - type: 'csv_header' + value: 'OF_LIMIT_AMT' + column_header: 'OF_LIMIT_AMT' + - type: 'csv_header' + value: 'FR_MP_LIMIT_AMT' + column_header: 'FR_MP_LIMIT_AMT' + - type: 'csv_header' + value: 'LIMIT_AMT' + column_header: 'LIMIT_AMT' \ No newline at end of file diff --git a/airflow/ods/fxcd/ELIGIBILITY_ISSUER/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml b/airflow/ods/fxcd/ELIGIBILITY_ISSUER/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml new file mode 100644 index 0000000..b69fef1 --- /dev/null +++ b/airflow/ods/fxcd/ELIGIBILITY_ISSUER/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml @@ -0,0 +1,44 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER +archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_ISSUER +workflow_name: w_ODS_FXCD_F_ELIGIBILITY_ISSUER +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE + ods_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER/FXCD_F_ELIGIBILITY_ISSUER + output_table: FXCD_F_ELIGIBILITY_ISSUER + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ELIGIBILITY_ISSUER_ID' + column_header: 'ELIGIBILITY_ISSUER_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_ISSUER_NAME' + column_header: 'ELIGIBILITY_ISSUER_NAME' + - type: 'csv_header' + value: 'ART_101_FLAG' + column_header: 'ART_101_FLAG' + - type: 'csv_header' + value: 'RATING_THRESHOLD' + column_header: 'RATING_THRESHOLD' + - type: 'csv_header' + value: 'DEFINITION' + column_header: 'DEFINITION' + - type: 'csv_header' + value: 'RATING_TERM_TYPE' + column_header: 'RATING_TERM_TYPE' + - type: 'csv_header' + value: 'RATING_UPLIFT_FACTOR' + column_header: 'RATING_UPLIFT_FACTOR' + - type: 'csv_header' + value: 'ELIG_FLAG' + column_header: 'ELIG_FLAG' \ No newline at end of file diff --git a/airflow/ods/fxcd/ELIGIBILITY_ISSUER_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml b/airflow/ods/fxcd/ELIGIBILITY_ISSUER_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml new file mode 100644 index 0000000..8ce6cb3 --- /dev/null +++ b/airflow/ods/fxcd/ELIGIBILITY_ISSUER_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml @@ -0,0 +1,32 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER_MAP +archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_ISSUER_MAP +workflow_name: w_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE + ods_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER_MAP/FXCD_F_ELIGIBILITY_ISSUER_MAP + output_table: FXCD_F_ELIGIBILITY_ISSUER_MAP + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ISSUER_ID' + column_header: 'ISSUER_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_ISSUER_ID' + column_header: 'ELIGIBILITY_ISSUER_ID' + - type: 'csv_header' + value: 'ELEGIBLE_FLAG' + column_header: 'ELEGIBLE_FLAG' + - type: 'csv_header' + value: 'MANUAL_ELIGIBLE_FLAG' + column_header: 'MANUAL_ELIGIBLE_FLAG' \ No newline at end of file diff --git a/airflow/ods/fxcd/ELIGIBILITY_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml b/airflow/ods/fxcd/ELIGIBILITY_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml new file mode 100644 index 0000000..749d059 --- /dev/null +++ b/airflow/ods/fxcd/ELIGIBILITY_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml @@ -0,0 +1,41 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ELIGIBILITY_MAP +archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_MAP +workflow_name: w_ODS_FXCD_F_ELIGIBILITY_MAP +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE + ods_prefix: INBOX/FXCD/ELIGIBILITY_MAP/FXCD_F_ELIGIBILITY_MAP + output_table: FXCD_F_ELIGIBILITY_MAP + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'CTP_ID' + column_header: 'CTP_ID' + - type: 'csv_header' + value: 'ELIGIBILITY_ID' + column_header: 'ELIGIBILITY_ID' + - type: 'csv_header' + value: 'OF_LIMIT_AMT' + column_header: 'OF_LIMIT_AMT' + - type: 'csv_header' + value: 'OF_FLAG' + column_header: 'OF_FLAG' + - type: 'csv_header' + value: 'FR_LIMIT_AMT' + column_header: 'FR_LIMIT_AMT' + - type: 'csv_header' + value: 'FR_MP_FLAG' + column_header: 'FR_MP_FLAG' + - type: 'csv_header' + value: 'LIMIT_AMT' + column_header: 'LIMIT_AMT' \ No newline at end of file diff --git a/airflow/ods/fxcd/ENTITY/config/m_ODS_FXCD_F_ENTITY_PARSE.yml b/airflow/ods/fxcd/ENTITY/config/m_ODS_FXCD_F_ENTITY_PARSE.yml new file mode 100644 index 0000000..0f60fd0 --- /dev/null +++ b/airflow/ods/fxcd/ENTITY/config/m_ODS_FXCD_F_ENTITY_PARSE.yml @@ -0,0 +1,56 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ENTITY +archive_prefix: ARCHIVE/FXCD/ENTITY +workflow_name: w_ODS_FXCD_F_ENTITY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ENTITY_PARSE + ods_prefix: INBOX/FXCD/ENTITY/FXCD_F_ENTITY + output_table: FXCD_F_ENTITY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'FKIT_CODE' + column_header: 'FKIT_CODE' + - type: 'csv_header' + value: 'ENTITY_NAME' + column_header: 'ENTITY_NAME' + - type: 'csv_header' + value: 'BVD_BANK_INDEX_NUMBER' + column_header: 'BVD_BANK_INDEX_NUMBER' + - type: 'csv_header' + value: 'RISK_COUNTRY_ID' + column_header: 'RISK_COUNTRY_ID' + - type: 'csv_header' + value: 'ST_MANUAL_RATING_FLAG' + column_header: 'ST_MANUAL_RATING_FLAG' + - type: 'csv_header' + value: 'LT_MANUAL_RATING_FLAG' + column_header: 'LT_MANUAL_RATING_FLAG' + - type: 'csv_header' + value: 'ST_SECOND_BEST_RATING' + column_header: 'ST_SECOND_BEST_RATING' + - type: 'csv_header' + value: 'LT_SECOND_BEST_RATING' + column_header: 'LT_SECOND_BEST_RATING' + - type: 'csv_header' + value: 'CAP_TIER1_AMT' + column_header: 'CAP_TIER1_AMT' + - type: 'csv_header' + value: 'CAP_EQUITY_AMT' + column_header: 'CAP_EQUITY_AMT' + - type: 'csv_header' + value: 'BLOOMBERG_TICKER' + column_header: 'BLOOMBERG_TICKER' \ No newline at end of file diff --git a/airflow/ods/fxcd/EQUIVALENCE_RULE/config/m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml b/airflow/ods/fxcd/EQUIVALENCE_RULE/config/m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml new file mode 100644 index 0000000..ff54189 --- /dev/null +++ b/airflow/ods/fxcd/EQUIVALENCE_RULE/config/m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml @@ -0,0 +1,35 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/EQUIVALENCE_RULE +archive_prefix: ARCHIVE/FXCD/EQUIVALENCE_RULE +workflow_name: w_ODS_FXCD_F_EQUIVALENCE_RULE +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE + ods_prefix: INBOX/FXCD/EQUIVALENCE_RULE/FXCD_F_EQUIVALENCE_RULE + output_table: FXCD_F_EQUIVALENCE_RULE + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'RATING_TERM_TYPE' + column_header: 'RATING_TERM_TYPE' + - type: 'csv_header' + value: 'AGENCY_ID' + column_header: 'AGENCY_ID' + - type: 'csv_header' + value: 'AGENCY_RATING' + column_header: 'AGENCY_RATING' + - type: 'csv_header' + value: 'NUMERICAL_EQUIVALENCE' + column_header: 'NUMERICAL_EQUIVALENCE' + - type: 'csv_header' + value: 'RATING_FACTOR' + column_header: 'RATING_FACTOR' \ No newline at end of file diff --git a/airflow/ods/fxcd/ISSUER/config/m_ODS_FXCD_F_ISSUER_PARSE.yml b/airflow/ods/fxcd/ISSUER/config/m_ODS_FXCD_F_ISSUER_PARSE.yml new file mode 100644 index 0000000..ba86e95 --- /dev/null +++ b/airflow/ods/fxcd/ISSUER/config/m_ODS_FXCD_F_ISSUER_PARSE.yml @@ -0,0 +1,77 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ISSUER +archive_prefix: ARCHIVE/FXCD/ISSUER +workflow_name: w_ODS_FXCD_F_ISSUER +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ISSUER_PARSE + ods_prefix: INBOX/FXCD/ISSUER/FXCD_F_ISSUER + output_table: FXCD_F_ISSUER + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ISSUER_ID' + column_header: 'ISSUER_ID' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'ACTIVE_FLAG' + column_header: 'ACTIVE_FLAG' + - type: 'csv_header' + value: 'ISSUER_COMMENT' + column_header: 'ISSUER_COMMENT' + - type: 'csv_header' + value: 'OFM_ISSUER_LMT' + column_header: 'OFM_ISSUER_LMT' + - type: 'csv_header' + value: 'OF_MANUAL_FLAG' + column_header: 'OF_MANUAL_FLAG' + - type: 'csv_header' + value: 'GLOBAL_LMT_AMNT' + column_header: 'GLOBAL_LMT_AMNT' + - type: 'csv_header' + value: 'MANUAL_GLOBAL_LMT_FLAG' + column_header: 'MANUAL_GLOBAL_LMT_FLAG' + - type: 'csv_header' + value: 'OF_ISSUER_CATEGORY' + column_header: 'OF_ISSUER_CATEGORY' + - type: 'csv_header' + value: 'FR_ISSUER_CATEGORY' + column_header: 'FR_ISSUER_CATEGORY' + - type: 'csv_header' + value: 'ISSUER_LMT' + column_header: 'ISSUER_LMT' + - type: 'csv_header' + value: 'FR_MANUAL_FLAG' + column_header: 'FR_MANUAL_FLAG' + - type: 'csv_header' + value: 'NCB_USD_LAMBDA' + column_header: 'NCB_USD_LAMBDA' + - type: 'csv_header' + value: 'ART_101_FLAG' + column_header: 'ART_101_FLAG' + - type: 'csv_header' + value: 'UPLIFT_IMPLIED_RATING' + column_header: 'UPLIFT_IMPLIED_RATING' + - type: 'csv_header' + value: 'LAMBDA_MANUAL_FLAG' + column_header: 'LAMBDA_MANUAL_FLAG' + - type: 'csv_header' + value: 'CHANGE_DESCRIPTION' + column_header: 'CHANGE_DESCRIPTION' + - type: 'csv_header' + value: 'LT_FRM_LIMIT_RATIO' + column_header: 'LT_FRM_LIMIT_RATIO' + - type: 'csv_header' + value: 'LT_FRM_LIMIT_RATIO_MANUAL_FLAG' + column_header: 'LT_FRM_LIMIT_RATIO_MANUAL_FLAG' \ No newline at end of file diff --git a/airflow/ods/fxcd/ISSUER_NCB_LIMIT/config/m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/ISSUER_NCB_LIMIT/config/m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..1e58343 --- /dev/null +++ b/airflow/ods/fxcd/ISSUER_NCB_LIMIT/config/m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml @@ -0,0 +1,38 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/ISSUER_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/ISSUER_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_ISSUER_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/ISSUER_NCB_LIMIT/FXCD_F_ISSUER_NCB_LIMIT + output_table: FXCD_F_ISSUER_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'ISSUER_ID' + column_header: 'ISSUER_ID' + - type: 'csv_header' + value: 'USD_LIMIT_AMT' + column_header: 'USD_LIMIT_AMT' + - type: 'csv_header' + value: 'JPY_LIMIT_AMT' + column_header: 'JPY_LIMIT_AMT' + - type: 'csv_header' + value: 'TOTAL_LIMIT_AMT' + column_header: 'TOTAL_LIMIT_AMT' + - type: 'csv_header' + value: 'LIMIT_TYPE' + column_header: 'LIMIT_TYPE' \ No newline at end of file diff --git a/airflow/ods/fxcd/LIQUIDITY_NCB_LIMIT/config/m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml b/airflow/ods/fxcd/LIQUIDITY_NCB_LIMIT/config/m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml new file mode 100644 index 0000000..74a112a --- /dev/null +++ b/airflow/ods/fxcd/LIQUIDITY_NCB_LIMIT/config/m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml @@ -0,0 +1,29 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/LIQUIDITY_NCB_LIMIT +archive_prefix: ARCHIVE/FXCD/LIQUIDITY_NCB_LIMIT +workflow_name: w_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE + ods_prefix: INBOX/FXCD/LIQUIDITY_NCB_LIMIT/FXCD_F_LIQUIDITY_NCB_LIMIT + output_table: FXCD_F_LIQUIDITY_NCB_LIMIT + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'USD_NCB_MIN_LIMIT_AMT' + column_header: 'USD_NCB_MIN_LIMIT_AMT' + - type: 'csv_header' + value: 'USD_NCB_MAX_LIMIT_AMT' + column_header: 'USD_NCB_MAX_LIMIT_AMT' \ No newline at end of file diff --git a/airflow/ods/fxcd/NCB_PORTFOLIO_SHARE/config/m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml b/airflow/ods/fxcd/NCB_PORTFOLIO_SHARE/config/m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml new file mode 100644 index 0000000..08388e6 --- /dev/null +++ b/airflow/ods/fxcd/NCB_PORTFOLIO_SHARE/config/m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml @@ -0,0 +1,47 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/NCB_PORTFOLIO_SHARE +archive_prefix: ARCHIVE/FXCD/NCB_PORTFOLIO_SHARE +workflow_name: w_ODS_FXCD_F_NCB_PORTFOLIO_SHARE +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE + ods_prefix: INBOX/FXCD/NCB_PORTFOLIO_SHARE/FXCD_F_NCB_PORTFOLIO_SHARE + output_table: FXCD_F_NCB_PORTFOLIO_SHARE + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'COUNTRY_ID' + column_header: 'COUNTRY_ID' + - type: 'csv_header' + value: 'COUNTRY_SHARE' + column_header: 'COUNTRY_SHARE' + - type: 'csv_header' + value: 'USD_COUNTRY_SHARE' + column_header: 'USD_COUNTRY_SHARE' + - type: 'csv_header' + value: 'JPY_COUNTRY_SHARE' + column_header: 'JPY_COUNTRY_SHARE' + - type: 'csv_header' + value: 'SCALED_COUNTRY_SHARE' + column_header: 'SCALED_COUNTRY_SHARE' + - type: 'csv_header' + value: 'SCALED_USD_COUNTRY_SHARE' + column_header: 'SCALED_USD_COUNTRY_SHARE' + - type: 'csv_header' + value: 'SCALED_JPY_COUNTRY_SHARE' + column_header: 'SCALED_JPY_COUNTRY_SHARE' + - type: 'csv_header' + value: 'CNY_COUNTRY_SHARE' + column_header: 'CNY_COUNTRY_SHARE' + - type: 'csv_header' + value: 'SCALED_CNY_COUNTRY_SHARE' + column_header: 'SCALED_CNY_COUNTRY_SHARE' diff --git a/airflow/ods/fxcd/RATING/config/m_ODS_FXCD_F_RATING_PARSE.yml b/airflow/ods/fxcd/RATING/config/m_ODS_FXCD_F_RATING_PARSE.yml new file mode 100644 index 0000000..6dc1ef5 --- /dev/null +++ b/airflow/ods/fxcd/RATING/config/m_ODS_FXCD_F_RATING_PARSE.yml @@ -0,0 +1,32 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/RATING +archive_prefix: ARCHIVE/FXCD/RATING +workflow_name: w_ODS_FXCD_F_RATING +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_RATING_PARSE + ods_prefix: INBOX/FXCD/RATING/FXCD_F_RATING + output_table: FXCD_F_RATING + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'ENTITY_ID' + column_header: 'ENTITY_ID' + - type: 'csv_header' + value: 'RATING_TERM_TYPE' + column_header: 'RATING_TERM_TYPE' + - type: 'csv_header' + value: 'AGENCY_ID' + column_header: 'AGENCY_ID' + - type: 'csv_header' + value: 'AGENCY_RATING' + column_header: 'AGENCY_RATING' diff --git a/airflow/ods/fxcd/RATING_AGENCY/config/m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml b/airflow/ods/fxcd/RATING_AGENCY/config/m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml new file mode 100644 index 0000000..62df3d3 --- /dev/null +++ b/airflow/ods/fxcd/RATING_AGENCY/config/m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml @@ -0,0 +1,29 @@ +# App configurations +encoding_type: latin1 + +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/FXCD/RATING_AGENCY +archive_prefix: ARCHIVE/FXCD/RATING_AGENCY +workflow_name: w_ODS_FXCD_F_RATING_AGENCY +validation_schema_path: None +file_type: csv + +tasks: + - task_name: m_ODS_FXCD_F_RATING_AGENCY_PARSE + ods_prefix: INBOX/FXCD/RATING_AGENCY/FXCD_F_RATING_AGENCY + output_table: FXCD_F_RATING_AGENCY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'AGENCY_ID' + column_header: 'AGENCY_ID' + - type: 'csv_header' + value: 'AGENCY_NAME' + column_header: 'AGENCY_NAME' + - type: 'csv_header' + value: 'ENABLED_FLAG' + column_header: 'ENABLED_FLAG' diff --git a/airflow/ods/lm/adhoc_adjustments/config/.gitkeep b/airflow/ods/lm/adhoc_adjustments/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd b/airflow/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd new file mode 100644 index 0000000..5154c77 --- /dev/null +++ b/airflow/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml b/airflow/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml new file mode 100644 index 0000000..866ac58 --- /dev/null +++ b/airflow/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml @@ -0,0 +1,78 @@ +tmpdir: /tmp +inbox_prefix: INBOX/LM/AdhocAdjustment +archive_prefix: ARCHIVE/LM/AdhocAdjustment +workflow_name: w_ODS_LM_ADHOC_ADJUSTMENT_MSG +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_HEADER_PARSE + ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_HEADER + output_table: LM_ADHOC_ADJUSTMENTS_HEADER + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:header/ns2:date' + column_header: 'ADJUSTMENT_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:header/ns2:version' + column_header: 'VERSION' + is_key: 'N' + - task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_ITEM_HEADER_PARSE + ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER + output_table: LM_ADHOC_ADJUSTMENTS_ITEM_HEADER + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:effectiveDate' + column_header: 'EFFECTIVE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast' + column_header: 'LAST_DATE_NOT_FORECAST' + is_key: 'N' + + - task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_ITEM_PARSE + ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_ITEM + output_table: LM_ADHOC_ADJUSTMENTS_ITEM + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName' + column_header: 'FORECAST_NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount' + column_header: 'ADJUSTMENT_AMOUNT' + is_key: 'N' + + diff --git a/airflow/ods/lm/adhoc_adjustments/dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py b/airflow/ods/lm/adhoc_adjustments/dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py new file mode 100644 index 0000000..29ba649 --- /dev/null +++ b/airflow/ods/lm/adhoc_adjustments/dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py @@ -0,0 +1,520 @@ +# dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "ADHOC_ADJUSTMENT"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_ADHOC_ADJUSTMENT" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_ADHOC_ADJUSTMENT" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_ADHOC_ADJUSTMENT" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_ADHOC_ADJUSTMENT", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_ADHOC_ADJUSTMENT", + trigger_dag_id="w_MOPDB_LM_ADHOC_ADJUSTMENT", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/balancesheet/config/.gitkeep b/airflow/ods/lm/balancesheet/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/balancesheet/config/balancesheet.xsd b/airflow/ods/lm/balancesheet/config/balancesheet.xsd new file mode 100644 index 0000000..741b1dd --- /dev/null +++ b/airflow/ods/lm/balancesheet/config/balancesheet.xsd @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Represents a balanceSheet for a CB + + + + + + + + + + + Represents an aggregated balanceSheet of all CBs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml b/airflow/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml new file mode 100644 index 0000000..d71ae2c --- /dev/null +++ b/airflow/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml @@ -0,0 +1,82 @@ +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem +archive_prefix: ARCHIVE/LM/AggregatedDailyFinancialStatementOfTheEurosystem +workflow_name: w_ODS_LM_BALANCESHEET +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/balancesheet/config/balancesheet.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_BALANCESHEET_HEADER_PARSE + ods_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem/LM_BALANCESHEET_HEADER + output_table: LM_BALANCESHEET_HEADER + namespaces: + ns2: 'http://escb.ecb.int/balancesheet' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:balanceSheetMessage/ns2:*' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:referenceDate' + column_header: 'REFERENCE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:version' + column_header: 'VERSION' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:status' + column_header: 'STATUS' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:freeText' + column_header: 'FREE_TEXT' + is_key: 'N' + + + - task_name: m_ODS_LM_BALANCESHEET_ITEM_PARSE + ods_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem/LM_BALANCESHEET_ITEM + output_table: LM_BALANCESHEET_ITEM + namespaces: + ns2: 'http://escb.ecb.int/balancesheet' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:balanceSheetMessage/ns2:*' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:level' + is_key: 'N' + column_header: 'ITEM_LEVEL' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:position' + is_key: 'N' + column_header: 'POSITION' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:itemType' + is_key: 'N' + column_header: 'ITEM_TYPE' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:fullyQualifiedPosition' + column_header: 'FULLY_QUALIFIED_POSITION' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:name' + column_header: 'NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:balanceSheetMessage//ns2:amount' + column_header: 'AMOUNT' + is_key: 'N' + + \ No newline at end of file diff --git a/airflow/ods/lm/balancesheet/dags/w_ODS_LM_BALANCESHEET.py b/airflow/ods/lm/balancesheet/dags/w_ODS_LM_BALANCESHEET.py new file mode 100644 index 0000000..8c68889 --- /dev/null +++ b/airflow/ods/lm/balancesheet/dags/w_ODS_LM_BALANCESHEET.py @@ -0,0 +1,520 @@ +# dags/m_ODS_LM_BALANCESHEET.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "BALANCESHEET"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_BALANCESHEET" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_BALANCESHEET" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_BALANCESHEET" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_BALANCESHEET", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_BALANCESHEET", + trigger_dag_id="w_MOPDB_LM_BALANCESHEET", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/csm_adjustment/config/.gitkeep b/airflow/ods/lm/csm_adjustment/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/csm_adjustment/config/csm_adjustment.xsd b/airflow/ods/lm/csm_adjustment/config/csm_adjustment.xsd new file mode 100644 index 0000000..5154c77 --- /dev/null +++ b/airflow/ods/lm/csm_adjustment/config/csm_adjustment.xsd @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml b/airflow/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml new file mode 100644 index 0000000..6dbf3a3 --- /dev/null +++ b/airflow/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml @@ -0,0 +1,83 @@ +tmpdir: /tmp +inbox_prefix: INBOX/LM/MonthlyCSMAdjustment +archive_prefix: ARCHIVE/LM/MonthlyCSMAdjustment +workflow_name: w_ODS_LM_CSM_ADJUSTMENT_MSG +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/csm_adjustment/config/csm_adjustment.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_CSM_ADJUSTMENT_HEADER_PARSE + ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_HEADER + output_table: LM_CSM_ADJUSTMENTS_HEADER + namespaces: + ns2: http://escb.ecb.int/csm-adjustment + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:year' + column_header: 'YEAR' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:month' + column_header: 'MONTH' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:version' + column_header: 'VERSION' + is_key: 'N' + - task_name: m_ODS_CSM_ADJUSTMENT_ITEM_HEADER_PARSE + ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_ITEM_HEADER + output_table: LM_CSM_ADJUSTMENTS_ITEM_HEADER + namespaces: + ns2: http://escb.ecb.int/csm-adjustment + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:effectiveDate' + column_header: 'EFFECTIVE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast' + column_header: 'LAST_DATE_NOT_FORECAST' + is_key: 'N' + + - task_name: m_ODS_CSM_ADJUSTMENT_ITEM_PARSE + ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_ITEM + output_table: LM_CSM_ADJUSTMENTS_ITEM + namespaces: + ns2: http://escb.ecb.int/csm-adjustment + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName' + column_header: 'FORECAST_NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount' + column_header: 'ADJUSTMENT_AMOUNT' + is_key: 'N' + + + \ No newline at end of file diff --git a/airflow/ods/lm/csm_adjustment/dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py b/airflow/ods/lm/csm_adjustment/dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py new file mode 100644 index 0000000..358566f --- /dev/null +++ b/airflow/ods/lm/csm_adjustment/dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py @@ -0,0 +1,520 @@ +# dags/m_ODS_LM_CSM_ADJUSTMENT.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "CSM_ADJUTMENT"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_CSM_ADJUSTMENT" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_CSM_ADJUSTMENT" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_CSM_ADJUSTMENT" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_CSM_ADJUSTMENT", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_CSM_ADJUSTMENT", + trigger_dag_id="w_MOPDB_LM_CSM_ADJUSTMENT", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/current_accounts/config/.gitkeep b/airflow/ods/lm/current_accounts/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/current_accounts/config/current_accounts.xsd b/airflow/ods/lm/current_accounts/config/current_accounts.xsd new file mode 100644 index 0000000..6810702 --- /dev/null +++ b/airflow/ods/lm/current_accounts/config/current_accounts.xsd @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + Represents current accounts of a CB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Represents the disaggregated current accounts + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml b/airflow/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml new file mode 100644 index 0000000..5e3c946 --- /dev/null +++ b/airflow/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml @@ -0,0 +1,91 @@ +tmpdir: /tmp +inbox_prefix: INBOX/LM/MinimumReservesRequirements +archive_prefix: ARCHIVE/LM/MinimumReservesRequirements +workflow_name: w_ODS_LM_CURRENT_ACCOUNTS +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/current_accounts/config/current_accounts.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_CURRENT_ACCOUNTS_HEADER_PARSE + ods_prefix: INBOX/LM/MinimumReservesRequirements/LM_CURRENT_ACCOUNTS_HEADER + output_table: LM_CURRENT_ACCOUNTS_HEADER + namespaces: + ns2: http://escb.ecb.int/ca + output_columns: + - type: 'xpath_element_id' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:version' + column_header: 'REVISION_NUMBER' + is_key: 'N' + - type: 'xpath' + value: 'ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:referenceDate' + column_header: 'REFERENCE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:freeText' + column_header: 'FREE_TEXT' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:currentAccountBSTotal' + column_header: 'CURRENT_ACCOUNT_BS_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:mrrForecastTotal' + column_header: 'MRR_FORECAST_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:currentAccountMessageTotal' + column_header: 'CURRENT_ACCOUNT_MESSAGE_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:mrrMessageTotal' + column_header: 'MRR_MESSAGE_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:periodicity' + column_header: 'PERIODICITY' + is_key: 'N' + + + - task_name: m_ODS_LM_CURRENT_ACCOUNTS_ITEM_PARSE + ods_prefix: INBOX/LM/MinimumReservesRequirements/LM_CURRENT_ACCOUNTS_ITEM + output_table: LM_CURRENT_ACCOUNTS_ITEM + namespaces: + ns2: http://escb.ecb.int/ca + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:mfiCode' + column_header: 'MFI_CODE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:bankName' + column_header: 'BANK_NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:currentAccount' + column_header: 'CURRENT_ACCOUNT' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:minimumReserveRequirement' + column_header: 'MINIMUM_RESERVE_REQUIREMENT' + is_key: 'N' + - type: 'xpath' + value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:comment' + column_header: 'COMMENT_' + is_key: 'N' \ No newline at end of file diff --git a/airflow/ods/lm/current_accounts/dags/w_ODS_LM_CURRENT_ACCOUNT.py b/airflow/ods/lm/current_accounts/dags/w_ODS_LM_CURRENT_ACCOUNT.py new file mode 100644 index 0000000..9fd4743 --- /dev/null +++ b/airflow/ods/lm/current_accounts/dags/w_ODS_LM_CURRENT_ACCOUNT.py @@ -0,0 +1,520 @@ +# dags/w_ODS_LM_ADHOC_CURRENT_ACCOUNTS.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "CURRENT_ACCOUNTS"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_CURRENT_ACCOUNTS" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_CURRENT_ACCOUNTS" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_CURRENT_ACCOUNTS" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_CURRENT_ACCOUNTS", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_CURRENT_ACCOUNTS", + trigger_dag_id="w_MOPDB_LM_CURRENT_ACCOUNTS", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/forecast/config/forecast.xsd b/airflow/ods/lm/forecast/config/forecast.xsd new file mode 100644 index 0000000..2c03c7f --- /dev/null +++ b/airflow/ods/lm/forecast/config/forecast.xsd @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Represents an autonomous factor forecast for a CB + + + + + + + + + + + Represents an aggregated autonomous factor forecast of all CBs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml b/airflow/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml new file mode 100644 index 0000000..3d5e44a --- /dev/null +++ b/airflow/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml @@ -0,0 +1,63 @@ +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/LM/EurosystemAutonomousFactorForecast +archive_prefix: ARCHIVE/LM/EurosystemAutonomousFactorForecast +workflow_name: w_ODS_LM_FORECAST +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/forecast/config/forecast.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_FORECAST_HEADER_PARSE + ods_prefix: INBOX/LM/EurosystemAutonomousFactorForecast/LM_FORECAST_HEADER + output_table: LM_FORECAST_HEADER + namespaces: + ns2: 'http://escb.ecb.int/forecast' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:forecastMessage/ns2:*' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:referenceDate' + column_header: 'REFERENCE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:version' + column_header: 'REVISION' + is_key: 'N' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:freeText' + column_header: 'FREE_TEXT' + is_key: 'N' + + - task_name: m_ODS_LM_FORECAST_ITEM_PARSE + ods_prefix: INBOX/LM/EurosystemAutonomousFactorForecast/LM_FORECAST_ITEM + output_table: LM_FORECAST_ITEM + namespaces: + ns2: 'http://escb.ecb.int/forecast' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:forecastMessage/ns2:*' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastDate' + column_header: 'FORECAST_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastName' + column_header: 'FORECAST_NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastValue' + column_header: 'FORECAST_VALUE' + is_key: 'N' \ No newline at end of file diff --git a/airflow/ods/lm/forecast/dags/w_ODS_LM_FORECAST.py b/airflow/ods/lm/forecast/dags/w_ODS_LM_FORECAST.py new file mode 100644 index 0000000..3bcd87b --- /dev/null +++ b/airflow/ods/lm/forecast/dags/w_ODS_LM_FORECAST.py @@ -0,0 +1,520 @@ +# dags/w_ODS_LM_FORECAST.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "FORECAST"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_FORECAST" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_FORECAST" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_FORECAST" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_FORECAST", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_FORECAST", + trigger_dag_id="w_MOPDB_LM_FORECAST", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/lm_common/.gitkeep b/airflow/ods/lm/lm_common/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/lm_common/lm.xsd b/airflow/ods/lm/lm_common/lm.xsd new file mode 100644 index 0000000..251a104 --- /dev/null +++ b/airflow/ods/lm/lm_common/lm.xsd @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/quarterly_adjustment/config/.gitkeep b/airflow/ods/lm/quarterly_adjustment/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml b/airflow/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml new file mode 100644 index 0000000..4761e7f --- /dev/null +++ b/airflow/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml @@ -0,0 +1,82 @@ +tmpdir: /tmp +inbox_prefix: INBOX/LM/QuarterlyRevaluationAdjustment +archive_prefix: ARCHIVE/LM/QuarterlyRevaluationAdjustment +workflow_name: w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_HEADER_PARSE + ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_HEADER + output_table: LM_QRE_ADJUSTMENTS_HEADER + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:year' + column_header: 'YEAR' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:quarter' + column_header: 'QUARTER' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:version' + column_header: 'VERSION' + is_key: 'N' + - task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_ITEM_HEADER_PARSE + ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_ITEM_HEADER + output_table: LM_QRE_ADJUSTMENTS_ITEM_HEADER + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:effectiveDate' + column_header: 'EFFECTIVE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast' + column_header: 'LAST_DATE_NOT_FORECAST' + is_key: 'N' + + - task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_ITEM_PARSE + ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_ITEM + output_table: LM_QRE_ADJUSTMENTS_ITEM + namespaces: + ns2: 'http://escb.ecb.int/csm-adjustment' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment' + column_header: 'A_HEADER_FK' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName' + column_header: 'FORECAST_NAME' + is_key: 'N' + - type: 'xpath' + value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount' + column_header: 'ADJUSTMENT_AMOUNT' + is_key: 'N' + + diff --git a/airflow/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd b/airflow/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd new file mode 100644 index 0000000..5154c77 --- /dev/null +++ b/airflow/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/quarterly_adjustment/dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py b/airflow/ods/lm/quarterly_adjustment/dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py new file mode 100644 index 0000000..e91abc6 --- /dev/null +++ b/airflow/ods/lm/quarterly_adjustment/dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py @@ -0,0 +1,520 @@ +# dags/m_ODS_LM_QUARTERLY_ADJUSTMENT.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml", + +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "QUARTERLY_ADJUTMENT"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_QUARTERLY_ADJUSTMENT" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_QUARTERLY_ADJUSTMENT" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_QUARTERLY_ADJUSTMENT" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_QUARTERLY_ADJUSTMENT", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_QRE_ADJUSTMENT", + trigger_dag_id="w_MOPDB_LM_QRE_ADJUSTMENT", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml b/airflow/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml new file mode 100644 index 0000000..85751ec --- /dev/null +++ b/airflow/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml @@ -0,0 +1,85 @@ +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/LM/DistributeStandingFacilities +archive_prefix: ARCHIVE/LM/DistributeStandingFacilities +workflow_name: w_ODS_LM_Standing_Facilities +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/sf.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_LM_Standing_Facilities_HEADER_PARSE + ods_prefix: INBOX/LM/DistributeStandingFacilities/LM_STANDING_FACILITIES_HEADER + output_table: LM_STANDING_FACILITIES_HEADER + namespaces: + ns2: 'http://escb.ecb.int/sf' + output_columns: + - type: 'xpath_element_id' + value: '//ns2:header' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns2:header/ns2:version' + column_header: 'REV_NUMBER' + is_key: 'N' + - type: 'xpath' + value: '//ns2:header/ns2:referenceDate' + column_header: 'REF_DATE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'FREE_TEXT' + - type: 'xpath' + value: '//ns2:header/ns2:marginalLendingBSTotal' + column_header: 'MLF_BS_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '//ns2:header/ns2:depositFacilityBSTotal' + column_header: 'DF_BS_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '//ns2:header/ns2:marginalLendingSFTotal' + column_header: 'MLF_SF_TOTAL' + is_key: 'N' + - type: 'xpath' + value: '//ns2:header/ns2:depositFacilitySFTotal' + column_header: 'DF_SF_TOTAL' + is_key: 'N' + + - task_name: m_ODS_LM_Standing_Facilities_ITEM_PARSE + ods_prefix: INBOX/LM/DistributeStandingFacilities/LM_STANDING_FACILITIES + output_table: LM_STANDING_FACILITIES + namespaces: + ns2: 'http://escb.ecb.int/sf' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '//ns2:header' + column_header: 'A_SFH_FK' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:country' + column_header: 'COUNTRY' + is_key: 'N' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:mfiCode' + column_header: 'MFI_ID' + is_key: 'N' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:bankName' + column_header: 'MFI_NAME' + is_key: 'N' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:marginalLending' + column_header: 'MARGINAL_LENDING_FACILITY' + is_key: 'N' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:depositFacility' + column_header: 'DEPOSIT_FACILITY' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'COMMENT_' diff --git a/airflow/ods/lm/standing_facilities/config/sf.xsd b/airflow/ods/lm/standing_facilities/config/sf.xsd new file mode 100644 index 0000000..8ef22f7 --- /dev/null +++ b/airflow/ods/lm/standing_facilities/config/sf.xsd @@ -0,0 +1,102 @@ + + + + + +Represents standing facilities of a CB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Represents the disaggregated standing facilities + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/ods/lm/standing_facilities/dags/.gitkeep b/airflow/ods/lm/standing_facilities/dags/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES.py b/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES.py new file mode 100644 index 0000000..32a4ab8 --- /dev/null +++ b/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES.py @@ -0,0 +1,519 @@ +# dags/w_ODS_LM_STANDING_FACILITIES.py +# Idempotent, per-object mtime tracking + +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta, datetime, timezone +from email.utils import parsedate_to_datetime + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# OCI settings +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML (single config for all files) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml", +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only) +PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float) + + +# Helpers + +def _oci_client(): + """ + Create an OCI Object Storage client. + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + # RP + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} if region else {} + logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + logging.info("RP not available: %s", e) + # IP + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} if region else {} + logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region")) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + logging.info("IP not available: %s", e) + + logging.error("Neither Resource Principals nor Instance Principals authentication found.") + raise RuntimeError("Failed to create OCI client") + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Build config-derived constants directly from YAML +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix") + if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()): + raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.") + OBJECT_PREFIX = OBJECT_PREFIX.strip() + logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +# New idempotency map (key -> last_processed_ts) +def _load_processed_map() -> dict[str, float]: + """ + Returns {object_key: last_processed_ts}. + Back-compat: if old set variable exists (list), treat those keys as ts=0. + """ + try: + raw = Variable.get(PROCESSED_TS_VAR, default_var="{}") + m = json.loads(raw) or {} + if isinstance(m, dict): + return {k: float(v) for k, v in m.items()} + except Exception: + pass + # Back-compat: migrate old set/list + try: + old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")) + if isinstance(old, list): + return {k: 0.0 for k in old} + except Exception: + pass + return {} + +def _save_processed_map(m: dict[str, float]) -> None: + Variable.set(PROCESSED_TS_VAR, json.dumps(m)) + +def _mark_processed_ts(objs: list[tuple[str, float]]): + """ + Update processed map with list of (object_key, mtime). + """ + if REPROCESS or not objs: + return + m = _load_processed_map() + for key, ts in objs: + try: + ts = float(ts) + except Exception: + continue + prev = float(m.get(key, 0.0)) + if ts > prev: + m[key] = ts + _save_processed_map(m) + logging.info("Processed map updated; size=%d", len(m)) + +# Object listing (per-key mtime) +def _list_new_xml_objects(prefix: str) -> list[dict]: + """ + List .xml objects and decide inclusion per-object: + include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0) + Returns: [{"name": "", "base": "", "mtime": }] + """ + if not OCI_NAMESPACE or not OCI_BUCKET: + raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set") + + client = _oci_client() + processed_map = _load_processed_map() + + try: + last_seen = float(Variable.get(LAST_TS_VAR, default_var="0")) + except Exception: + last_seen = 0.0 + + logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s", + last_seen, len(processed_map), prefix) + + # NOTE: add pagination if needed + resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix) + + new_items: list[dict] = [] + newest_ts = last_seen + + for o in (resp.data.objects or []): + name = (o.name or "").strip() + base = name.rsplit("/", 1)[-1] if name else "" + logging.info("Processing object: %s", base) + + # Skip folder markers / empty keys + if not name or name.endswith('/') or not base: + logging.debug("Skip: folder marker or empty key: %r", name) + continue + + if not base.lower().endswith(".xml"): + logging.debug("Skip: not .xml: %r", name) + continue + + # Resolve mtime + ts = None + t = getattr(o, "time_created", None) + if t: + try: + ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0 + except Exception: + ts = None + + if ts is None: + try: + head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name) + lm = head.headers.get("last-modified") or head.headers.get("Last-Modified") + if lm: + dt = parsedate_to_datetime(lm) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + ts = dt.timestamp() + logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts) + except Exception as e: + logging.warning("head_object failed for %s: %s", name, e) + + if ts is None: + ts = datetime.now(timezone.utc).timestamp() + logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts) + + last_proc_ts = float(processed_map.get(name, 0.0)) + include = REPROCESS or (ts > last_proc_ts) + + logging.info( + "Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s", + name, ts, last_proc_ts, REPROCESS, include + ) + + if not include: + continue + + item = {"name": name, "base": base, "mtime": ts} + new_items.append(item) + if ts > newest_ts: + newest_ts = ts + + # Watermark advanced for visibility (optional) + if not REPROCESS and new_items and newest_ts > last_seen: + Variable.set(LAST_TS_VAR, str(newest_ts)) + logging.info("Advanced watermark from %s to %s", last_seen, newest_ts) + + new_items.sort(key=lambda x: x["mtime"]) # ascending + logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix) + return new_items + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (polling): single YAML config for all XML files in OCI', + schedule_interval=None, # Run EVERY 10 MIN + catchup=False, + max_active_runs=1, + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "STANDING_FACILITIES"], +) as dag: + + @af_task(task_id="poll_oci_for_xml") + def poll_oci_for_xml(): + """ + Lists new .xml objects and prepares a workload list. + Returns {"workload": [{"object": "", "base": "", "mtime": } ...]} + """ + if not OBJECT_PREFIX: + raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.") + + new_objs = _list_new_xml_objects(OBJECT_PREFIX) + logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2)) + print("New .xml objects found:", json.dumps(new_objs, indent=2)) + + # already contains base + mtime + workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs] + logging.info("Prepared workload items: %d", len(workload)) + print("Prepared workload:", json.dumps(workload, indent=2)) + return {"workload": workload} + + @af_task(task_id="init_workflow") + def init_workflow(polled: dict): + """Initialize workflow; start MRDS workflow; build per-file task configs.""" + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + workload = (polled or {}).get("workload") or [] + + # Airflow context for run_id + from airflow.operators.python import get_current_context + ctx = get_current_context() + run_id = str(ctx['ti'].run_id) + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build TASK_CONFIGS dynamically: one per file, sequential numbering + task_base_name = "m_ODS_LM_STANDING_FACILITIES" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], # pass basename to MRDS (adjust if you need full key) + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes object + mtime + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs)) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # resolve full object key + mtime by matching base name from workload + full_object_key, object_mtime = None, None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + object_mtime = w.get('mtime') + break + + # Print/log the file being processed + logging.info("%s: picking file %s (object=%s, mtime=%s)", + task_name, source_filename, full_object_key or source_filename, object_mtime) + print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})") + + try: + # NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key' + mrds_main( + workflow_context, + source_filename, # or full_object_key if required in your env + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, source_filename) + raise + + # Mark processed with the mtime we saw during poll + if full_object_key and object_mtime: + _mark_processed_ts([(full_object_key, object_mtime)]) + + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_STANDING_FACILITIES" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + def check_success_for_mopdb(**context): + """Check if all processing tasks succeeded before triggering MOPDB.""" + from airflow.utils.state import State + + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + # Check finalize_workflow task + finalize_task = dag_run.get_task_instance('finalize_workflow') + if finalize_task.state == State.FAILED: + has_failures = True + failure_reasons.append("finalize_workflow failed") + + # Check all mapped tasks (per-file processing) + mapped_task_id = "m_ODS_LM_STANDING_FACILITIES" + mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + for task_instance in mapped_tasks: + if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}: + has_failures = True + map_idx = getattr(task_instance, 'map_index', 'unknown') + failure_reasons.append(f"Processing task failed at index {map_idx}") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + # Check if all mapped tasks were skipped (no files to process) + all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True + + if all_skipped or not mapped_tasks: + error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + # Operators & Dependencies + poll_task = poll_oci_for_xml() + init_out = init_workflow(poll_task) + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_STANDING_FACILITIES", max_active_tis_per_dag=1) + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_STANDING_FACILITY", + trigger_dag_id="w_MOPDB_LM_STANDING_FACILITY", + conf={ + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=False, # CHANGED: Don't wait for completion + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds + retries=0, + ) + + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end + ) + + # CHANGED: Chain with check task before trigger + poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good + +logging.info( + "EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.", + OBJECT_PREFIX, PROCESSED_TS_VAR +) diff --git a/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES_event_driven.py b/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES_event_driven.py new file mode 100644 index 0000000..a51ffde --- /dev/null +++ b/airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES_event_driven.py @@ -0,0 +1,354 @@ +# dags/w_ODS_LM_STANDING_FACILITIES_event.py +import sys +import os +import json +import logging +from pathlib import Path +from datetime import timedelta + +from airflow import DAG +from airflow.models import Variable +from airflow.decorators import task as af_task +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.operators.empty import EmptyOperator +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +# Import libs +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi') +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from mrds.core import main as mrds_main + + +# DAG / Defaults + +dag_id = Path(__file__).stem + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id, +} + +# Optional OCI settings (not used for listing in event mode, but used for processed set keys) +OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE") +OCI_BUCKET = os.getenv("INBOX_BUCKET") + +# Config YAML path (env override) +CONFIG_YAML = os.getenv( + "EXDI_SINGLE_CONFIG_YAML", + "/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml", +) +logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML) + +# Idempotency controls +REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes")) +PROCESSED_SET_VAR = f"{dag_id}__processed_objects" + +# If your MRDS expects full object keys for pFileUri, keep this True (default). +# Set EXDI_MRDS_USE_FULL_URI=false to send only the basename instead. +USE_FULL_URI_FOR_MRDS = (os.getenv("EXDI_MRDS_USE_FULL_URI", "true").lower() in ("1", "true", "yes")) + + +# Helpers + +def _load_yaml(cfg_path: str) -> dict: + import yaml + p = Path(cfg_path) + if not p.exists(): + raise FileNotFoundError(f"Config YAML not found: {cfg_path}") + return yaml.safe_load(p.read_text()) or {} + +# Load inbox_prefix for logging / optional validation +try: + CONFIG_DATA = _load_yaml(CONFIG_YAML) + OBJECT_PREFIX = (CONFIG_DATA.get("inbox_prefix") or "").strip() or None + logging.info("YAML inbox_prefix (context only): %s", OBJECT_PREFIX) +except Exception as e: + logging.error("Failed to load CONFIG_YAML %s: %s", CONFIG_YAML, e) + OBJECT_PREFIX = None + +def _mark_processed(objs: list[str]): + """Add object keys to processed set (bounded).""" + if REPROCESS or not objs: + return + try: + processed = list(set(json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")))) + except Exception: + processed = [] + cap = 5000 + # append new (unique) keys + for o in objs: + if o not in processed: + processed.append(o) + if len(processed) > cap: + processed = processed[-cap:] + logging.info("Updated processed set size=%d", len(processed)) + Variable.set(PROCESSED_SET_VAR, json.dumps(processed)) + + +def _extract_workload_from_conf(conf: dict) -> list[dict]: + """Build workload from dag_run.conf | Returns: [{"object": "", "base": ""} ...]""" + out = [] + if not conf: + return out + + single = conf.get("object") + many = conf.get("objects") + + def _add(obj, mtime=None): + if isinstance(obj, str) and obj.strip(): + key = obj.strip() + out.append({ + "object": key, + "base": key.rsplit("/", 1)[-1], + "mtime": mtime, + }) + + if isinstance(single, str): + _add(single, conf.get("mtime")) # optional single mtime + + if isinstance(many, list): + for o in many: + if isinstance(o, dict): + _add(o.get("object"), o.get("mtime")) + else: + _add(o) + + return out + + +# DAG + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='EXDI workflow (event-driven): process objects from dag_run.conf using a single YAML config', + schedule_interval=None, # Manual/event only + catchup=False, + is_paused_upon_creation=True, # show up paused initially + render_template_as_native_obj=True, + tags=["EXDI", "MRDS", "ODS", "OCI", "event"], +) as dag: + + @af_task(task_id="init_workflow") + def init_workflow(): + """ + Read dag_run.conf, create MRDS workflow, and build per-file task configs. + """ + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing = [] + if not username: missing.append("MRDS_LOADER_DB_USER") + if not password: missing.append("MRDS_LOADER_DB_PASS") + if not tnsalias: missing.append("MRDS_LOADER_DB_TNS") + raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}") + + # Access dag_run.conf + from airflow.operators.python import get_current_context + ctx = get_current_context() + dag_run = ctx.get("dag_run") + conf = (dag_run.conf or {}) if dag_run else {} + + workload = _extract_workload_from_conf(conf) + if not workload: + raise AirflowSkipException("No objects provided in dag_run.conf (expected 'object' or 'objects').") + + # Sort by mtime (first incoming file first) + workload.sort(key=lambda w: w.get("mtime") if w.get("mtime") is not None else float("inf")) # added fall back mtime, if Not needed can remove + + # Start MRDS workflow run + run_id = str(ctx['ti'].run_id) + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id) + + workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key + } + + # Build per-file task configs + task_base_name = "m_ODS_LM_STANDING_FACILITIES" + task_configs = [] + for idx, w in enumerate(workload, start=1): + task_configs.append({ + "task_name": f"{task_base_name}_{idx}", + "source_filename": w["base"], + "config_file": CONFIG_YAML, + }) + + bundle = { + "workflow_history_key": a_workflow_history_key, + "workflow_context": workflow_context, + "workload": workload, # includes full 'object' keys + "task_configs": task_configs, # list-of-dicts for mapping + "env": env, + } + + logging.info("Event init complete; tasks=%d; objects=%s", + len(task_configs), [w['object'] for w in workload]) + return bundle + + @af_task(task_id="get_task_configs") + def get_task_configs(init_bundle: dict): + return init_bundle["task_configs"] + + def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context): + """Run MRDS for a single file (sequential via mapped task).""" + ti = context['ti'] + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Config file not found: {config_file}") + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + workflow_context = init_bundle.get('workflow_context') + workload = init_bundle.get('workload') or [] + if not workflow_context: + raise AirflowFailException("No workflow_context from init_workflow") + + # Re-read processed set + try: + processed = set(json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))) + except Exception: + processed = set() + + # Resolve full object key by matching basename + full_object_key = None + for w in workload: + if w.get('base') == source_filename: + full_object_key = w.get('object') + break + + if (not REPROCESS) and full_object_key and (full_object_key in processed): + logging.info("%s: skipping already-processed %s", task_name, full_object_key) + ti.xcom_push(key='task_status', value='SUCCESS_NOOP') + return "NOOP" + + # Decide pFileUri for MRDS + if USE_FULL_URI_FOR_MRDS and full_object_key: + file_uri = full_object_key + else: + # fall back to basename + file_uri = source_filename + + logging.info("%s: picking file %s (object=%s) -> MRDS pFileUri=%s", + task_name, source_filename, full_object_key or source_filename, file_uri) + print(f"{task_name}: MRDS pFileUri -> {file_uri}") + + try: + mrds_main( + workflow_context, + file_uri, # pass the URI MRDS should match in A_SOURCE_FILE_CONFIG + config_file, + generate_workflow_context=False + ) + except Exception: + logging.exception("%s: MRDS failed on %s", task_name, file_uri) + raise + + if full_object_key: + _mark_processed([full_object_key]) + ti.xcom_push(key='task_status', value='SUCCESS') + logging.info("%s: success", task_name) + return "SUCCESS" + + def finalise_workflow_task(**context): + """Finalize workflow across all per-file tasks (mapped).""" + from airflow.utils.state import State + + ti = context['ti'] + dag_run = context['dag_run'] + + init_bundle = ti.xcom_pull(task_ids='init_workflow') or {} + a_workflow_history_key = init_bundle.get('workflow_history_key') + if a_workflow_history_key is None: + raise AirflowFailException("No workflow history key; cannot finalise workflow") + + mapped_task_id = "m_ODS_LM_STANDING_FACILITIES" + tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id] + + # If no mapped TIs (shouldn't happen unless no work), succeed + if not tis: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key) + return + + any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis) + if not any_failed: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key) + return + + failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}] + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs) + raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}") + + # Operators & Dependencies + init_out = init_workflow() + task_cfgs = get_task_configs(init_out) + + @af_task(task_id="m_ODS_LM_STANDING_FACILITIES", max_active_tis_per_dag=1) # ensures only one mapped task instance to run at once, so they execute sequentially + def mapped_run(task_name: str, source_filename: str, config_file: str, **context): + return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context) + + per_file = mapped_run.expand_kwargs(task_cfgs) + + # Trigger the next DAG and wait for it to finish successfully + trigger_mopdb = TriggerDagRunOperator( + task_id="Trigger_w_MOPDB_LM_STANDING_FACILITY", + trigger_dag_id="w_MOPDB_LM_STANDING_FACILITY", + conf={ + # pass along useful context to the next DAG + "source_dag": dag_id, + "upstream_run_id": "{{ run_id }}", + "objects": "{{ (ti.xcom_pull(task_ids='init_workflow')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='init_workflow') else [] }}", + "workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}" + }, + wait_for_completion=True, # Until the triggered DAG completes + allowed_states=["success"], # treat only SUCCESS as success + failed_states=["failed"], # anything else -> fail + poke_interval=30, # how often to check status (secs) + ) + + # Final "everything went fine" marker — only runs if the triggered DAG succeeded + all_good = EmptyOperator( + task_id="All_went_well", + trigger_rule=TriggerRule.ALL_SUCCESS, + ) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + init_out >> task_cfgs >> per_file >> finalize_workflow >> trigger_mopdb >> all_good + +logging.info("EXDI EVENT DAG ready... Expect object keys in dag_run.conf (object / objects). YAML inbox_prefix=%s", OBJECT_PREFIX) diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml new file mode 100644 index 0000000..a284f7a --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml @@ -0,0 +1,6 @@ +# Flow Configuration +CONNECTOR_TYPE: "casper_connector" +ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS/RQSD_ANNEX_" +TASK_NAME: "m_ODS_RQSD_CASPER" +OUTPUT_TABLE: "RQSD_ANNEX_" +COLLECTION_ID: "1537" \ No newline at end of file diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml new file mode 100644 index 0000000..14a52cd --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml @@ -0,0 +1,19 @@ +# Flow Configuration +CONNECTOR_TYPE: "devo_connector" +ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS" +TASK_NAME: "m_ODS_RQSD_FX" +OUTPUT_TABLE: "RQSD_FX" +DEVO_QUERY: "select * from + (select tec_ingestion_date, + obs_value, + pow(10, cast(unit_mult as int)) as divisor, + series_key, + time_period, + ROW_NUMBER() OVER (PARTITION BY series_key, time_period ORDER BY tec_ingestion_date DESC) AS rn + from crp_other_pub.fx + where freq = 'Q' + and data_type_fm = 'HSTE' + and provider_fm = 'BL' + and instrument_fm = 'FX' + and currency != 'EUR') t +where t.rn = 1" diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml new file mode 100644 index 0000000..ebbebed --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml @@ -0,0 +1,34 @@ +# static configs +tmpdir: /tmp +inbox_prefix: INBOX/RQSD/RQSD_PROCESS +archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS +workflow_name: w_ODS_RQSD_PROCESS_DEVO +validation_schema_path: None +file_type: csv + +# task configs +tasks: + - task_name: m_ODS_RQSD_FX_PARSE + ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_FX + output_table: RQSD_FX + output_columns: + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'tec_ingestion_date' + column_header: 'tec_ingestion_date' + - type: 'csv_header' + value: 'obs_value' + column_header: 'obs_value' + - type: 'csv_header' + value: 'divisor' + column_header: 'divisor' + - type: 'csv_header' + value: 'series_key' + column_header: 'series_key' + - type: 'csv_header' + value: 'time_period' + column_header: 'time_period' + - type: 'csv_header' + value: 'rn' + column_header: 'rn' diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml new file mode 100644 index 0000000..0581231 --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml @@ -0,0 +1,31 @@ +# Flow Configuration +CONNECTOR_TYPE: "devo_connector" +ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS" +TASK_NAME: "m_ODS_RQSD_OBSERVATIONS" +OUTPUT_TABLE: "RQSD_OBSERVATIONS" +DEVO_QUERY: "with latest_dates as ( +SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_all_mrds +UNION +SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_all_mrds +UNION +SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_fin_all_mrds +UNION +SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_fin_all_mrds +UNION +SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_2_all_mrds), +latest_update as ( +SELECT max(max_date) as max_date FROM latest_dates), +latest as ( +select case when count(max_date) = 0 then '1999-01-01 10:28:22' else max(max_date) end max_date +from latest_update) +SELECT a.*, + CAST(NULL AS STRING) AS tec_source_system, + CAST(NULL AS STRING) AS tec_dataset, + CAST(NULL AS STRING) AS tec_surrogate_key, + CAST(NULL AS STRING) AS tec_crc, + CAST(NULL AS STRING) AS tec_ingestion_date, + CAST(NULL AS STRING) AS tec_version_id, + CAST(NULL AS STRING) AS tec_execution_date, + CAST(NULL AS STRING) AS tec_run_id, + CAST(NULL AS STRING) AS tec_business_date + FROM crp_rqsd.tcrqsd_observations a where receivedfilereceiveddate > (select max(max_date) from latest)" diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml new file mode 100644 index 0000000..6bc05be --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml @@ -0,0 +1,181 @@ +# static configs +tmpdir: /tmp +inbox_prefix: INBOX/RQSD/RQSD_PROCESS +archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS +workflow_name: w_ODS_RQSD_PROCESS_DEVO +validation_schema_path: None +file_type: csv + +# task configs +tasks: + - task_name: m_ODS_RQSD_OBSERVATIONS_PARSE + ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_OBSERVATIONS + output_table: RQSD_OBSERVATIONS + output_columns: + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'datacollectioncode' + column_header: 'datacollectioncode' + - type: 'csv_header' + value: 'datacollectionname' + column_header: 'datacollectionname' + - type: 'csv_header' + value: 'datacollectionowner' + column_header: 'datacollectionowner' + - type: 'csv_header' + value: 'reportingcyclename' + column_header: 'reportingcyclename' + - type: 'csv_header' + value: 'reportingcyclestatus' + column_header: 'reportingcyclestatus' + - type: 'csv_header' + value: 'modulecode' + column_header: 'modulecode' + - type: 'csv_header' + value: 'modulename' + column_header: 'modulename' + - type: 'csv_header' + value: 'moduleversionnumber' + column_header: 'moduleversionnumber' + - type: 'csv_header' + value: 'reportingentitycollectionuniqueid' + column_header: 'reportingentitycollectionuniqueid' + - type: 'csv_header' + value: 'entityattributereportingcode' + column_header: 'entityattributereportingcode' + - type: 'csv_header' + value: 'reportingentityname' + column_header: 'reportingentityname' + - type: 'csv_header' + value: 'reportingentityentitytype' + column_header: 'reportingentityentitytype' + - type: 'csv_header' + value: 'entityattributecountry' + column_header: 'entityattributecountry' + - type: 'csv_header' + value: 'entitygroupentityname' + column_header: 'entitygroupentityname' + - type: 'csv_header' + value: 'obligationmodulereferencedate' + column_header: 'obligationmodulereferencedate' + - type: 'csv_header' + value: 'obligationmoduleremittancedate' + column_header: 'obligationmoduleremittancedate' + - type: 'csv_header' + value: 'receivedfilereceiveddate' + column_header: 'receivedfilereceiveddate' + - type: 'csv_header' + value: 'obligationmoduleexpected' + column_header: 'obligationmoduleexpected' + - type: 'csv_header' + value: 'receivedfileversionnumber' + column_header: 'receivedfileversionnumber' + - type: 'csv_header' + value: 'revalidationversionnumber' + column_header: 'revalidationversionnumber' + - type: 'csv_header' + value: 'revalidationdate' + column_header: 'revalidationdate' + - type: 'csv_header' + value: 'receivedfilesystemfilename' + column_header: 'receivedfilesystemfilename' + - type: 'csv_header' + value: 'obligationstatusstatus' + column_header: 'obligationstatusstatus' + - type: 'csv_header' + value: 'filestatussetsubmissionstatus' + column_header: 'filestatussetsubmissionstatus' + - type: 'csv_header' + value: 'filestatussetvalidationstatus' + column_header: 'filestatussetvalidationstatus' + - type: 'csv_header' + value: 'filestatussetexternalvalidationstatus' + column_header: 'filestatussetexternalvalidationstatus' + - type: 'csv_header' + value: 'numberoferrors' + column_header: 'numberoferrors' + - type: 'csv_header' + value: 'numberofwarnings' + column_header: 'numberofwarnings' + - type: 'csv_header' + value: 'delayindays' + column_header: 'delayindays' + - type: 'csv_header' + value: 'failedattempts' + column_header: 'failedattempts' + - type: 'csv_header' + value: 'observationvalue' + column_header: 'observationvalue' + - type: 'csv_header' + value: 'observationtextvalue' + column_header: 'observationtextvalue' + - type: 'csv_header' + value: 'observationdatevalue' + column_header: 'observationdatevalue' + - type: 'csv_header' + value: 'datapointsetdatapointidentifier' + column_header: 'datapointsetdatapointidentifier' + - type: 'csv_header' + value: 'datapointsetlabel' + column_header: 'datapointsetlabel' + - type: 'csv_header' + value: 'obsrvdescdatatype' + column_header: 'obsrvdescdatatype' + - type: 'csv_header' + value: 'ordinatecode' + column_header: 'ordinatecode' + - type: 'csv_header' + value: 'ordinateposition' + column_header: 'ordinateposition' + - type: 'csv_header' + value: 'tablename' + column_header: 'tablename' + - type: 'csv_header' + value: 'isstock' + column_header: 'isstock' + - type: 'csv_header' + value: 'scale' + column_header: 'scale' + - type: 'csv_header' + value: 'currency' + column_header: 'currency' + - type: 'csv_header' + value: 'numbertype' + column_header: 'numbertype' + - type: 'csv_header' + value: 'ismandatory' + column_header: 'ismandatory' + - type: 'csv_header' + value: 'decimalplaces' + column_header: 'decimalplaces' + - type: 'csv_header' + value: 'serieskey' + column_header: 'serieskey' + - type: 'csv_header' + value: 'tec_source_system' + column_header: 'tec_source_system' + - type: 'csv_header' + value: 'tec_dataset' + column_header: 'tec_dataset' + - type: 'csv_header' + value: 'tec_surrogate_key' + column_header: 'tec_surrogate_key' + - type: 'csv_header' + value: 'tec_crc' + column_header: 'tec_crc' + - type: 'csv_header' + value: 'tec_ingestion_date' + column_header: 'tec_ingestion_date' + - type: 'csv_header' + value: 'tec_version_id' + column_header: 'tec_version_id' + - type: 'csv_header' + value: 'tec_execution_date' + column_header: 'tec_execution_date' + - type: 'csv_header' + value: 'tec_run_id' + column_header: 'tec_run_id' + - type: 'csv_header' + value: 'tec_business_date' + column_header: 'tec_business_date' \ No newline at end of file diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml new file mode 100644 index 0000000..28218c3 --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml @@ -0,0 +1,736 @@ +# Flow Configuration +CONNECTOR_TYPE: "devo_connector" +ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS" +TASK_NAME: "m_ODS_RQSD_SUBA_DEVO" +OUTPUT_TABLE: "RQSD_SUBA_DEVO" +DEVO_QUERY: "with suba_reporting_requirements_temp as ( + SELECT sender, + riad_code, + name, + tid, + entity_id, + module_name, + CASE + WHEN cons_level = 'IND' THEN 'SOLO' + WHEN cons_level = 'CON' THEN 'CONS' + ELSE cons_level + END AS cons_level, + accounting_standard, + template_id, + reference_date, + expected, + module_id, + dsd_id, + changed_date, + tec_source_system, + tec_dataset, + tec_surrogate_key, + tec_crc, + tec_ingestion_date, + tec_version_id, + tec_execution_date, + tec_run_id, + tec_business_date + FROM crp_suba06.suba_reporting_requirements), + observations_1_2 as + ( + select OBLIGATIONMODULEREFERENCEDATE, + RECEIVEDFILERECEIVEDDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS ETWDR_COMP_KEY, + REPORTINGENTITYCOLLECTIONUNIQUEID, + DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS STRING), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBS_VALUE + FROM crp_rqsd.rqsd_rqsd01_observations + WHERE MODULECODE = 'SCOPF' + AND TABLENAME = 'etwdr' + AND obligationmodulereferencedate not like '%2019-12-06%' + AND obligationmodulereferencedate not like '%2017%'), + max_version_1_2_fin_all as + ( + SELECT max(RECEIVEDFILERECEIVEDDATE) as max_version, reportingentitycollectionuniqueid, obligationmodulereferencedate + FROM observations_1_2 + group by reportingentitycollectionuniqueid, obligationmodulereferencedate), + annex_1_2_fin_all_pivoted as + (SELECT + OBLIGATIONMODULEREFERENCEDATE, + ETWDR_COMP_KEY, + RECEIVEDFILERECEIVEDDATE, + MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'ann_1_2_ref_date' THEN OBS_VALUE END) AS ANN_1_2_REF_DATE, + MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_mfi_id' THEN OBS_VALUE END) AS MFI_ID, + MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_lei' THEN OBS_VALUE END) AS LEGAL_ENTITY_ID, + MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_name' THEN OBS_VALUE END) AS INST_NAME, + MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_submitter' THEN OBS_VALUE END) AS ETWDR_SUBMITTER + FROM + observations_1_2 + WHERE RECEIVEDFILERECEIVEDDATE in (select max_version from max_version_1_2_fin_all) + GROUP BY + OBLIGATIONMODULEREFERENCEDATE, ETWDR_COMP_KEY, RECEIVEDFILERECEIVEDDATE), + scopf as + ( + select mfi_id, inst_name, legal_entity_id, obligationmodulereferencedate, ann_1_2_ref_date from annex_1_2_fin_all_pivoted where etwdr_submitter = 'SUP' + ), + EXP_COREP_CONS as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'N' + end as EXP_COREP_CONS + from suba_reporting_requirements_temp + where template_id = 'tgC_03.00' + and MODULE_NAME = 'COREP_OF' + and CONS_LEVEL = 'CONS' + ), + EXP_COREP_SOLO as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'N' + end as EXP_COREP_SOLO + from suba_reporting_requirements_temp + where template_id = 'tgC_03.00' + and MODULE_NAME = 'COREP_OF' + and CONS_LEVEL = 'SOLO' + ), + EXP_FINREP_CONS as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'N' + end as EXP_FINREP_CONS + from suba_reporting_requirements_temp + where template_id = 'tgF_00.01' + and MODULE_NAME in ('FINREP9', 'FINREP9_DP') + and CONS_LEVEL = 'CONS' + ), + EXP_FINREP_SOLO as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'N' + end as EXP_FINREP_SOLO + from suba_reporting_requirements_temp + where template_id = 'tgF_00.01' + and MODULE_NAME in ('FINREP9', 'FINREP9_DP') + and CONS_LEVEL = 'SOLO' + ), + EXP_LEV_CONS as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'full&trans' + when EXPECTED = 'O' then 'none' + else 'none' + end as EXP_LEV_CONS + from suba_reporting_requirements_temp + where template_id = 'tgC_47.00' + and MODULE_NAME in ('COREP_LR') + and CONS_LEVEL = 'CONS' + ), + EXP_LEV_SOLO as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'full&trans' + when EXPECTED = 'O' then 'none' + else 'none' + end as EXP_LEV_SOLO + from suba_reporting_requirements_temp + where template_id = 'tgC_47.00' + and MODULE_NAME in ('COREP_LR') + and CONS_LEVEL = 'SOLO' + ), + EXP_LCR_CONS as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'O' + end as EXP_LCR_CONS + from suba_reporting_requirements_temp + where template_id = 'tgC_76.00' + and MODULE_NAME = 'COREP_LCR_DA' + and CONS_LEVEL = 'CONS' + ), + EXP_LCR_SOLO as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'O' + end as EXP_LCR_SOLO + from suba_reporting_requirements_temp + where template_id = 'tgC_76.00' + and MODULE_NAME = 'COREP_LCR_DA' + and CONS_LEVEL = 'SOLO' + ), + EXP_NSFR_CONS as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'O' + end as EXP_NSFR_CONS + from suba_reporting_requirements_temp + where template_id = 'tgC_84.00' + and MODULE_NAME = 'COREP_NSFR' + and CONS_LEVEL = 'CONS' + ), + EXP_NSFR_SOLO as ( + select riad_code, + reference_date, + module_name, + cons_level, + template_id, + case + when EXPECTED = 'E' then 'Y' + when EXPECTED = 'O' then 'O' + else 'O' + end as EXP_NSFR_SOLO + from suba_reporting_requirements_temp + where template_id = 'tgC_84.00' + and MODULE_NAME = 'COREP_NSFR' + and CONS_LEVEL = 'SOLO' + ), + corep_cons_c100 as ( + select riad_code, + reported_period, + cons_level, + name, + lei, + c_0100_r0010_c0010, + c_0100_r0020_c0010, + c_0100_r0015_c0010 + from crp_suba06.suba_c_0100 + where cons_level = 'CONS' + ), + corep_solo_c100 as ( + select riad_code, + reported_period, + cons_level, + lei, + c_0100_r0010_c0010, + c_0100_r0020_c0010, + c_0100_r0015_c0010 + from crp_suba06.suba_c_0100 + where cons_level = 'SOLO' + ), + corep_cons_c300 as ( + select riad_code, + reported_period, + cons_level, + lei, + c_0300_r0010_c0010, + c_0300_r0030_c0010, + c_0300_r0050_c0010 + from crp_suba06.suba_c_0300 + where cons_level = 'CONS' + ), + corep_solo_c300 as ( + select riad_code, + reported_period, + cons_level, + lei, + c_0300_r0010_c0010, + c_0300_r0030_c0010, + c_0300_r0050_c0010 + from crp_suba06.suba_c_0300 + where cons_level = 'SOLO' + ), + corep_cons_c200 as ( + select riad_code, + reported_period, + cons_level, + lei, + c_0200_r0010_c0010 + from crp_suba06.suba_c_0200 + where cons_level = 'CONS' and reported_period<'2025-03-31 00:00:00' + union all + select riad_code, + reported_period, + cons_level, + lei, + c_0200a_r0010_c0010 + from crp_suba06.suba_c_0200a as c_0200_r0010_c0010 + where cons_level = 'CONS' and reported_period>='2025-03-31 00:00:00' + ), + corep_solo_c200 as ( + select riad_code, + reported_period, + cons_level, + lei, + c_0200_r0010_c0010 + from crp_suba06.suba_c_0200 + where cons_level = 'SOLO' and reported_period<'2025-03-31 00:00:00' + union all + select riad_code, + reported_period, + cons_level, + lei, + c_0200a_r0010_c0010 + from crp_suba06.suba_c_0200a as c_0200_r0010_c0010 + where cons_level = 'SOLO' and reported_period>='2025-03-31 00:00:00' + ), + finrep_cons as ( + select riad_code, + reported_period, + cons_level, + lei, + f_0101_r0380_c0010 + from crp_suba06.suba_f_0101 + where cons_level = 'CONS' + ), + finrep_solo as ( + select riad_code, + reported_period, + cons_level, + lei, + f_0101_r0380_c0010 + from crp_suba06.suba_f_0101 + where cons_level = 'SOLO' + ), + lev_cons as ( + select riad_code, + reported_period, + cons_level, + lei, + c_4700_r0330_c0010, + c_4700_r0340_c0010, + c_4700_r0410_c0010 + from crp_suba06.suba_c_4700 + where cons_level = 'CONS' + ), + lev_solo as ( + select riad_code, + reported_period, + cons_level, + lei, + c_4700_r0330_c0010, + c_4700_r0340_c0010, + c_4700_r0410_c0010 + from crp_suba06.suba_c_4700 + where cons_level = 'SOLO' + ), + lcr_cons as ( + select riad_code, + reported_period, + cons_level, + lei, + c_7600a_r0030_c0010 + from crp_suba06.suba_c_7600a + where cons_level = 'CONS' + ), + lcr_solo as ( + select riad_code, + reported_period, + cons_level, + lei, + c_7600a_r0030_c0010 + from crp_suba06.suba_c_7600a + where cons_level = 'SOLO' + ), + nsfr_cons as ( + select riad_code, + reported_period, + cons_level, + lei, + c_8400a_r0220_c0040 + from crp_suba06.suba_c_8400a + where cons_level = 'CONS' + ), + nsfr_solo as ( + select riad_code, + reported_period, + cons_level, + lei, + c_8400a_r0220_c0040 + from crp_suba06.suba_c_8400a + where cons_level = 'SOLO' + ), + liq_subgroups as ( + SELECT entity_id, + substr(entity_id, 0, 20) as short_id, + name, + start_date, + end_date + from crp_suba06.suba_entity_master_data_stage2 + where substr(entity_id, -12) = 'CRDLIQSUBGRP' + ), + liq_subgroup_lcr as ( + select liq_subgroups.entity_id, + short_id, + liq_subgroups.name, + start_date, + end_date, + suba_c_7600a.reported_period, + c_7600a_r0030_c0010 + from liq_subgroups + left join crp_suba06.suba_c_7600a on liq_subgroups.entity_id = suba_c_7600a.entity_id + where start_date <= reported_period + and end_date >= reported_period + ), + liq_subgroup_nsfr as ( + select liq_subgroups.entity_id, + short_id, + liq_subgroups.name, + start_date, + end_date, + suba_c_8400a.reported_period, + c_8400a_r0220_c0040 + from liq_subgroups + left join crp_suba06.suba_c_8400a on liq_subgroups.entity_id = suba_c_8400a.entity_id + where start_date <= reported_period + and end_date >= reported_period + ), + liq_subgroup_data_riad as ( + select liq_subgroup_lcr.entity_id, + liq_subgroup_lcr.short_id, + liq_subgroup_lcr.name, + liq_subgroup_lcr.start_date, + liq_subgroup_lcr.end_date, + liq_subgroup_lcr.reported_period, + c_7600a_r0030_c0010 as c_7600a_r0030_c0010_subgr, + c_8400a_r0220_c0040 as c_8400a_r0220_c0040_subgr, + riad_code + from liq_subgroup_lcr + left join crp_suba06.suba_entity_master_data_stage2 on liq_subgroup_lcr.short_id = suba_entity_master_data_stage2.entity_id + and suba_entity_master_data_stage2.start_date <= liq_subgroup_lcr.reported_period + and suba_entity_master_data_stage2.end_date >= liq_subgroup_lcr.reported_period + left join liq_subgroup_nsfr on liq_subgroup_lcr.entity_id = liq_subgroup_nsfr.entity_id + and liq_subgroup_lcr.reported_period = liq_subgroup_nsfr.reported_period + ), + all_dates as ( + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_0100 + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_0300 + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_0200 + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_0200a + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_f_0101 + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_4700 + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_7600a + union + select riad_code, + reported_period, + reception_date + from crp_suba06.suba_c_8400a + union + select riad_code, + reference_date, + cast(changed_date as STRING) as reception_date + from suba_reporting_requirements_temp + where template_id in ( + 'tgC_03.00', + 'tgF_00.01', + 'tgC_47.00', + 'tgC_76.00', + 'tgC_84.00' + ) + and reference_date in ( + select distinct obligationmodulereferencedate + from SCOPF + ) + ), + max_dates as ( + select reported_period, + riad_code, + max(reception_date) as receivedfilereceiveddate + from all_dates + group by reported_period, + riad_code + ) + select SCOPF.obligationmodulereferencedate, + cast(null as String) as reportingentitycollectionuniqueid, + cast(NULL as DECIMAL(38, 10)) as receivedfileversionnumber, + max_dates.receivedfilereceiveddate, + cast(NULL as STRING) as revalidationdate, + SCOPF.ann_1_2_ref_date as ref_date, + concat(SCOPF.mfi_id, SCOPF.legal_entity_id) as inst_comp_key, + SCOPF.mfi_id, + SCOPF.legal_entity_id, + SCOPF.inst_name, + 'EUR' as currency, + cast(NULL as STRING) as reported_by_supervisor, + cast(NULL as STRING) as confirmed_by_supervisor, + coalesce(EXP_COREP_CONS.EXP_COREP_CONS, 'N') as EXP_COREP_CONS, + coalesce (EXP_COREP_SOLO.EXP_COREP_SOLO, 'N') as EXP_COREP_SOLO, + coalesce (EXP_FINREP_CONS.EXP_FINREP_CONS, 'N') as EXP_FINREP_CONS, + coalesce (EXP_FINREP_SOLO.EXP_FINREP_SOLO, 'N') as EXP_FINREP_SOLO, + coalesce(EXP_LEV_CONS.EXP_LEV_CONS, 'none') as EXP_LEV_CONS, + coalesce (EXP_LEV_SOLO.EXP_LEV_SOLO, 'none') as EXP_LEV_SOLO, + coalesce (EXP_LCR_CONS.EXP_LCR_CONS, 'N') as EXP_LCR_CONS, + coalesce (EXP_LCR_SOLO.EXP_LCR_SOLO, 'N') as EXP_LCR_SOLO, + coalesce (EXP_NSFR_CONS.EXP_NSFR_CONS, 'N') as EXP_NSFR_CONS, + coalesce (EXP_NSFR_SOLO.EXP_NSFR_SOLO, 'N') as EXP_NSFR_SOLO, + cast( + corep_cons_c100.c_0100_r0020_c0010 as DECIMAL(38, 10) + ) as cons_cet1_amt, + cast( + corep_cons_c100.c_0100_r0015_c0010 as DECIMAL(38, 10) + ) as cons_tier1_amt, + cast( + corep_cons_c100.c_0100_r0010_c0010 as DECIMAL(38, 10) + ) as cons_tot_cap_amt, + cast( + corep_cons_c300.c_0300_r0010_c0010 as DECIMAL(38, 10) + ) as cons_cet1_ratio, + cast( + corep_cons_c300.c_0300_r0030_c0010 as DECIMAL(38, 10) + ) as cons_tier1_ratio, + cast( + corep_cons_c300.c_0300_r0050_c0010 as DECIMAL(38, 10) + ) as cons_tot_cap_ratio, + cast( + corep_cons_c200.c_0200_r0010_c0010 as DECIMAL(38, 10) + ) as cons_risk_wght_assets, + cast( + corep_solo_c100.c_0100_r0020_c0010 as DECIMAL(38, 10) + ) as solo_cet1_amt, + cast( + corep_solo_c100.c_0100_r0015_c0010 as DECIMAL(38, 10) + ) as solo_tier1_amt, + cast( + corep_solo_c100.c_0100_r0010_c0010 as DECIMAL(38, 10) + ) as solo_tot_cap_amt, + cast( + corep_solo_c300.c_0300_r0010_c0010 as DECIMAL(38, 10) + ) as solo_cet1_ratio, + cast( + corep_solo_c300.c_0300_r0030_c0010 as DECIMAL(38, 10) + ) as solo_tier1_ratio, + cast( + corep_solo_c300.c_0300_r0050_c0010 as DECIMAL(38, 10) + ) as solo_tot_cap_ratio, + cast( + corep_solo_c200.c_0200_r0010_c0010 as DECIMAL(38, 10) + ) as solo_risk_wght_assets, + cast(finrep_cons.f_0101_r0380_c0010 as DECIMAL(38, 10)) as cons_tot_assets, + cast(finrep_solo.f_0101_r0380_c0010 as DECIMAL(38, 10)) as solo_tot_assets, + cast(lev_cons.c_4700_r0330_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_full, + cast(lev_cons.c_4700_r0340_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_trans, + cast(lev_cons.c_4700_r0410_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_req, + CASE + WHEN lev_cons.c_4700_r0410_c0010 IS NULL THEN NULL + WHEN lev_cons.c_4700_r0410_c0010 = 0.0300000000 THEN 'N' + ELSE 'Y' + END AS cons_lev_ratio_adj, + cast(lev_solo.c_4700_r0330_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_full, + cast(lev_solo.c_4700_r0340_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_trans, + cast(lev_solo.c_4700_r0410_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_req, + CASE + WHEN lev_solo.c_4700_r0410_c0010 IS NULL THEN NULL + WHEN lev_solo.c_4700_r0410_c0010 = 0.0300000000 THEN 'N' + ELSE 'Y' + END AS solo_lev_ratio_adj, + cast( + case + WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_7600a_r0030_c0010_subgr + ELSE lcr_cons.c_7600a_r0030_c0010 + END as DECIMAL(38, 10) + ) as cons_lc_ratio, + cast(lcr_solo.c_7600a_r0030_c0010 as DECIMAL(38, 10)) as solo_lc_ratio, + cast( + case + WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_8400a_r0220_c0040_subgr + ELSE nsfr_cons.c_8400a_r0220_c0040 + END as DECIMAL(38, 10) + ) as cons_nsfr_ratio, + cast(nsfr_solo.c_8400a_r0220_c0040 as DECIMAL(38, 10)) as solo_nsfr_ratio, + cast(NULL as STRING) as submitter_comment, + cast(NULL as STRING) as datacollectioncode, + cast(NULL as STRING) as reportingcyclename, + cast(NULL as STRING) as reportingcyclestatus, + cast(NULL as STRING) as modulecode, + cast(NULL as DECIMAL(38, 10)) as moduleversionnumber, + cast(NULL as STRING) as reportingentityname, + cast(NULL as STRING) as entityattributecountry, + cast(NULL as STRING) as entitygroupentityname, + cast(NULL as STRING) as obligationmoduleremittancedate, + cast(NULL as STRING) as obligationmoduleexpected, + cast(NULL as DEcimal(38, 10)) as revalidationversionnumber, + cast(NULL as STRING) as receivedfilesystemfilename, + cast(NULL as STRING) as obligationstatusstatus, + cast(NULL as STRING) as filestatussetsubmissionstatus, + cast(NULL as STRING) as filestatussetvalidationstatus, + cast(NULL as DECIMAL(38, 10)) as numberoferrors, + cast(NULL as DECIMAL(38, 10)) as numberofwarnings, + cast(NULL as Decimal(38, 10)) as delayindays, + cast(NULL as DECIMAL(38, 10)) as failedattempts, + cast(NULL as STRING) as tablename, + cast(NULL as STRING) as tec_source_system, + cast(NULL as STRING) as tec_dataset, + cast(NULL as STRING) as tec_surrogate_key, + cast(NULL as STRING) as tec_crc, + cast(NULL as TIMESTAMP) as tec_ingestion_date, + cast(NULL as STRING) as tec_version_id, + cast(NULL as TIMESTAMP) as tec_execution_date, + cast(NULL as STRING) as tec_run_id, + cast(NULL as TIMESTAMP) as tec_business_date + from SCOPF + left join EXP_COREP_CONS on ( + scopf.obligationmodulereferencedate = EXP_COREP_CONS.reference_date + and scopf.mfi_id = EXP_COREP_CONS.riad_code + ) + left join EXP_COREP_SOLO on ( + scopf.obligationmodulereferencedate = EXP_COREP_SOLO.reference_date + and scopf.mfi_id = EXP_COREP_SOLO.riad_code + ) + left join EXP_FINREP_CONS on ( + scopf.obligationmodulereferencedate = EXP_FINREP_CONS.reference_date + and scopf.mfi_id = EXP_FINREP_CONS.riad_code + ) + left join EXP_FINREP_SOLO on ( + scopf.obligationmodulereferencedate = EXP_FINREP_SOLO.reference_date + and scopf.mfi_id = EXP_FINREP_SOLO.riad_code + ) + left join EXP_LEV_CONS on ( + scopf.obligationmodulereferencedate = EXP_LEV_CONS.reference_date + and scopf.mfi_id = EXP_LEV_CONS.riad_code + ) + left join EXP_LEV_SOLO on ( + scopf.obligationmodulereferencedate = EXP_LEV_SOLO.reference_date + and scopf.mfi_id = EXP_LEV_SOLO.riad_code + ) + left join EXP_LCR_CONS on ( + scopf.obligationmodulereferencedate = EXP_LCR_CONS.reference_date + and scopf.mfi_id = EXP_LCR_CONS.riad_code + ) + left join EXP_LCR_SOLO on ( + scopf.obligationmodulereferencedate = EXP_LCR_SOLO.reference_date + and scopf.mfi_id = EXP_LCR_SOLO.riad_code + ) + left join EXP_NSFR_CONS on ( + scopf.obligationmodulereferencedate = EXP_NSFR_CONS.reference_date + and scopf.mfi_id = EXP_NSFR_CONS.riad_code + ) + left join EXP_NSFR_SOLO on ( + scopf.obligationmodulereferencedate = EXP_NSFR_SOLO.reference_date + and scopf.mfi_id = EXP_NSFR_SOLO.riad_code + ) + left join corep_cons_c100 on ( + SCOPF.obligationmodulereferencedate = corep_cons_c100.reported_period + and SCOPF.mfi_id = corep_cons_c100.riad_code + ) + left join corep_cons_c300 on ( + SCOPF.obligationmodulereferencedate = corep_cons_c300.reported_period + and SCOPF.mfi_id = corep_cons_c300.riad_code + ) + left join corep_cons_c200 on ( + SCOPF.obligationmodulereferencedate = corep_cons_c200.reported_period + and SCOPF.mfi_id = corep_cons_c200.riad_code + ) + left join corep_solo_c100 on ( + SCOPF.obligationmodulereferencedate = corep_solo_c100.reported_period + and SCOPF.mfi_id = corep_solo_c100.riad_code + ) + left join corep_solo_c200 on ( + SCOPF.obligationmodulereferencedate = corep_solo_c200.reported_period + and SCOPF.mfi_id = corep_solo_c200.riad_code + ) + left join corep_solo_c300 on ( + SCOPF.obligationmodulereferencedate = corep_solo_c300.reported_period + and SCOPF.mfi_id = corep_solo_c300.riad_code + ) + left join finrep_cons on ( + SCOPF.obligationmodulereferencedate = finrep_cons.reported_period + and SCOPF.mfi_id = finrep_cons.riad_code + ) + left join finrep_solo on ( + SCOPF.obligationmodulereferencedate = finrep_solo.reported_period + and SCOPF.mfi_id = finrep_solo.riad_code + ) + left join lev_cons on ( + SCOPF.obligationmodulereferencedate = lev_cons.reported_period + and SCOPF.mfi_id = lev_cons.riad_code + ) + left join lev_solo on ( + SCOPF.obligationmodulereferencedate = lev_solo.reported_period + and SCOPF.mfi_id = lev_solo.riad_code + ) + left join lcr_cons on ( + SCOPF.obligationmodulereferencedate = lcr_cons.reported_period + and SCOPF.mfi_id = lcr_cons.riad_code + ) + left join lcr_solo on ( + SCOPF.obligationmodulereferencedate = lcr_solo.reported_period + and SCOPF.mfi_id = lcr_solo.riad_code + ) + left join nsfr_cons on ( + SCOPF.obligationmodulereferencedate = nsfr_cons.reported_period + and SCOPF.mfi_id = nsfr_cons.riad_code + ) + left join nsfr_solo on ( + SCOPF.obligationmodulereferencedate = nsfr_solo.reported_period + and SCOPF.mfi_id = nsfr_solo.riad_code + ) + left join liq_subgroup_data_riad on ( + SCOPF.obligationmodulereferencedate = liq_subgroup_data_riad.reported_period + and SCOPF.mfi_id = liq_subgroup_data_riad.riad_code + ) + left join max_dates on ( + SCOPF.obligationmodulereferencedate = max_dates.reported_period + and SCOPF.mfi_id = max_dates.riad_code + ) + order by 1,2" diff --git a/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml new file mode 100644 index 0000000..4d850fc --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml @@ -0,0 +1,259 @@ +# static configs +tmpdir: /tmp +inbox_prefix: INBOX/RQSD/RQSD_PROCESS +archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS +workflow_name: w_ODS_RQSD_PROCESS_DEVO +validation_schema_path: None +file_type: csv + +# task configs +tasks: + - task_name: m_ODS_RQSD_SUBA_DEVO_PARSE + ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_SUBA_DEVO + output_table: RQSD_SUBA_DEVO + output_columns: + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'obligationmodulereferencedate' + column_header: 'obligationmodulereferencedate' + - type: 'csv_header' + value: 'reportingentitycollectionuniqueid' + column_header: 'reportingentitycollectionuniqueid' + - type: 'csv_header' + value: 'receivedfileversionnumber' + column_header: 'receivedfileversionnumber' + - type: 'csv_header' + value: 'receivedfilereceiveddate' + column_header: 'receivedfilereceiveddate' + - type: 'csv_header' + value: 'revalidationdate' + column_header: 'revalidationdate' + - type: 'csv_header' + value: 'ref_date' + column_header: 'ref_date' + - type: 'csv_header' + value: 'inst_comp_key' + column_header: 'inst_comp_key' + - type: 'csv_header' + value: 'mfi_id' + column_header: 'mfi_id' + - type: 'csv_header' + value: 'legal_entity_id' + column_header: 'legal_entity_id' + - type: 'csv_header' + value: 'inst_name' + column_header: 'inst_name' + - type: 'csv_header' + value: 'currency' + column_header: 'currency' + - type: 'csv_header' + value: 'reported_by_supervisor' + column_header: 'reported_by_supervisor' + - type: 'csv_header' + value: 'confirmed_by_supervisor' + column_header: 'confirmed_by_supervisor' + - type: 'csv_header' + value: 'exp_corep_cons' + column_header: 'exp_corep_cons' + - type: 'csv_header' + value: 'exp_corep_solo' + column_header: 'exp_corep_solo' + - type: 'csv_header' + value: 'exp_finrep_cons' + column_header: 'exp_finrep_cons' + - type: 'csv_header' + value: 'exp_finrep_solo' + column_header: 'exp_finrep_solo' + - type: 'csv_header' + value: 'exp_lev_cons' + column_header: 'exp_lev_cons' + - type: 'csv_header' + value: 'exp_lev_solo' + column_header: 'exp_lev_solo' + - type: 'csv_header' + value: 'exp_lcr_cons' + column_header: 'exp_lcr_cons' + - type: 'csv_header' + value: 'exp_lcr_solo' + column_header: 'exp_lcr_solo' + - type: 'csv_header' + value: 'exp_nsfr_cons' + column_header: 'exp_nsfr_cons' + - type: 'csv_header' + value: 'exp_nsfr_solo' + column_header: 'exp_nsfr_solo' + - type: 'csv_header' + value: 'cons_cet1_amt' + column_header: 'cons_cet1_amt' + - type: 'csv_header' + value: 'cons_tier1_amt' + column_header: 'cons_tier1_amt' + - type: 'csv_header' + value: 'cons_tot_cap_amt' + column_header: 'cons_tot_cap_amt' + - type: 'csv_header' + value: 'cons_cet1_ratio' + column_header: 'cons_cet1_ratio' + - type: 'csv_header' + value: 'cons_tier1_ratio' + column_header: 'cons_tier1_ratio' + - type: 'csv_header' + value: 'cons_tot_cap_ratio' + column_header: 'cons_tot_cap_ratio' + - type: 'csv_header' + value: 'cons_risk_wght_assets' + column_header: 'cons_risk_wght_assets' + - type: 'csv_header' + value: 'solo_cet1_amt' + column_header: 'solo_cet1_amt' + - type: 'csv_header' + value: 'solo_tier1_amt' + column_header: 'solo_tier1_amt' + - type: 'csv_header' + value: 'solo_tot_cap_amt' + column_header: 'solo_tot_cap_amt' + - type: 'csv_header' + value: 'solo_cet1_ratio' + column_header: 'solo_cet1_ratio' + - type: 'csv_header' + value: 'solo_tier1_ratio' + column_header: 'solo_tier1_ratio' + - type: 'csv_header' + value: 'solo_tot_cap_ratio' + column_header: 'solo_tot_cap_ratio' + - type: 'csv_header' + value: 'solo_risk_wght_assets' + column_header: 'solo_risk_wght_assets' + - type: 'csv_header' + value: 'cons_tot_assets' + column_header: 'cons_tot_assets' + - type: 'csv_header' + value: 'solo_tot_assets' + column_header: 'solo_tot_assets' + - type: 'csv_header' + value: 'cons_lev_ratio_full' + column_header: 'cons_lev_ratio_full' + - type: 'csv_header' + value: 'cons_lev_ratio_trans' + column_header: 'cons_lev_ratio_trans' + - type: 'csv_header' + value: 'cons_lev_ratio_req' + column_header: 'cons_lev_ratio_req' + - type: 'csv_header' + value: 'cons_lev_ratio_adj' + column_header: 'cons_lev_ratio_adj' + - type: 'csv_header' + value: 'solo_lev_ratio_full' + column_header: 'solo_lev_ratio_full' + - type: 'csv_header' + value: 'solo_lev_ratio_trans' + column_header: 'solo_lev_ratio_trans' + - type: 'csv_header' + value: 'solo_lev_ratio_req' + column_header: 'solo_lev_ratio_req' + - type: 'csv_header' + value: 'solo_lev_ratio_adj' + column_header: 'solo_lev_ratio_adj' + - type: 'csv_header' + value: 'cons_lc_ratio' + column_header: 'cons_lc_ratio' + - type: 'csv_header' + value: 'solo_lc_ratio' + column_header: 'solo_lc_ratio' + - type: 'csv_header' + value: 'cons_nsfr_ratio' + column_header: 'cons_nsfr_ratio' + - type: 'csv_header' + value: 'solo_nsfr_ratio' + column_header: 'solo_nsfr_ratio' + - type: 'csv_header' + value: 'submitter_comment' + column_header: 'submitter_comment' + - type: 'csv_header' + value: 'datacollectioncode' + column_header: 'datacollectioncode' + - type: 'csv_header' + value: 'reportingcyclename' + column_header: 'reportingcyclename' + - type: 'csv_header' + value: 'reportingcyclestatus' + column_header: 'reportingcyclestatus' + - type: 'csv_header' + value: 'modulecode' + column_header: 'modulecode' + - type: 'csv_header' + value: 'moduleversionnumber' + column_header: 'moduleversionnumber' + - type: 'csv_header' + value: 'reportingentityname' + column_header: 'reportingentityname' + - type: 'csv_header' + value: 'entityattributecountry' + column_header: 'entityattributecountry' + - type: 'csv_header' + value: 'entitygroupentityname' + column_header: 'entitygroupentityname' + - type: 'csv_header' + value: 'obligationmoduleremittancedate' + column_header: 'obligationmoduleremittancedate' + - type: 'csv_header' + value: 'obligationmoduleexpected' + column_header: 'obligationmoduleexpected' + - type: 'csv_header' + value: 'revalidationversionnumber' + column_header: 'revalidationversionnumber' + - type: 'csv_header' + value: 'receivedfilesystemfilename' + column_header: 'receivedfilesystemfilename' + - type: 'csv_header' + value: 'obligationstatusstatus' + column_header: 'obligationstatusstatus' + - type: 'csv_header' + value: 'filestatussetsubmissionstatus' + column_header: 'filestatussetsubmissionstatus' + - type: 'csv_header' + value: 'filestatussetvalidationstatus' + column_header: 'filestatussetvalidationstatus' + - type: 'csv_header' + value: 'numberoferrors' + column_header: 'numberoferrors' + - type: 'csv_header' + value: 'numberofwarnings' + column_header: 'numberofwarnings' + - type: 'csv_header' + value: 'delayindays' + column_header: 'delayindays' + - type: 'csv_header' + value: 'failedattempts' + column_header: 'failedattempts' + - type: 'csv_header' + value: 'tablename' + column_header: 'tablename' + - type: 'csv_header' + value: 'tec_source_system' + column_header: 'tec_source_system' + - type: 'csv_header' + value: 'tec_dataset' + column_header: 'tec_dataset' + - type: 'csv_header' + value: 'tec_surrogate_key' + column_header: 'tec_surrogate_key' + - type: 'csv_header' + value: 'tec_crc' + column_header: 'tec_crc' + - type: 'csv_header' + value: 'tec_ingestion_date' + column_header: 'tec_ingestion_date' + - type: 'csv_header' + value: 'tec_version_id' + column_header: 'tec_version_id' + - type: 'csv_header' + value: 'tec_execution_date' + column_header: 'tec_execution_date' + - type: 'csv_header' + value: 'tec_run_id' + column_header: 'tec_run_id' + - type: 'csv_header' + value: 'tec_business_date' + column_header: 'tec_business_date' \ No newline at end of file diff --git a/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS.py b/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS.py new file mode 100644 index 0000000..3a8d9f1 --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS.py @@ -0,0 +1,524 @@ +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.operators.dummy import DummyOperator +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from datetime import datetime, timedelta +import logging + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from devo_connector import DevoConnector +from mrds.core import main as mrds_main +from mrds.utils.security_utils import get_verified_run_id, verify_run_id + +TASK_CONFIGS = { + "m_ODS_RQSD_OBSERVATIONS": { + "flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml", + "env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml", + "source_filename": "RQSD_OBSERVATIONS.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml" + }, + "m_ODS_RQSD_FX": { + "flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml", + "env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml", + "source_filename": "RQSD_FX.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml" + }, + "m_ODS_RQSD_SUBA_DEVO": { + "flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml", + "env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml", + "source_filename": "RQSD_SUBA_DEVO.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml" + } +} + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +dag_id = os.path.splitext(os.path.basename(__file__))[0] + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id +} + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='Run devo RQSD data ingestion workflow with conditional MRDS processing - Multi-task', + #schedule_interval='*/10 * * * *', # every 10 minutes + schedule_interval=None, + catchup=False, + max_active_runs=1, + tags=["Devo", "RQSD", "MRDS", "Connector", "Multi-Task"] +) as dag: + + + def init_workflow_task(**context): + try: + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env_vars = { + 'MRDS_ENV': os.getenv("MRDS_ENV"), + 'MRDS_LOADER_DB_USER': os.getenv("MRDS_LOADER_DB_USER"), + 'MRDS_LOADER_DB_PASS': '***MASKED***' if os.getenv("MRDS_LOADER_DB_PASS") else None, + 'MRDS_LOADER_DB_TNS': os.getenv("MRDS_LOADER_DB_TNS"), + 'BUCKET_NAMESPACE': os.getenv("BUCKET_NAMESPACE"), + } + + for key, value in env_vars.items(): + logging.info(f"{key}: {value}") + + run_id_value = get_verified_run_id(context) + + logging.info(f"Task ID: {context.get('task_instance_key_str', 'N/A')}") + logging.info(f"Run ID: {run_id_value}") + logging.info(f"Execution Date: {context.get('execution_date', 'N/A')}") + + dag_obj = context.get('dag') + dag_id_str = dag_obj.dag_id if dag_obj else 'N/A' + logging.info(f"DAG ID: {dag_id_str}") + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing_vars = [] + if not username: + missing_vars.append("MRDS_LOADER_DB_USER") + if not password: + missing_vars.append("MRDS_LOADER_DB_PASS") + if not tnsalias: + missing_vars.append("MRDS_LOADER_DB_TNS") + + error_msg = f"Missing required environment variables: {', '.join(missing_vars)}" + logging.error(error_msg) + raise ValueError(error_msg) + + logging.info(f"Initializing {workflow_name} workflow for env '{env}'") + + workflow_run_id = run_id_value + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, workflow_run_id) + logging.info(f"Initialized workflow with history key: {a_workflow_history_key}") + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + ti = context['ti'] + ti.xcom_push(key='workflow_history_key', value=a_workflow_history_key) + ti.xcom_push(key='workflow_context', value=workflow_context) + ti.xcom_push(key='env', value=env) + + logging.info("Workflow initialization completed successfully") + + except Exception as e: + logging.error(f"Error initializing workflow: {e}", exc_info=True) + raise + + def run_devo_connector(**context): + ti = context['ti'] + task_id = context['task'].task_id + + try: + if task_id.startswith('devo_'): + task_name = task_id.replace('devo_', '') + else: + task_name = task_id + + task_config = TASK_CONFIGS.get(task_name) + if not task_config: + error_msg = f"No configuration found for task: {task_name}" + logging.error(error_msg) + ti.xcom_push(key='row_count', value=0) + ti.xcom_push(key='devo_success', value=False) + ti.xcom_push(key='should_run_mrds', value=False) + ti.xcom_push(key='error_message', value=error_msg) + raise ValueError(error_msg) + + flow_config_path = task_config["flow_config_path"] + env_config_path = task_config["env_config_path"] + + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow') + env = ti.xcom_pull(key='env', task_ids='init_workflow') + + if not workflow_context: + error_msg = "No workflow_context from init task" + logging.error(error_msg) + ti.xcom_push(key='row_count', value=0) + ti.xcom_push(key='devo_success', value=False) + ti.xcom_push(key='should_run_mrds', value=False) + ti.xcom_push(key='error_message', value=error_msg) + raise ValueError(error_msg) + + logging.info(f"Starting Devo connector for env '{env}' - {task_name}") + + devo_connector = DevoConnector( + flow_config_path=flow_config_path, + env_config_path=env_config_path, + env=env, + logger=logging.getLogger(f"devo_connector_{env}_{task_name}") + ) + + row_count = devo_connector.run(workflow_context) + + logging.info(f"Devo connector completed successfully for {task_name}. Processed {row_count} rows.") + + ti.xcom_push(key='row_count', value=row_count) + ti.xcom_push(key='devo_success', value=True) + ti.xcom_push(key='should_run_mrds', value=row_count > 0) + ti.xcom_push(key='error_message', value=None) + + return row_count + + except Exception as e: + error_msg = f"Error running Devo connector: {str(e)}" + logging.error(error_msg, exc_info=True) + ti.xcom_push(key='row_count', value=0) + ti.xcom_push(key='devo_success', value=False) + ti.xcom_push(key='should_run_mrds', value=False) + ti.xcom_push(key='error_message', value=error_msg) + raise + + def check_should_run_mrds(**context): + ti = context['ti'] + dag_run = context['dag_run'] + task_id = context['task'].task_id + + if task_id.startswith('check_'): + task_name = task_id.replace('check_', '') + else: + task_name = task_id + + devo_task_id = f'devo_{task_name}' + + devo_task_instance = dag_run.get_task_instance(devo_task_id) + logging.info(f"Devo task {devo_task_id} state: {devo_task_instance.state}") + + if devo_task_instance.state == 'failed': + error_msg = ti.xcom_pull(key='error_message', task_ids=devo_task_id) + logging.info(f"Devo connector failed for {task_name} - skipping MRDS task. Error: {error_msg}") + raise AirflowSkipException(f"Devo connector failed for {task_name}") + + should_run_mrds = ti.xcom_pull(key='should_run_mrds', task_ids=devo_task_id) + row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id) + + if task_name == "m_ODS_RQSD_OBSERVATIONS" and (not should_run_mrds or row_count == 0): + logging.info(f"OBSERVATIONS task has no data (row_count: {row_count}) - marking to skip all subsequent tasks") + ti.xcom_push(key='skip_all_tasks', value=True) + raise AirflowSkipException(f"No OBSERVATIONS data found (row_count: {row_count}) - skipping all subsequent processing") + + if not should_run_mrds or row_count == 0: + logging.info(f"Skipping MRDS task for {task_name} - row count: {row_count}") + raise AirflowSkipException(f"No data to process for {task_name} (row_count: {row_count})") + + logging.info(f"MRDS task should run for {task_name} - row count: {row_count}") + return True + + def check_should_run_parallel_tasks(**context): + ti = context['ti'] + task_id = context['task'].task_id + + if task_id.startswith('check_'): + task_name = task_id.replace('check_', '') + else: + task_name = task_id + + skip_all_tasks = ti.xcom_pull(key='skip_all_tasks', task_ids='check_m_ODS_RQSD_OBSERVATIONS') + + if skip_all_tasks: + logging.info(f"Skipping {task_name} - OBSERVATIONS task had no data") + raise AirflowSkipException(f"Skipping {task_name} - OBSERVATIONS task had no data") + + return check_should_run_mrds(**context) + + def run_mrds_task(**context): + ti = context['ti'] + task_id = context['task'].task_id + + try: + if task_id.endswith('_PARSE'): + task_name = task_id.replace('_PARSE', '') + else: + task_name = task_id + + devo_task_id = f'devo_{task_name}' + + task_config = TASK_CONFIGS.get(task_name) + if not task_config: + raise ValueError(f"No configuration found for task: {task_name}") + + source_filename = task_config["source_filename"] + config_file = task_config["config_file"] + + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow') + row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id) + + if not workflow_context: + raise ValueError("No workflow_context from init task") + + logging.info(f"Starting MRDS task for {task_name} with workflow context: {workflow_context}") + logging.info(f"Processing {row_count} rows from Devo connector") + + mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=False) + + logging.info(f"MRDS task completed successfully for {task_name}") + return "SUCCESS" + + except Exception as e: + logging.error(f"Error running MRDS task: {e}", exc_info=True) + raise + + def check_success_for_mopdb(**context): + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + for task_name in TASK_CONFIGS.keys(): + devo_task_id = f'devo_{task_name}' + mrds_task_id = f'{task_name}_PARSE' + + devo_task = dag_run.get_task_instance(devo_task_id) + mrds_task = dag_run.get_task_instance(mrds_task_id) + + if devo_task.state == 'failed': + has_failures = True + failure_reasons.append(f"{task_name}: Devo connector failed") + + if mrds_task.state == 'failed': + has_failures = True + failure_reasons.append(f"{task_name}: MRDS task failed") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + all_skipped = all( + dag_run.get_task_instance(f'{task_name}_PARSE').state == 'skipped' + for task_name in TASK_CONFIGS.keys() + ) + + if all_skipped: + error_msg = "All MRDS tasks were skipped (no data to process) - skipping MOPDB trigger" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + def end_log_table_task(**context): + try: + logging.info("End log table task - always runs at the end") + return "SUCCESS" + except Exception as e: + logging.error(f"Error in end log table task: {e}", exc_info=True) + raise + + def finalise_workflow_task(**context): + a_workflow_history_key = None + try: + ti = context['ti'] + dag_run = context['dag_run'] + + a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='init_workflow') + + if a_workflow_history_key is None: + raise ValueError("No workflow history key found in XCom; cannot finalise workflow") + + workflow_success = True + failure_reasons = [] + + for task_name in TASK_CONFIGS.keys(): + devo_task_id = f'devo_{task_name}' + mrds_task_id = f'{task_name}_PARSE' + + devo_task = dag_run.get_task_instance(devo_task_id) + mrds_task = dag_run.get_task_instance(mrds_task_id) + + if devo_task.state == 'failed': + workflow_success = False + failure_reasons.append(f"{task_name}: Devo connector failed") + + if mrds_task.state == 'failed': + workflow_success = False + failure_reasons.append(f"{task_name}: MRDS task failed") + elif mrds_task.state == 'skipped': + row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id) + devo_success = ti.xcom_pull(key='devo_success', task_ids=devo_task_id) + + if devo_success and row_count == 0: + logging.info(f"{task_name} - MRDS task was skipped due to no data - this is normal") + elif not devo_success: + workflow_success = False + failure_reasons.append(f"{task_name}: Devo connector failed, MRDS skipped") + + trigger_mopdb_task = dag_run.get_task_instance('trigger_mopdb_dag') + if trigger_mopdb_task.state == 'failed': + workflow_success = False + failure_reasons.append("MOPDB trigger failed") + + end_log_task = dag_run.get_task_instance('end_log_table') + if end_log_task.state == 'failed': + workflow_success = False + failure_reasons.append("End log table failed") + + if workflow_success: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS") + else: + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error(f"Finalised workflow with history key {a_workflow_history_key} as FAILED") + logging.error(f"Failure reasons: {', '.join(failure_reasons)}") + raise AirflowFailException(f"Workflow failed: {', '.join(failure_reasons)}") + + except AirflowFailException: + raise + except Exception as e: + logging.error(f"Error finalizing workflow: {e}", exc_info=True) + try: + if a_workflow_history_key: + mrds_finalise_workflow(a_workflow_history_key, "N") + except: + pass + raise AirflowFailException(f"Workflow finalization failed: {e}") + + init_workflow = PythonOperator( + task_id='init_workflow', + python_callable=init_workflow_task, + provide_context=True, + retries=0, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + trigger_mopdb_dag = TriggerDagRunOperator( + task_id='trigger_mopdb_dag', + trigger_dag_id='w_MOPDB_RQSD_PROCESS', + wait_for_completion=False, + trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, + retries=0, + ) + + end_log_table = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + task_names = list(TASK_CONFIGS.keys()) + first_task_name = task_names[0] + parallel_task_names = task_names[1:] + + first_devo_task = PythonOperator( + task_id=f'devo_{first_task_name}', + python_callable=run_devo_connector, + provide_context=True, + retries=0, + ) + + first_check_task = PythonOperator( + task_id=f'check_{first_task_name}', + python_callable=check_should_run_mrds, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + first_mrds_task = PythonOperator( + task_id=f'{first_task_name}_PARSE', + python_callable=run_mrds_task, + provide_context=True, + retries=0, + ) + + parallel_tasks = [] + for task_name in parallel_task_names: + devo_task = PythonOperator( + task_id=f'devo_{task_name}', + python_callable=run_devo_connector, + provide_context=True, + retries=0, + ) + + check_task = PythonOperator( + task_id=f'check_{task_name}', + python_callable=check_should_run_parallel_tasks, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + retries=0, + ) + + mrds_task = PythonOperator( + task_id=f'{task_name}_PARSE', + python_callable=run_mrds_task, + provide_context=True, + retries=0, + ) + + first_mrds_task >> devo_task >> check_task >> mrds_task + parallel_tasks.extend([devo_task, check_task, mrds_task]) + + init_workflow >> first_devo_task >> first_check_task >> first_mrds_task + + all_final_tasks = [first_mrds_task] + for task_name in parallel_task_names: + mrds_task_id = f'{task_name}_PARSE' + for task in parallel_tasks: + if task.task_id == mrds_task_id: + all_final_tasks.append(task) + break + + for task in all_final_tasks: + task >> check_mopdb + + check_mopdb >> end_log_table >> trigger_mopdb_dag >> finalize_workflow diff --git a/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS_MANUAL.py b/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS_MANUAL.py new file mode 100644 index 0000000..7bd35bc --- /dev/null +++ b/airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS_MANUAL.py @@ -0,0 +1,424 @@ +""" +DAG ID set to w_ODS_RQSD_PROCESS_MANUAL +Removed OBSERVATIONS from TASK_CONFIGS (only FX and SUBA_DEVO remain) +Updated run_devo_connector to always set should_run_mrds=True (no row count check) +Simplified check_should_run_mrds to always proceed if devo task succeeded (manual mode) +Removed special OBSERVATIONS handling logic +Trigger DAG ID changed to w_MOPDB_RQSD_PROCESS_MANUAL +Added "MANUAL" tag +Both FX and SUBA_DEVO tasks run in parallel from init +Removed skipped task handling for no data scenarios in finalize +""" + +import sys +import os +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.operators.dummy import DummyOperator +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from datetime import datetime, timedelta +import logging + +try: + from airflow.exceptions import AirflowFailException, AirflowSkipException +except Exception: + from airflow.exceptions import AirflowException as AirflowFailException + from airflow.exceptions import AirflowSkipException + +sys.path.append('/opt/airflow/python/connectors/devo') +sys.path.append('/opt/airflow/python/mrds_common') +sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd') + +from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow +from devo_connector import DevoConnector +from mrds.core import main as mrds_main +from mrds.utils.security_utils import get_verified_run_id, verify_run_id + +TASK_CONFIGS = { + "m_ODS_RQSD_FX": { + "flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml", + "env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml", + "source_filename": "RQSD_FX.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml" + }, + "m_ODS_RQSD_SUBA_DEVO": { + "flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml", + "env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml", + "source_filename": "RQSD_SUBA_DEVO.csv", + "config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml" + } +} + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +dag_id = "w_ODS_RQSD_PROCESS_MANUAL" + +WORKFLOW_CONFIG = { + "database_name": "ODS", + "workflow_name": dag_id +} + +with DAG( + dag_id=dag_id, + default_args=default_args, + description='Manual trigger for RQSD data processing', + schedule_interval=None, + catchup=False, + tags=["Devo", "RQSD", "MRDS", "Connector", "Multi-Task", "MANUAL"] +) as dag: + + def init_workflow_task(**context): + try: + database_name = WORKFLOW_CONFIG["database_name"] + workflow_name = WORKFLOW_CONFIG["workflow_name"] + + env_vars = { + 'MRDS_ENV': os.getenv("MRDS_ENV"), + 'MRDS_LOADER_DB_USER': os.getenv("MRDS_LOADER_DB_USER"), + 'MRDS_LOADER_DB_PASS': '***MASKED***' if os.getenv("MRDS_LOADER_DB_PASS") else None, + 'MRDS_LOADER_DB_TNS': os.getenv("MRDS_LOADER_DB_TNS"), + 'BUCKET_NAMESPACE': os.getenv("BUCKET_NAMESPACE"), + } + + for key, value in env_vars.items(): + logging.info(f"{key}: {value}") + + run_id_value = get_verified_run_id(context) + + logging.info(f"Task ID: {context.get('task_instance_key_str', 'N/A')}") + logging.info(f"Run ID: {run_id_value}") + logging.info(f"Execution Date: {context.get('execution_date', 'N/A')}") + + dag_obj = context.get('dag') + dag_id_str = dag_obj.dag_id if dag_obj else 'N/A' + logging.info(f"DAG ID: {dag_id_str}") + + env = os.getenv("MRDS_ENV", "dev") + username = os.getenv("MRDS_LOADER_DB_USER") + password = os.getenv("MRDS_LOADER_DB_PASS") + tnsalias = os.getenv("MRDS_LOADER_DB_TNS") + + if not all([username, password, tnsalias]): + missing_vars = [] + if not username: + missing_vars.append("MRDS_LOADER_DB_USER") + if not password: + missing_vars.append("MRDS_LOADER_DB_PASS") + if not tnsalias: + missing_vars.append("MRDS_LOADER_DB_TNS") + + error_msg = f"Missing required environment variables: {', '.join(missing_vars)}" + logging.error(error_msg) + raise ValueError(error_msg) + + logging.info(f"Initializing {workflow_name} workflow for env '{env}'") + + workflow_run_id = run_id_value + + a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, workflow_run_id) + logging.info(f"Initialized workflow with history key: {a_workflow_history_key}") + + workflow_context = { + "run_id": workflow_run_id, + "a_workflow_history_key": a_workflow_history_key + } + + ti = context['ti'] + ti.xcom_push(key='workflow_history_key', value=a_workflow_history_key) + ti.xcom_push(key='workflow_context', value=workflow_context) + ti.xcom_push(key='env', value=env) + + logging.info("Workflow initialization completed successfully") + + except Exception as e: + logging.error(f"Error initializing workflow: {e}", exc_info=True) + raise + + def run_devo_connector(**context): + try: + ti = context['ti'] + task_id = context['task'].task_id + + if task_id.startswith('devo_'): + task_name = task_id.replace('devo_', '') + else: + task_name = task_id + + task_config = TASK_CONFIGS.get(task_name) + if not task_config: + raise ValueError(f"No configuration found for task: {task_name}") + + flow_config_path = task_config["flow_config_path"] + env_config_path = task_config["env_config_path"] + + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow') + env = ti.xcom_pull(key='env', task_ids='init_workflow') + + if not workflow_context: + raise ValueError("No workflow_context from init task") + + logging.info(f"Starting Devo connector for env '{env}' - {task_name}") + + devo_connector = DevoConnector( + flow_config_path=flow_config_path, + env_config_path=env_config_path, + env=env, + logger=logging.getLogger(f"devo_connector_{env}_{task_name}") + ) + + row_count = devo_connector.run(workflow_context) + + logging.info(f"Devo connector completed successfully for {task_name}. Processed {row_count} rows.") + + ti.xcom_push(key='row_count', value=row_count) + ti.xcom_push(key='devo_success', value=True) + ti.xcom_push(key='should_run_mrds', value=True) + + return row_count + + except Exception as e: + logging.error(f"Error running Devo connector: {e}", exc_info=True) + ti = context['ti'] + ti.xcom_push(key='row_count', value=0) + ti.xcom_push(key='devo_success', value=False) + ti.xcom_push(key='should_run_mrds', value=False) + raise + + def check_should_run_mrds(**context): + ti = context['ti'] + dag_run = context['dag_run'] + task_id = context['task'].task_id + + if task_id.startswith('check_'): + task_name = task_id.replace('check_', '') + else: + task_name = task_id + + devo_task_id = f'devo_{task_name}' + + devo_task_instance = dag_run.get_task_instance(devo_task_id) + logging.info(f"Devo task state: {devo_task_instance.state}") + + if devo_task_instance.state == 'failed': + logging.info(f"Devo connector failed for {task_name} - skipping MRDS task") + raise AirflowSkipException(f"Devo connector failed for {task_name}") + + logging.info(f"MRDS task will run for {task_name} - manual trigger mode") + return True + + def run_mrds_task(**context): + try: + ti = context['ti'] + task_id = context['task'].task_id + + if task_id.endswith('_PARSE'): + task_name = task_id.replace('_PARSE', '') + else: + task_name = task_id + + devo_task_id = f'devo_{task_name}' + + task_config = TASK_CONFIGS.get(task_name) + if not task_config: + raise ValueError(f"No configuration found for task: {task_name}") + + source_filename = task_config["source_filename"] + config_file = task_config["config_file"] + + workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow') + row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id) + + if not workflow_context: + raise ValueError("No workflow_context from init task") + + logging.info(f"Starting MRDS task for {task_name} with workflow context: {workflow_context}") + logging.info(f"Processing {row_count} rows from Devo connector") + + mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=False) + + logging.info(f"MRDS task completed successfully for {task_name}") + return "SUCCESS" + + except Exception as e: + logging.error(f"Error running MRDS task: {e}", exc_info=True) + raise + + def check_success_for_mopdb(**context): + try: + ti = context['ti'] + dag_run = context['dag_run'] + + has_failures = False + failure_reasons = [] + + for task_name in TASK_CONFIGS.keys(): + devo_task_id = f'devo_{task_name}' + mrds_task_id = f'{task_name}_PARSE' + + devo_task = dag_run.get_task_instance(devo_task_id) + mrds_task = dag_run.get_task_instance(mrds_task_id) + + if devo_task.state == 'failed': + has_failures = True + failure_reasons.append(f"{task_name}: Devo connector failed") + + if mrds_task.state == 'failed': + has_failures = True + failure_reasons.append(f"{task_name}: MRDS task failed") + + if has_failures: + error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}" + logging.info(error_msg) + raise AirflowSkipException(error_msg) + + logging.info("All tasks completed successfully - proceeding to trigger MOPDB") + return "SUCCESS" + + except AirflowSkipException: + raise + except Exception as e: + logging.error(f"Error checking success for MOPDB: {e}", exc_info=True) + raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}") + + def end_log_table_task(**context): + try: + logging.info("End log table task - always runs at the end") + return "SUCCESS" + except Exception as e: + logging.error(f"Error in end log table task: {e}", exc_info=True) + raise + + def finalise_workflow_task(**context): + try: + ti = context['ti'] + dag_run = context['dag_run'] + + a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='init_workflow') + + if a_workflow_history_key is None: + raise ValueError("No workflow history key found in XCom; cannot finalise workflow") + + workflow_success = True + failure_reasons = [] + + for task_name in TASK_CONFIGS.keys(): + devo_task_id = f'devo_{task_name}' + mrds_task_id = f'{task_name}_PARSE' + + devo_task = dag_run.get_task_instance(devo_task_id) + mrds_task = dag_run.get_task_instance(mrds_task_id) + + if devo_task.state == 'failed': + workflow_success = False + failure_reasons.append(f"{task_name}: Devo connector failed") + + if mrds_task.state == 'failed': + workflow_success = False + failure_reasons.append(f"{task_name}: MRDS task failed") + + trigger_mopdb_task = dag_run.get_task_instance('trigger_mopdb_dag') + if trigger_mopdb_task.state == 'failed': + workflow_success = False + failure_reasons.append("MOPDB trigger failed") + + end_log_task = dag_run.get_task_instance('end_log_table') + if end_log_task.state == 'failed': + workflow_success = False + failure_reasons.append("End log table failed") + + if workflow_success: + mrds_finalise_workflow(a_workflow_history_key, "Y") + logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS") + else: + mrds_finalise_workflow(a_workflow_history_key, "N") + logging.error(f"Finalised workflow with history key {a_workflow_history_key} as FAILED") + logging.error(f"Failure reasons: {', '.join(failure_reasons)}") + raise AirflowFailException(f"Workflow failed: {', '.join(failure_reasons)}") + + except AirflowFailException: + raise + except Exception as e: + logging.error(f"Error finalizing workflow: {e}", exc_info=True) + try: + if 'a_workflow_history_key' in locals() and a_workflow_history_key: + mrds_finalise_workflow(a_workflow_history_key, "N") + except: + pass + raise AirflowFailException(f"Workflow finalization failed: {e}") + + init_workflow = PythonOperator( + task_id='init_workflow', + python_callable=init_workflow_task, + provide_context=True, + ) + + check_mopdb = PythonOperator( + task_id='check_success_for_mopdb', + python_callable=check_success_for_mopdb, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + trigger_mopdb_dag = TriggerDagRunOperator( + task_id='trigger_mopdb_dag', + trigger_dag_id='w_MOPDB_RQSD_PROCESS_MANUAL', + wait_for_completion=False, + trigger_rule=TriggerRule.NONE_FAILED, + ) + + end_log_table = PythonOperator( + task_id='end_log_table', + python_callable=end_log_table_task, + trigger_rule=TriggerRule.ALL_DONE, + ) + + finalize_workflow = PythonOperator( + task_id='finalize_workflow', + python_callable=finalise_workflow_task, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + all_tasks = [] + for task_name in TASK_CONFIGS.keys(): + devo_task = PythonOperator( + task_id=f'devo_{task_name}', + python_callable=run_devo_connector, + provide_context=True, + ) + + check_task = PythonOperator( + task_id=f'check_{task_name}', + python_callable=check_should_run_mrds, + provide_context=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + mrds_task = PythonOperator( + task_id=f'{task_name}_PARSE', + python_callable=run_mrds_task, + provide_context=True, + ) + + devo_task >> check_task >> mrds_task + all_tasks.extend([devo_task, check_task, mrds_task]) + + devo_tasks = [task for task in all_tasks if task.task_id.startswith('devo_')] + mrds_tasks = [task for task in all_tasks if task.task_id.endswith('_PARSE')] + + init_workflow >> devo_tasks + + for mrds_task in mrds_tasks: + mrds_task >> check_mopdb + + check_mopdb >> trigger_mopdb_dag >> end_log_table >> finalize_workflow diff --git a/airflow/ods/tms/README.md b/airflow/ods/tms/README.md new file mode 100644 index 0000000..80575d4 --- /dev/null +++ b/airflow/ods/tms/README.md @@ -0,0 +1,236 @@ +# DAG Factory for TMS Data Ingestion + +## Overview +This repository contains a **DAG factory** that generates multiple Apache Airflow DAGs to ingest data from a **Treasury Management System (TMS)** into the data warehouse. + +The factory dynamically creates one DAG per TMS dataset, using **YAML-based layouts** to define parameters and metadata. Each DAG: +- Calls the **TMSDB CLI connector** (`TMSDB.py`) to retrieve data in CSV format. +- Loads the data into object storage. +- Creates or refreshes **Oracle external tables** if needed. +- Registers workflow metadata in MRDS tables. +- Processes the landed file for downstream use. + +--- + +## Components + +### 1. DAG Factory (`create_dag`) +- **Purpose**: Auto-generates DAGs for each TMS dataset. +- **Inputs**: + - `TMS-layouts/.yml`: defines report parameters, visible/hidden flags, virtual/replacement parameters. + - `config/TMS.yml`: holds system-wide TMS connection info and storage prefixes. +- **Outputs**: + - Airflow DAG objects named like `w_ODS_TMS_`. + +### 2. TMSDB Connector (`TMSDB.py`) +- **Purpose**: CLI tool that interacts with the TMS service. +- **Commands**: + - `retrieve`: fetch rows from TMS into CSV, spool to storage, return exit codes (`0 = data`, `1 = no data`). + - `create-oracle-table`: generate an Oracle DDL file based on dataset metadata. + - `create-model`: generate dbt models for dataset integration. +- **Behavior**: + - Adds synthetic columns (`A_KEY`, `A_WORKFLOW_HISTORY_KEY`). + - Supports additional columns via `-c`. + - Uploads to object storage if `bucket:path/file.csv` is given. + +### 3. Manage Files (`mf`) +Utilities for file-level operations: +- `execute_query(sql)` +- `add_source_file_config(...)` +- `process_source_file(prefix, file)` +- `create_external_table(table, source, prefix)` +- `add_column_date_format(...)` + +### 4. Manage Runs (`mr`) +Utilities for workflow tracking: +- `init_workflow(db, wf_name, run_id)` +- `set_workflow_property(key, db, name, value)` +- `finalise_workflow(key, status)` +- `select_ods_tab(table, expr, cond)` + +--- + +## How a DAG Works + +### DAG Structure +Each DAG has a single task: +- `retrieve_report`: a `PythonOperator` that orchestrates all steps internally. + +### Task Flow +1. **Read YAML configs** + - Parameters split into visible (exposed in Airflow UI) and hidden. + - System config (URL, creds, bucket/prefix) loaded from `config/TMS.yml`. + +2. **Parameter processing** + - Cartesian product of parameter lists. + - Support for: + - `column(...)` aligned columns. + - `select(...)` SQL evaluation (restricted tables only). + - Virtual parameters (dropped later). + - Replace-parameter logic. + +3. **Workflow init** + - `mr.init_workflow` creates a workflow key. + +4. **Data retrieval** + - Build a `TMSDB.py retrieve` command. + - Run via subprocess. + - Handle return codes: + - `0`: data returned. + - `1`: no data → workflow finalized as success. + - `!=0`: error → workflow finalized as failure. + +5. **First-run bootstrap** + - If no config exists for the dataset: + - Run `TMSDB.py create-oracle-table` to generate SQL. + - Execute SQL via `mf.execute`. + - Add date formats and external table with `mf.create_external_table`. + - Register config with `mf.add_source_file_config`. + +6. **File processing** + - `mf.process_source_file(prefix, filename)` ingests the CSV. + +7. **Workflow finalization** + - `mr.finalise_workflow(wf_key, 'Y' | 'N')`. + +--- + +## Example DAG +Example: `w_ODS_TMS_TRANSACTION` + +```python +with DAG( + dag_id="w_ODS_TMS_TRANSACTION", + default_args=default_args, + schedule_interval=None, + start_date=datetime(2025, 1, 1), + catchup=False, + params={"date_from": "2025-01-01", "date_to": "2025-01-31"}, +) as dag: + + retrieve_report = PythonOperator( + task_id="retrieve_report", + python_callable=execute_report, + execution_timeout=timedelta(minutes=30), + ) +``` + +--- + +## Repository Layout +``` + tms/ + ├─ generate_tm_ods_dags.py # DAG generator script (calls create_dag many times) + ├─ TMS-layouts/ + │ ├─ w_ODS_TMS_TRANSACTION.yml + │ └─ ... + ├─ config/ + │ └─ TMS.yml + └─ TMS-tables/ # Create table SQL scripts +``` + +--- + +## Security Considerations +- **`eval()` is dangerous.** + Only `select(...)` is allowed, and it’s whitelisted to safe tables. +- **No raw shell commands.** + Use `subprocess.run([...], shell=False)` for safety. +- **Secrets in config.** + TMS username/password are stored in `TMS.yml` → best stored in Airflow Connections/Secrets Manager. +- **Exit codes matter.** + Workflow correctness relies on `TMSDB.py` returning the right codes (`0`, `1`, other). + +--- + +## Extending the Factory + +### Add a new dataset +1. Create a YAML layout in `TMS-layouts/`, e.g.: + + ```yaml + parameters: + date_from: + value: "2025-01-01" + date_to: + value: "2025-01-31" + ``` + +2. Add a line in `dag_factory.py`: + + ```python + create_dag("w_ODS_TMS_NEWENTITY") + ``` + +3. Deploy the DAG file to Airflow. + +### Run a DAG manually +In the Airflow UI: +1. Find DAG `w_ODS_TMS_`. +2. Trigger DAG → optionally override visible parameters. +3. Monitor logs for `retrieve_report`. + +--- + +## Diagram +**DAG Factory Flow** +```mermaid +flowchart LR + subgraph DAGFactory["Dag Factory"] + direction TB + B["Load TMS config (TMS.yml)"] --> C["Load dataset layout (YAML)"] + C --> D["Extract visible & hidden parameters"] + D --> E["Define Airflow DAG with retrieve_report task"] + E --> F["Register DAG globally"] + F --> G["Repeat for each dataset name"] + G --> H["All DAGs available in Airflow"] + end + A["Airflow parses dag_factory.py"] --> DAGFactory + +``` + +**Sample DAG Execution Flow** +```mermaid +flowchart LR + subgraph ExampleDAG + direction TB + B[Read YAML configs] --> C[Build parameter combinations] + C --> D["Evaluate select(...) and replace virtual params"] + D --> E["Init workflow (mr.init_workflow)"] + E --> F["Run TMSDB.py retrieve (subprocess)"] + + %% Branches on return codes + F --> |rc=1: No data| G[Finalise workflow success] + F --> |rc=0: Data returned| H[Check if source file config exists] + F --> |rc!=0: Error| M[Finalise workflow failure] + + %% Config missing branch + H --> |"Config missing (first run)"| I[Run TMSDB.py create-oracle-table → Generate DDL] + I --> J[Execute DDL via mf.execute → Create Oracle external table] + J --> K["Register file source config (mf.add_source_file_config)"] + K --> L["Process landed file (mf.process_source_file)"] + L --> N[Finalise workflow success] + + %% Config exists branch + H --> |Config exists| P["Process landed file (mf.process_source_file)"] + P --> N[Finalise workflow success] + end +``` +--- + +## Dependencies +- **Airflow 2.x** +- **Python 3.9+** +- **mrds** package (providing `utils.manage_files` and `utils.manage_runs`) +- **Oracle client / Impala client** (for table creation & querying) +- **Object storage client** (for uploading CSVs) + +--- + +## Summary +The DAG factory is a scalable way to create **dozens of ingestion DAGs** for TMS datasets with minimal boilerplate. It leverages: +- **YAML configs** for parameters, +- **TMSDB CLI** for data retrieval and DDL generation, +- **MRDS utilities** for workflow tracking and file handling. + +It standardizes ingestion while keeping each dataset’s DAG lightweight and uniform. diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr new file mode 100644 index 0000000..40d2170 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr @@ -0,0 +1,1641 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=acm-currency-flow +Name=Currency Flow Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Currency Flow Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=acmvoucher_voucher_number,acmentry_id,acmledger_id,acmvoucher_id,acmledger_id_user,acmvoucher_booking_date,acmvoucher_posting_date,acmperiod_id_user,acmperiodordernumber_number,umpathnode_id,umpathnode_upath,acmentry_event_date,acmentryorigin_id_user,acmentryorigingroup_id_user,acmentry_figure_id,accountingtreatment_name,acmbookkeepingtype_id_user,acmentry_description,acmeventtype_id_user,acmentry_fx_rate,acmentrytype_id_user,acmmappingrule_id,acmmappingrule_id_user,acmentry_currency_id,acmentry_booking_currency_id,transactionkind_name,transactiontype_name,acmentry_t_currency_id,acmentry_t_currency_2_id,acmentry_counterparty_id,acmentry_counterparty_group_id,acmentry_portfolio_id,acmentry_package_type_id,acmentry_issuer_id,acmentry_issuer_group_id,acmentry_bank_id,acmcptyrelation_id_user,acmentry_event_id,acmentry_reverse_event_id,acmentry_number,acmentry_source_reference,acmentry_package_map_id,acmtransactionsign_name,acmentry_t_opening_date,acmentry_t_value_date,acmentry_t_maturity_date,acmentry_c_payment_date,acmentry_nominal_amount,acmentry_book_value,acmentry_deal_rate,acmentry_deal_price,acmentry_date_basis,acmentry_instrument_id,acmentry_leg_id,acmentry_leg_group,acmentry_load_id,acmentry_account_holder_id,acmentry_bank_account,acmentry_bank_account_ccy_id,acmentry_counterparty_code,acmentry_classification_id,acmcmmstate_id_user,acmentry_cmm_reconciliation_id,acmentry_cmm_group_id,acmentryoriginentity_id_user,acmentry_cmm_payment_method_id,acmentry_one_time_cpty_id,acmcmmviewtype_id_user,acmplcategory_id_user,acmcurrencyentrystate_id_user,acmcurrencyposition_id_user,acmgroupingrule_id,acmgroupingrule_id_user,acmentry_ccy_group_id,acmentry_ccy_amount,acmentry_ccy_booking_amount,acmentry_ccy_result,acmentry_param_0,acmentry_param_1,acmentry_param_2,acmentry_param_3,acmentry_param_4,acmentry_param_5,acmentry_param_6,acmentry_param_7,acmentry_param_8,acmentry_param_9,acmentry_param_10,acmentry_param_11,acmentry_param_12,acmentry_param_13,acmentry_param_14,acmentry_param_15,acmentry_param_16,acmentry_param_17,acmentry_param_18,acmentry_param_19 +Selected=accountingtreatment_name,acmbookkeepingtype_id_user,acmcmmstate_id_user,acmcmmviewtype_id_user,acmcptyrelation_id_user,acmcurrencyentrystate_id_user,acmcurrencyposition_id_user,acmentry_account_holder_id,acmentry_bank_account,acmentry_bank_account_ccy_id,acmentry_bank_id,acmentry_book_value,acmentry_booking_currency_id,acmentry_c_payment_date,acmentry_ccy_amount,acmentry_ccy_booking_amount,acmentry_ccy_group_id,acmentry_ccy_result,acmentry_classification_id,acmentry_cmm_group_id,acmentry_cmm_payment_method_id,acmentry_cmm_reconciliation_id,acmentry_counterparty_code,acmentry_counterparty_group_id,acmentry_counterparty_id,acmentry_currency_id,acmentry_date_basis,acmentry_deal_price,acmentry_deal_rate,acmentry_description,acmentry_event_date,acmentry_event_id,acmentry_figure_id,acmentry_fx_rate,acmentry_id,acmentry_instrument_id,acmentry_issuer_group_id,acmentry_issuer_id,acmentry_leg_group,acmentry_leg_id,acmentry_load_id,acmentry_nominal_amount,acmentry_number,acmentry_one_time_cpty_id,acmentry_package_map_id,acmentry_package_type_id,acmentry_param_0,acmentry_param_1,acmentry_param_10,acmentry_param_11,acmentry_param_12,acmentry_param_13,acmentry_param_14,acmentry_param_15,acmentry_param_16,acmentry_param_17,acmentry_param_18,acmentry_param_19,acmentry_param_2,acmentry_param_3,acmentry_param_4,acmentry_param_5,acmentry_param_6,acmentry_param_7,acmentry_param_8,acmentry_param_9,acmentry_portfolio_id,acmentry_reverse_event_id,acmentry_source_reference,acmentry_t_currency_2_id,acmentry_t_currency_id,acmentry_t_maturity_date,acmentry_t_opening_date,acmentry_t_value_date,acmentryorigin_id_user,acmentryoriginentity_id_user,acmentryorigingroup_id_user,acmentrytype_id_user,acmeventtype_id_user,acmgroupingrule_id,acmgroupingrule_id_user,acmledger_id,acmledger_id_user,acmmappingrule_id,acmmappingrule_id_user,acmperiod_id_user,acmperiodordernumber_number,acmplcategory_id_user,acmtransactionsign_name,acmvoucher_booking_date,acmvoucher_id,acmvoucher_posting_date,acmvoucher_voucher_number,transactionkind_name,transactiontype_name,umpathnode_id,umpathnode_upath +Grouping= +Sorting=acmentry_currency_id,acmvoucher_id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= +ACMLedger_id= +ACMPeriod_id= +StartDate= +EndDate= +Currency_id= +CcyGroupId= +ACMCurrencyEntryState_id= + + +[Format acmvoucher_voucher_number] +Name=acmvoucher_voucher_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_id] +Name=acmentry_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmledger_id] +Name=acmledger_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_id] +Name=acmvoucher_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=103 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmledger_id_user] +Name=acmledger_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_booking_date] +Name=acmvoucher_booking_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=161 +width_in_characters=32 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_posting_date] +Name=acmvoucher_posting_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmperiod_id_user] +Name=acmperiod_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmperiodordernumber_number] +Name=acmperiodordernumber_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umpathnode_id] +Name=umpathnode_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umpathnode_upath] +Name=umpathnode_upath +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_event_date] +Name=acmentry_event_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryorigin_id_user] +Name=acmentryorigin_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryorigingroup_id_user] +Name=acmentryorigingroup_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_figure_id] +Name=acmentry_figure_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accountingtreatment_name] +Name=accountingtreatment_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmbookkeepingtype_id_user] +Name=acmbookkeepingtype_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_description] +Name=acmentry_description +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmeventtype_id_user] +Name=acmeventtype_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_fx_rate] +Name=acmentry_fx_rate +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentrytype_id_user] +Name=acmentrytype_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmmappingrule_id] +Name=acmmappingrule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmmappingrule_id_user] +Name=acmmappingrule_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_currency_id] +Name=acmentry_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_booking_currency_id] +Name=acmentry_booking_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transactionkind_name] +Name=transactionkind_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transactiontype_name] +Name=transactiontype_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_currency_id] +Name=acmentry_t_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_currency_2_id] +Name=acmentry_t_currency_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_id] +Name=acmentry_counterparty_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_group_id] +Name=acmentry_counterparty_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_portfolio_id] +Name=acmentry_portfolio_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_package_type_id] +Name=acmentry_package_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_issuer_id] +Name=acmentry_issuer_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_issuer_group_id] +Name=acmentry_issuer_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_id] +Name=acmentry_bank_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcptyrelation_id_user] +Name=acmcptyrelation_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_event_id] +Name=acmentry_event_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_reverse_event_id] +Name=acmentry_reverse_event_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_number] +Name=acmentry_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_source_reference] +Name=acmentry_source_reference +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_package_map_id] +Name=acmentry_package_map_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmtransactionsign_name] +Name=acmtransactionsign_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_opening_date] +Name=acmentry_t_opening_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=97 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_value_date] +Name=acmentry_t_value_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_maturity_date] +Name=acmentry_t_maturity_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=101 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_c_payment_date] +Name=acmentry_c_payment_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_nominal_amount] +Name=acmentry_nominal_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_book_value] +Name=acmentry_book_value +Expression= +report_total= +date_format= +time_format= +Justify= +width=143 +width_in_characters=28 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_deal_rate] +Name=acmentry_deal_rate +Expression= +report_total= +date_format= +time_format= +Justify= +width=121 +width_in_characters=24 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_deal_price] +Name=acmentry_deal_price +Expression= +report_total= +date_format= +time_format= +Justify= +width=110 +width_in_characters=22 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_date_basis] +Name=acmentry_date_basis +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_instrument_id] +Name=acmentry_instrument_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_leg_id] +Name=acmentry_leg_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_leg_group] +Name=acmentry_leg_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_load_id] +Name=acmentry_load_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_account_holder_id] +Name=acmentry_account_holder_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_account] +Name=acmentry_bank_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_account_ccy_id] +Name=acmentry_bank_account_ccy_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_code] +Name=acmentry_counterparty_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_classification_id] +Name=acmentry_classification_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcmmstate_id_user] +Name=acmcmmstate_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_reconciliation_id] +Name=acmentry_cmm_reconciliation_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_group_id] +Name=acmentry_cmm_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryoriginentity_id_user] +Name=acmentryoriginentity_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_payment_method_id] +Name=acmentry_cmm_payment_method_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_one_time_cpty_id] +Name=acmentry_one_time_cpty_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcmmviewtype_id_user] +Name=acmcmmviewtype_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmplcategory_id_user] +Name=acmplcategory_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcurrencyentrystate_id_user] +Name=acmcurrencyentrystate_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcurrencyposition_id_user] +Name=acmcurrencyposition_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmgroupingrule_id] +Name=acmgroupingrule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmgroupingrule_id_user] +Name=acmgroupingrule_id_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_group_id] +Name=acmentry_ccy_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=80 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_amount] +Name=acmentry_ccy_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_booking_amount] +Name=acmentry_ccy_booking_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=14 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_result] +Name=acmentry_ccy_result +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_0] +Name=acmentry_param_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_1] +Name=acmentry_param_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_2] +Name=acmentry_param_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_3] +Name=acmentry_param_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_4] +Name=acmentry_param_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_5] +Name=acmentry_param_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_6] +Name=acmentry_param_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_7] +Name=acmentry_param_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_8] +Name=acmentry_param_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_9] +Name=acmentry_param_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_10] +Name=acmentry_param_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_11] +Name=acmentry_param_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_12] +Name=acmentry_param_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_13] +Name=acmentry_param_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_14] +Name=acmentry_param_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_15] +Name=acmentry_param_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_16] +Name=acmentry_param_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_17] +Name=acmentry_param_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_18] +Name=acmentry_param_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_19] +Name=acmentry_param_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml new file mode 100644 index 0000000..a50373a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml @@ -0,0 +1,15 @@ + + +parameters: + StartDate: + value: "t-10d" + hidden: false + + EndDate: + value: "t" + hidden: false + + ACMLedger_id: + value: 1 + hidden: true + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr new file mode 100644 index 0000000..275068b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr @@ -0,0 +1,3143 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=acm-entry-state-ledger-group +Name=Accounting Entry State for Ledger Group Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Accounting Entry State for Ledger Group Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=acmgroupingrule2_id,acmgroupingrule2_id_user,acmgroupingrule1_id,acmgroupingrule1_id_user,acmmappingrule_id,acmmappingrule_id_user,acmentry_event_id,acmentry_pase_id,acmentry_cmmse_id,acmentry_voucher_id,acmvoucher_voucher_number,acmentry_aggregation_id,acmentry_id,acmledger_chart_id,acmentry_ledger_id,repccy,reportccyamount,acmrevcancelstate1_id_user,acmentry_account_id,acmaccount1_name,acmaccount1_id_user,acmaccount1_erp_account,acmplcategory_id_user,acmledger_id_user,acmentry_classification_id,acmentrystate1_id_user,acmentry_export_id,acmentry_status,acmentry_reference_date,acmentry_event_date,acmentry_booking_date,acmentry_modification_time,acmentryorigin_id_user,acmentryorigingroup_id_user,acmbookkeepingtype_id_user,acmeventtype_id_user,acmentry_figure_id,acmentry_description,acmentry_mapping_description,acmentry_amount,acmentry_booking_amount,acmentry_fx_rate,acmentrytype_id_user,acmmappingaction_id_user,acmentry_parent_id,acmsign1_name,acmsign2_name,acmentry_original_amount,acmentry_orig_booking_amount,acmdrcr_id_user,acmproject_id_user,acmcostcenter_id_user,acmspecialtreatment_id_user,acmrebookingtype_id_user,acmentry_secondary_account_id,acmaccount2_id_user,acmentry_tag,acmentry_currency_id,acmentry_booking_currency_id,transactionkind_name,transactiontype_name,acmentry_t_currency_id,acmentry_t_currency_2_id,acmentry_counterparty_id,acmentry_counterparty_group_id,acmcptyrelation_id_user,acmentry_portfolio_id,acmentry_package_type_id,acmentry_issuer_id,acmentry_issuer_group_id,acmentry_bank_id,acmentry_batch_id,acmentry_reverse_event_id,acmentry_number,acmentry_package_map_id,acmtransactionsign_name,acmentry_t_opening_date,acmentry_t_value_date,acmentry_t_maturity_date,acmentry_c_payment_date,acmentry_nominal_amount,acmentry_book_value,acmentry_deal_rate,acmentry_deal_price,acmentry_date_basis,umpathnode_id,umpathnode_upath,acmentry_instrument_id,acmentry_leg_id,acmentry_leg_group,acmentry_bank_account,acmentry_hedge_id,acmentry_hedge_risk,acmhedgelegcat_id_user,acmhedgeeffcat_id_user,acmhedgecostcat_id_user,acmhedgequalifcat_id_user,acmentry_hedge_type,acmentry_hedge_test_method,acmentry_hedge_strategy_code,acmentry_hedge_design_percent,acmentry_hedge_eff_percent,acmentry_hedge_doc_eff_bal,acmentry_hedge_book_eff_bal,acmentry_hedge_doc_ineff_bal,acmentry_hedge_book_ineff_bal,hedgededesignationtype_name,acmentry_adj_discharge_date,acmctbtyperule_id_user,acmentry_match_id,accountingtreatment_name,acmentry_param_0,acmentry_param_1,acmentry_param_2,acmentry_param_3,acmentry_param_4,acmentry_param_5,acmentry_param_6,acmentry_param_7,acmentry_param_8,acmentry_param_9,acmentry_pre_voucher_id,acmentry_flags,acmentry_flags_2,acmvouchertype2_id_user,acmperiod_id_user,acmvoucher_flags,acmvoucherstate_id_user,acmvoucher_booking_date,acmvoucher_modification_time,acmentry_cmm_reconciliation_id,acmentry_cmm_group_id,acmcmmstate_id_user,acmentryoriginentity_id_user,acmcmmviewtype_id_user,acmentry_cmm_payment_method_id,acmentry_load_id,acmentry_source_reference,acmentry_one_time_cpty_id,acmentry_figure_amount,acmentry_figure_booking_amount,acmrevcancelstate2_id_user,acmentrystate2_id_user,acmentry_counterparty_code,acmentry_ctb_sequence_id,acmentry_ctb_previous_date,acmvouchertype_id_user,acmrevcancelstate3_id_user,acmvoucher_posting_date,acmvoucher_req_booking_date,acmvoucher_bundle_id,acmvoucher_reversal_bundle_id,acmvoucher_rev_voucher_id,acmentry_account_holder_id,acmentry_acc_holder_group_id,acmentry_acc_holder_code,acmentry_bank_account_ccy_id,acmentry_continuation_number,acmentry_logical_reference_id,acmentry_param_10,acmentry_param_11,acmentry_param_12,acmentry_param_13,acmentry_param_14,acmentry_param_15,acmentry_param_16,acmentry_param_17,acmentry_param_18,acmentry_param_19,ccy_state,ccy_position,acmentry_ccy_grouping_rule_id,ccy_grouping_rule,acmentry_ccy_group_id,acmentry_ccy_amount,acmentry_ccy_booking_amount,acmentry_ccy_result,valuationhierarchy_name,ctb_category,original_time_band,acmentry_n_option_date,option_sign,acmentry_facility_id,acmentry_fac_rep_group,acmentry_owner_unit_id,acmentry_counterparty_unit_id,acmentry_issuer_unit_id,client_unit_relation_id,report_owner_unit_group_id +Selected=acmaccount1_id_user,acmplcategory_id_user,acmaccount1_name,acmentry_account_id,accountingtreatment_name,acmmappingaction_id_user,acmentry_adj_discharge_date,acmentry_aggregation_id,acmgroupingrule1_id_user,acmgroupingrule1_id,acmsign1_name,acmentry_bank_id,acmentry_bank_account,acmentry_bank_account_ccy_id,acmentry_account_holder_id,acmentry_acc_holder_code,acmentry_acc_holder_group_id,acmentry_batch_id,acmentry_book_value,acmentry_booking_amount,acmentry_booking_currency_id,acmentry_booking_date,acmbookkeepingtype_id_user,acmentry_param_0,acmentry_cmmse_id,acmentry_cmm_group_id,acmentry_cmm_payment_method_id,acmentry_cmm_reconciliation_id,acmcmmstate_id_user,ctb_category,acmentry_ctb_previous_date,acmentry_ctb_sequence_id,acmctbtyperule_id_user,acmrevcancelstate2_id_user,acmentry_c_payment_date,acmentry_figure_id,acmledger_chart_id,acmentry_classification_id,client_unit_relation_id,acmentry_continuation_number,acmcostcenter_id_user,acmhedgecostcat_id_user,acmentry_counterparty_id,acmentry_counterparty_code,acmentry_counterparty_group_id,acmcptyrelation_id_user,acmentry_counterparty_unit_id,acmentry_ccy_amount,acmentry_ccy_booking_amount,acmentry_ccy_group_id,acmentry_ccy_result,ccy_grouping_rule,acmentry_ccy_grouping_rule_id,ccy_position,ccy_state,acmdrcr_id_user,acmentry_date_basis,acmentry_deal_price,acmentry_deal_rate,acmentry_description,acmentry_amount,acmentry_currency_id,acmaccount1_erp_account,acmentry_param_1,acmentry_id,acmentrytype_id_user,acmvouchertype2_id_user,acmentry_event_date,acmentry_event_id,acmeventtype_id_user,acmentry_export_id,acmentry_facility_id,acmentry_fac_rep_group,acmentry_figure_amount,acmentry_figure_booking_amount,acmentry_flags,acmentry_flags_2,acmentry_fx_rate,acmsign2_name,acmgroupingrule2_id_user,acmgroupingrule2_id,acmentry_hedge_book_eff_bal,acmentry_hedge_book_ineff_bal,hedgededesignationtype_name,acmentry_hedge_design_percent,acmentry_hedge_doc_eff_bal,acmentry_hedge_doc_ineff_bal,acmhedgeeffcat_id_user,acmentry_hedge_eff_percent,acmhedgequalifcat_id_user,acmhedgelegcat_id_user,acmentry_hedge_risk,acmentry_hedge_strategy_code,acmentry_hedge_id,acmentry_hedge_test_method,acmentry_hedge_type,acmentry_instrument_id,umpathnode_upath,umpathnode_id,acmentry_issuer_id,acmentry_issuer_group_id,acmentry_issuer_unit_id,acmledger_id_user,acmentry_ledger_id,acmentry_leg_id,acmentry_leg_group,acmentry_load_id,acmentry_logical_reference_id,acmentry_mapping_description,acmmappingrule_id_user,acmmappingrule_id,acmentry_match_id,acmentry_modification_time,acmentry_n_option_date,acmentry_nominal_amount,acmentry_one_time_cpty_id,option_sign,acmentryorigin_id_user,acmentryoriginentity_id_user,acmentryorigingroup_id_user,acmentry_original_amount,acmentry_orig_booking_amount,original_time_band,acmentrystate2_id_user,acmentry_owner_unit_id,acmentry_package_map_id,acmentry_package_type_id,acmentry_param_10,acmentry_param_11,acmentry_param_12,acmentry_param_13,acmentry_param_14,acmentry_param_15,acmentry_param_17,acmentry_param_18,acmentry_param_19,acmentry_param_5,acmentry_param_6,acmentry_param_7,acmentry_param_8,acmentry_param_9,acmentry_parent_id,acmentry_pase_id,acmperiod_id_user,acmentry_portfolio_id,acmentry_pre_voucher_id,acmentry_param_16,acmproject_id_user,acmentry_reference_date,acmentry_number,repccy,reportccyamount,report_owner_unit_group_id,acmvoucher_req_booking_date,acmvoucher_reversal_bundle_id,acmentry_reverse_event_id,acmrevcancelstate1_id_user,acmvoucher_rev_voucher_id,acmaccount2_id_user,acmentry_secondary_account_id,acmentry_source_reference,acmspecialtreatment_id_user,acmrebookingtype_id_user,acmentrystate1_id_user,acmentry_status,acmentry_tag,acmentry_t_currency_id,acmentry_t_currency_2_id,transactionkind_name,acmentry_t_maturity_date,acmentry_t_opening_date,acmentry_param_3,acmentry_param_4,acmtransactionsign_name,transactiontype_name,acmentry_t_value_date,acmvoucher_bundle_id,valuationhierarchy_name,acmcmmviewtype_id_user,acmvoucher_booking_date,acmvoucher_flags,acmvoucher_modification_time,acmvoucher_voucher_number,acmvoucher_posting_date,acmrevcancelstate3_id_user,acmvoucherstate_id_user,acmentry_voucher_id,acmvouchertype_id_user,acmentry_param_2 +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=ACMLedgerGroup_id,ACMClientUnit_id,ACMClientUnitGroup_id,StartEntryBookingDate,EndEntryBookingDate,StartEventDate,EndEventDate,StartTransactionMaturityDate,EndTransactionMaturityDate,ACMVoucherState_id,ACMEntryState_id,ACMBookkeepingType_id,ACMEntry_ids,Event_id,Number,LogicalReferenceNumber,Hedge,ACMEntryOriginEntity_id,ACMEntryOriginGroup_id,ACMEntryOrigin_id,ACMDrCr_id,Aggregation_id,Aggregation_id_2,DocumentCcy,ACMChart_id,Portfolio_id,Facility_id,Instrument_id,ACMCostCenter_id,ACMProject_id,AdditionalGrouping,ACMLedger_owner_id,ACMLedger_contexts,ReportCcy,ACMScenario_id,TranslationDate,RowsNotLimited +ACMLedgerGroup_id=1 +ACMClientUnit_id= +ACMClientUnitGroup_id= +StartEntryBookingDate= +EndEntryBookingDate= +StartEventDate= +EndEventDate= +StartTransactionMaturityDate= +EndTransactionMaturityDate= +ACMVoucherState_id= +ACMEntryState_id= +ACMBookkeepingType_id= +ACMEntry_ids= +Event_id= +Number= +LogicalReferenceNumber= +Hedge= +ACMEntryOriginEntity_id= +ACMEntryOriginGroup_id= +ACMEntryOrigin_id= +ACMDrCr_id= +Aggregation_id= +Aggregation_id_2= +DocumentCcy= +ACMChart_id= +Portfolio_id= +Facility_id= +Instrument_id= +ACMCostCenter_id= +ACMProject_id= +AdditionalGrouping= +ACMLedger_owner_id= +ACMLedger_contexts= +ReportCcy= +ACMScenario_id= +TranslationDate= +RowsNotLimited=2 + + +[Format acmgroupingrule2_id] +Name=ACMGROUPINGRULE2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmgroupingrule2_id_user] +Name=ACMGROUPINGRULE2_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmgroupingrule1_id] +Name=ACMGROUPINGRULE1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmgroupingrule1_id_user] +Name=ACMGROUPINGRULE1_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmmappingrule_id] +Name=ACMMAPPINGRULE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmmappingrule_id_user] +Name=ACMMAPPINGRULE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_event_id] +Name=ACMENTRY_EVENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_pase_id] +Name=ACMENTRY_PASE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmmse_id] +Name=ACMENTRY_CMMSE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_voucher_id] +Name=ACMENTRY_VOUCHER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_voucher_number] +Name=ACMVOUCHER_VOUCHER_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_aggregation_id] +Name=ACMENTRY_AGGREGATION_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_id] +Name=ACMENTRY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmledger_chart_id] +Name=ACMLEDGER_CHART_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ledger_id] +Name=ACMENTRY_LEDGER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format repccy] +Name=REPCCY +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reportccyamount] +Name=REPORTCCYAMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmrevcancelstate1_id_user] +Name=ACMREVCANCELSTATE1_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_account_id] +Name=ACMENTRY_ACCOUNT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmaccount1_name] +Name=ACMACCOUNT1_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmaccount1_id_user] +Name=ACMACCOUNT1_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmaccount1_erp_account] +Name=ACMACCOUNT1_ERP_ACCOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmplcategory_id_user] +Name=ACMPLCATEGORY_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmledger_id_user] +Name=ACMLEDGER_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_classification_id] +Name=ACMENTRY_CLASSIFICATION_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentrystate1_id_user] +Name=ACMENTRYSTATE1_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_export_id] +Name=ACMENTRY_EXPORT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_status] +Name=ACMENTRY_STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_reference_date] +Name=ACMENTRY_REFERENCE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_event_date] +Name=ACMENTRY_EVENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_booking_date] +Name=ACMENTRY_BOOKING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_modification_time] +Name=ACMENTRY_MODIFICATION_TIME +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryorigin_id_user] +Name=ACMENTRYORIGIN_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryorigingroup_id_user] +Name=ACMENTRYORIGINGROUP_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmbookkeepingtype_id_user] +Name=ACMBOOKKEEPINGTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmeventtype_id_user] +Name=ACMEVENTTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_figure_id] +Name=ACMENTRY_FIGURE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_description] +Name=ACMENTRY_DESCRIPTION +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_mapping_description] +Name=ACMENTRY_MAPPING_DESCRIPTION +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_amount] +Name=ACMENTRY_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_booking_amount] +Name=ACMENTRY_BOOKING_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_fx_rate] +Name=ACMENTRY_FX_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentrytype_id_user] +Name=ACMENTRYTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmmappingaction_id_user] +Name=ACMMAPPINGACTION_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_parent_id] +Name=ACMENTRY_PARENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmsign1_name] +Name=ACMSIGN1_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmsign2_name] +Name=ACMSIGN2_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_original_amount] +Name=ACMENTRY_ORIGINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_orig_booking_amount] +Name=ACMENTRY_ORIG_BOOKING_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmdrcr_id_user] +Name=ACMDRCR_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmproject_id_user] +Name=ACMPROJECT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcostcenter_id_user] +Name=ACMCOSTCENTER_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmspecialtreatment_id_user] +Name=ACMSPECIALTREATMENT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmrebookingtype_id_user] +Name=ACMREBOOKINGTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_secondary_account_id] +Name=ACMENTRY_SECONDARY_ACCOUNT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmaccount2_id_user] +Name=ACMACCOUNT2_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_tag] +Name=ACMENTRY_TAG +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_currency_id] +Name=ACMENTRY_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_booking_currency_id] +Name=ACMENTRY_BOOKING_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transactionkind_name] +Name=TRANSACTIONKIND_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transactiontype_name] +Name=TRANSACTIONTYPE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_currency_id] +Name=ACMENTRY_T_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_currency_2_id] +Name=ACMENTRY_T_CURRENCY_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_id] +Name=ACMENTRY_COUNTERPARTY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_group_id] +Name=ACMENTRY_COUNTERPARTY_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcptyrelation_id_user] +Name=ACMCPTYRELATION_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_portfolio_id] +Name=ACMENTRY_PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_package_type_id] +Name=ACMENTRY_PACKAGE_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_issuer_id] +Name=ACMENTRY_ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_issuer_group_id] +Name=ACMENTRY_ISSUER_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_id] +Name=ACMENTRY_BANK_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_batch_id] +Name=ACMENTRY_BATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_reverse_event_id] +Name=ACMENTRY_REVERSE_EVENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_number] +Name=ACMENTRY_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_package_map_id] +Name=ACMENTRY_PACKAGE_MAP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmtransactionsign_name] +Name=ACMTRANSACTIONSIGN_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_opening_date] +Name=ACMENTRY_T_OPENING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_value_date] +Name=ACMENTRY_T_VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_t_maturity_date] +Name=ACMENTRY_T_MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_c_payment_date] +Name=ACMENTRY_C_PAYMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_nominal_amount] +Name=ACMENTRY_NOMINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_book_value] +Name=ACMENTRY_BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_deal_rate] +Name=ACMENTRY_DEAL_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_deal_price] +Name=ACMENTRY_DEAL_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_date_basis] +Name=ACMENTRY_DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umpathnode_id] +Name=UMPATHNODE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umpathnode_upath] +Name=UMPATHNODE_UPATH +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_instrument_id] +Name=ACMENTRY_INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_leg_id] +Name=ACMENTRY_LEG_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_leg_group] +Name=ACMENTRY_LEG_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_account] +Name=ACMENTRY_BANK_ACCOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_id] +Name=ACMENTRY_HEDGE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_risk] +Name=ACMENTRY_HEDGE_RISK +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmhedgelegcat_id_user] +Name=ACMHEDGELEGCAT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmhedgeeffcat_id_user] +Name=ACMHEDGEEFFCAT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmhedgecostcat_id_user] +Name=ACMHEDGECOSTCAT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmhedgequalifcat_id_user] +Name=ACMHEDGEQUALIFCAT_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_type] +Name=ACMENTRY_HEDGE_TYPE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_test_method] +Name=ACMENTRY_HEDGE_TEST_METHOD +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_strategy_code] +Name=ACMENTRY_HEDGE_STRATEGY_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_design_percent] +Name=ACMENTRY_HEDGE_DESIGN_PERCENT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_eff_percent] +Name=ACMENTRY_HEDGE_EFF_PERCENT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_doc_eff_bal] +Name=ACMENTRY_HEDGE_DOC_EFF_BAL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_book_eff_bal] +Name=ACMENTRY_HEDGE_BOOK_EFF_BAL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_doc_ineff_bal] +Name=ACMENTRY_HEDGE_DOC_INEFF_BAL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_hedge_book_ineff_bal] +Name=ACMENTRY_HEDGE_BOOK_INEFF_BAL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format hedgededesignationtype_name] +Name=HEDGEDEDESIGNATIONTYPE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_adj_discharge_date] +Name=ACMENTRY_ADJ_DISCHARGE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmctbtyperule_id_user] +Name=ACMCTBTYPERULE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_match_id] +Name=ACMENTRY_MATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accountingtreatment_name] +Name=ACCOUNTINGTREATMENT_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_0] +Name=ACMENTRY_PARAM_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_1] +Name=ACMENTRY_PARAM_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_2] +Name=ACMENTRY_PARAM_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_3] +Name=ACMENTRY_PARAM_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_4] +Name=ACMENTRY_PARAM_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_5] +Name=ACMENTRY_PARAM_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_6] +Name=ACMENTRY_PARAM_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_7] +Name=ACMENTRY_PARAM_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_8] +Name=ACMENTRY_PARAM_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_9] +Name=ACMENTRY_PARAM_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_pre_voucher_id] +Name=ACMENTRY_PRE_VOUCHER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_flags] +Name=ACMENTRY_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_flags_2] +Name=ACMENTRY_FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvouchertype2_id_user] +Name=ACMVOUCHERTYPE2_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmperiod_id_user] +Name=ACMPERIOD_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_flags] +Name=ACMVOUCHER_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucherstate_id_user] +Name=ACMVOUCHERSTATE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_booking_date] +Name=ACMVOUCHER_BOOKING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_modification_time] +Name=ACMVOUCHER_MODIFICATION_TIME +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_reconciliation_id] +Name=ACMENTRY_CMM_RECONCILIATION_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_group_id] +Name=ACMENTRY_CMM_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcmmstate_id_user] +Name=ACMCMMSTATE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentryoriginentity_id_user] +Name=ACMENTRYORIGINENTITY_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmcmmviewtype_id_user] +Name=ACMCMMVIEWTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_cmm_payment_method_id] +Name=ACMENTRY_CMM_PAYMENT_METHOD_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_load_id] +Name=ACMENTRY_LOAD_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_source_reference] +Name=ACMENTRY_SOURCE_REFERENCE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_one_time_cpty_id] +Name=ACMENTRY_ONE_TIME_CPTY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_figure_amount] +Name=ACMENTRY_FIGURE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_figure_booking_amount] +Name=ACMENTRY_FIGURE_BOOKING_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmrevcancelstate2_id_user] +Name=ACMREVCANCELSTATE2_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentrystate2_id_user] +Name=ACMENTRYSTATE2_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_code] +Name=ACMENTRY_COUNTERPARTY_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ctb_sequence_id] +Name=ACMENTRY_CTB_SEQUENCE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ctb_previous_date] +Name=ACMENTRY_CTB_PREVIOUS_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvouchertype_id_user] +Name=ACMVOUCHERTYPE_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmrevcancelstate3_id_user] +Name=ACMREVCANCELSTATE3_ID_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_posting_date] +Name=ACMVOUCHER_POSTING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_req_booking_date] +Name=ACMVOUCHER_REQ_BOOKING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_bundle_id] +Name=ACMVOUCHER_BUNDLE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_reversal_bundle_id] +Name=ACMVOUCHER_REVERSAL_BUNDLE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmvoucher_rev_voucher_id] +Name=ACMVOUCHER_REV_VOUCHER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_account_holder_id] +Name=ACMENTRY_ACCOUNT_HOLDER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_acc_holder_group_id] +Name=ACMENTRY_ACC_HOLDER_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_acc_holder_code] +Name=ACMENTRY_ACC_HOLDER_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_bank_account_ccy_id] +Name=ACMENTRY_BANK_ACCOUNT_CCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_continuation_number] +Name=ACMENTRY_CONTINUATION_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_logical_reference_id] +Name=ACMENTRY_LOGICAL_REFERENCE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_10] +Name=ACMENTRY_PARAM_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_11] +Name=ACMENTRY_PARAM_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_12] +Name=ACMENTRY_PARAM_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_13] +Name=ACMENTRY_PARAM_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_14] +Name=ACMENTRY_PARAM_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_15] +Name=ACMENTRY_PARAM_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_16] +Name=ACMENTRY_PARAM_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_17] +Name=ACMENTRY_PARAM_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_18] +Name=ACMENTRY_PARAM_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_param_19] +Name=ACMENTRY_PARAM_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ccy_state] +Name=CCY_STATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ccy_position] +Name=CCY_POSITION +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_grouping_rule_id] +Name=ACMENTRY_CCY_GROUPING_RULE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ccy_grouping_rule] +Name=CCY_GROUPING_RULE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_group_id] +Name=ACMENTRY_CCY_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_amount] +Name=ACMENTRY_CCY_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_booking_amount] +Name=ACMENTRY_CCY_BOOKING_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_ccy_result] +Name=ACMENTRY_CCY_RESULT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format valuationhierarchy_name] +Name=VALUATIONHIERARCHY_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ctb_category] +Name=CTB_CATEGORY +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format original_time_band] +Name=ORIGINAL_TIME_BAND +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_n_option_date] +Name=ACMENTRY_N_OPTION_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format option_sign] +Name=OPTION_SIGN +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_facility_id] +Name=ACMENTRY_FACILITY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_fac_rep_group] +Name=ACMENTRY_FAC_REP_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_owner_unit_id] +Name=ACMENTRY_OWNER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_counterparty_unit_id] +Name=ACMENTRY_COUNTERPARTY_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format acmentry_issuer_unit_id] +Name=ACMENTRY_ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_unit_relation_id] +Name=CLIENT_UNIT_RELATION_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format report_owner_unit_group_id] +Name=REPORT_OWNER_UNIT_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml new file mode 100644 index 0000000..2ed88d1 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml @@ -0,0 +1,20 @@ + + +parameters: + StartEntryBookingDate: + value: "t-10d" + hidden: false + + EndEntryBookingDate: + value: "t" + hidden: false + + StartEventDate: + value: "" + hidden: true + + EndEventDate: + value: "" + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr new file mode 100644 index 0000000..5e1e45c --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr @@ -0,0 +1,338 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=activity +Name=Activity Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Activity Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=group_id,id,type_id,domain_id,flags,name,service_name,service_group,start_date,time_zone_id,end_date,prerequisite_id,owner_id,type_name,start_date_local,end_date_local +Selected=group_id,id,type_id,domain_id,flags,name,service_name,service_group,start_date,time_zone_id,end_date,prerequisite_id,owner_id,type_name,start_date_local,end_date_local +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= + + +[Format group_id] +Name=group_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type_id] +Name=type_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format domain_id] +Name=domain_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format service_name] +Name=service_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format service_group] +Name=service_group +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format start_date] +Name=start_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name=time_zone_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format end_date] +Name=end_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format prerequisite_id] +Name=prerequisite_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format owner_id] +Name=owner_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type_name] +Name=type_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format start_date_local] +Name=start_date_local +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format end_date_local] +Name=end_date_local +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr new file mode 100644 index 0000000..146e6ee --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr @@ -0,0 +1,248 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=activity-log-due +Name=Activity Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Activity Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=start_date = '' +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id +Selected=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id +Grouping= +Sorting=due_date +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from,to,due_date_from,due_date_to,state,batch_id +from= +to= +due_date_from= +due_date_to= +state= +batch_id= + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=152 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format number] +Name=NUMBER +Expression= +report_total= +date_format= +time_format= +Justify=right +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format batch_id] +Name=BATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=125 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format prerequisite_id] +Name=PREREQUISITE_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=96 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format due_date] +Name=DUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format start_date] +Name=START_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=163 +width_in_characters=32 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format finish_date] +Name=FINISH_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=126 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format description] +Name=DESCRIPTION +Expression= +report_total= +date_format= +time_format= +Justify=left +width=353 +width_in_characters=70 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format domain_id] +Name=DOMAIN +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format user_id] +Name=USER_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr new file mode 100644 index 0000000..71db57e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr @@ -0,0 +1,248 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=activity-log +Name=Activity Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Activity Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id +Selected=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from,to,due_date_from,due_date_to,state,batch_id +from= +to= +due_date_from= +due_date_to= +state= +batch_id= + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format number] +Name=NUMBER_ +Expression= +report_total= +date_format= +time_format= +Justify=right +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format batch_id] +Name=BATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format prerequisite_id] +Name=PREREQUISITE_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=125 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format due_date] +Name=DUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format start_date] +Name=START_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format finish_date] +Name=FINISH_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=218 +width_in_characters=43 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format description] +Name=DESCRIPTION +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format domain_id] +Name=DOMAIN_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format user_id] +Name=USER_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=88 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json new file mode 100644 index 0000000..d5dad20 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json @@ -0,0 +1,13 @@ + +{ + "from": + { + "value": "t-1d" + }, + "to": + { + "value": "t+1d" + } +} + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml new file mode 100644 index 0000000..466bb5b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml @@ -0,0 +1,12 @@ + + +parameters: + from: + value: "t-1d" + hidden: false + + to: + value: + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr new file mode 100644 index 0000000..dd7bfa5 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr @@ -0,0 +1,896 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=balance +Name=Balance Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Balance Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=number,id,date,amount,accrued_interest_percent,client_id,client_unit_id,bank_id,bank_unit_id,account_id,currency_id,cp_client_id,cp_client_unit_id,type,order_number,flags,name,portfolio_id,instrument_id,instrument_group,state_id,payment_id,payment_date,payment_currency_id,payment_amount,fx_spot_rate,affect_number,payment_client_id,payment_client_unit_id,credit_client_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19 +Selected=number,id,date,amount,accrued_interest_percent,client_id,client_unit_id,bank_id,bank_unit_id,account_id,currency_id,cp_client_id,cp_client_unit_id,type,order_number,flags,name,portfolio_id,instrument_id,instrument_group,state_id,payment_id,payment_date,payment_currency_id,payment_amount,fx_spot_rate,affect_number,payment_client_id,payment_client_unit_id,credit_client_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19 +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from_date,to_date,portfolio_id,currency_id,currency_class_id,client_id,client_unit_id,bank_id,bank_unit_id,account_id,minimum_state_id,rule_id,display_all_balances_p,display_ai_p +from_date= +to_date= +portfolio_id= +currency_id= +currency_class_id= +client_id= +client_unit_id= +bank_id= +bank_unit_id= +account_id= +minimum_state_id= +rule_id= +display_all_balances_p= +display_ai_p= + + +[Format number] +Name=NUMBER_ +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date] +Name=DATE_ +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format amount] +Name=AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest_percent] +Name=ACCRUED_INTEREST_PERCENT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name=CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name=CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bank_id] +Name=BANK_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bank_unit_id] +Name=BANK_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format account_id] +Name=ACCOUNT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=CP_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=CP_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type] +Name=TYPE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format order_number] +Name=ORDER_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify=left +width=119 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=NAME +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_id] +Name=STATE_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_id] +Name=PAYMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_date] +Name=PAYMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_currency_id] +Name=PAYMENT_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_amount] +Name=PAYMENT_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_spot_rate] +Name=FX_SPOT_RATE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format affect_number] +Name=AFFECT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_client_id] +Name=PAYMENT_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_client_unit_id] +Name=PAYMENT_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format credit_client_id] +Name=CREDIT_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_0] +Name=PARAM_0 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_1] +Name=PARAM_1 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_2] +Name=PARAM_2 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_3] +Name=PARAM_3 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_4] +Name=PARAM_4 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_5] +Name=PARAM_5 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_6] +Name=PARAM_6 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_7] +Name=PARAM_7 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_8] +Name=PARAM_8 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_9] +Name=PARAM_9 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_10] +Name=PARAM_10 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_11] +Name=PARAM_11 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_12] +Name=PARAM_12 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_13] +Name=PARAM_13 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_14] +Name=PARAM_14 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_15] +Name=PARAM_15 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_16] +Name=PARAM_16 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_17] +Name=PARAM_17 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_18] +Name=PARAM_18 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format param_19] +Name=PARAM_19 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml new file mode 100644 index 0000000..9e0b390 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml @@ -0,0 +1,20 @@ + + +parameters: + from_date: + value: "t-10d" + hidden: false + + to_date: + value: "t" + hidden: false + + display_all_balances_p: + value: 1 + hidden: true + + display_ai_p: + value: 1 + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr new file mode 100644 index 0000000..fa84785 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr @@ -0,0 +1,418 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=blackoutlog +Name=Blackout History Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Blackout History Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_date,log_user,log_action,log_event,log_type,id,isin,programme,active_from,active_until,event_date,user1,user2,flags,description,comment,log_caller,rule_id,name,tms_instrument_id,log_id +Selected=log_id,log_date,log_user,log_action,id,isin,programme,active_from,active_until,event_date,tms_instrument_id,rule_id,description,comment,user1,user2,name +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, +DigitGrouping=03 + + +[Parameters] + + +[Format log_date] +Name=Log Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=130 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_user] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=88 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_action] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_event] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=81 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format isin] +Name=ISIN +Expression= +report_total= +date_format= +time_format= +Justify=left +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format programme] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_from] +Name=Active From +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=Active Until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=97 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format event_date] +Name=Event Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=101 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=75 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format description] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=147 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_caller] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=125 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=202 +width_in_characters=40 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=130 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format tms_instrument_id] +Name=TMS Instrument ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=279 +width_in_characters=55 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format log_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml new file mode 100644 index 0000000..78e85e2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml @@ -0,0 +1,9 @@ + + +parameters: + + min_log_id: + value: select("w_ODS_TMS_BLACKOUT_LOG", "max(log_id) + 1") + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr new file mode 100644 index 0000000..b9171c4 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr @@ -0,0 +1,145 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=branch +Name=Branch Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Branch Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,number,name,order_number +Selected=id,name,number,order_number +Grouping= +Sorting=number,order_number +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=149 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format number] +Name=number_ +Expression= +report_total= +date_format= +time_format= +Justify=right +width=90 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=221 +width_in_characters=44 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format order_number] +Name=order_number +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr new file mode 100644 index 0000000..795ff22 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr @@ -0,0 +1,161 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=calendar +Name=Calendar Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Calendar Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,name,holiday_date,holiday_reason,default_holidays +Selected=id,name,holiday_date,holiday_reason,default_holidays +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=NAME +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format holiday_date] +Name=HOLIDAY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=156 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format holiday_reason] +Name=HOLIDAY_REASON +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format default_holidays] +Name=DEFAULT_HOLIDAYS +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.fkr new file mode 100644 index 0000000..b705235 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.fkr @@ -0,0 +1,3521 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=cashflow +Name=Cashflow Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Cashflow Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=affect_number,number,id,amount,currency_id,actual_amount,actual_currency_id,leg_group,active_since,active_until,since_when,until_when,base_currency_id,currency_class_id,transaction_currency_id,transaction_currency_2_id,type_id,subtype_id,leg_id,nominal_rate,fixing_quote,factor,rate_type,fixing_rate_id,fixing_period_id,cap,floor,date_basis,fx_rate,fx_spot_rate,fx_book_rate,cashflow_type_name,payment_date,payment_amount,payment_currency_id,spread,cashflow_value_date,cashflow_flags,cashflow_flags_2,instrument_type_id,instrument_group,local_client_id,local_client_unit_id,local_bank_1_id,local_bank_1_unit_id,local_account_1_id,local_bank_1_swift_code,local_account_1_closing_date,local_bank_2_id,local_account_2_id,local_bank_2_swift_code,local_account_2_closing_date,local_bank_3_id,local_account_3_id,local_bank_3_swift_code,local_account_3_closing_date,local_bank_4_id,local_account_4_id,local_bank_4_swift_code,local_account_4_closing_date,other_client_id,other_client_unit_id,other_bank_1_id,other_bank_1_unit_id,other_account_1_id,other_bank_1_swift_code,other_bank_2_id,other_account_2_id,other_bank_2_swift_code,other_bank_3_id,other_account_3_id,other_bank_3_swift_code,other_bank_4_id,other_account_4_id,other_bank_4_swift_code,nominal_amount,deal_rate,cashflow_maturity_date,package_main_type_id,package_type_id,package_id,top_portfolio_id,num,comment,comment_1,comment_2,comment_3,comment_4,fixing_offset,premium_type,p_date_basis,transaction_date,transaction_opening_date,transaction_value_date,transaction_maturity_date,market_id,instrument_id,issuer_id,issuer_unit_id,sign_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,maturity_date,expiry_date,opening_date,opening_time,closing_date,deal_price,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,agreed_amount,trading_unit,book_value,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,secondary_instrument_id,collateral_number,broker_id,cp_client_id,cp_client_unit_id,cp_trader_id,cp_portfolio_id,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,transaction_type_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19 +Selected=param_13,accrued_interest,active_since,active_until,affect_number,agreed_amount,amount,audit_number,base_currency_id,batch_id,book_value,cap,cashflow_flags,cashflow_flags_2,cashflow_maturity_date,cashflow_type_name,cashflow_value_date,closing_date,collateral_number,comment,comment_1,comment_2,comment_3,comment_4,contexts,cp_client_id,other_account_1_id,other_account_2_id,other_account_3_id,other_account_4_id,other_bank_1_id,other_bank_2_id,other_bank_2_swift_code,other_bank_3_id,other_bank_3_swift_code,other_bank_4_id,other_bank_4_swift_code,other_bank_1_swift_code,other_bank_1_unit_id,cp_portfolio_id,cp_trader_id,cp_client_unit_id,creator_tag,currency_id,currency_class_id,date,date_basis,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,deal_price,deal_rate,param_1,expiry_date,fx_book_rate,mattr_4,fx_rate,fx_spot_rate,sattr_1,factor,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,fixing_offset,fixing_period_id,fixing_quote,fixing_rate_id,flags,flags_2,flags_3,floor,id,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_7,iattr_8,iattr_9,instrument_id,instrument_group,instrument_type_id,issuer_id,issuer_unit_id,kind_id,leg_id,leg_group,sattr_4,logical_number,sattr_3,margin1,margin2,market_id,param_11,mattr_0,mattr_1,mattr_2,mattr_3,maturity_date,param_9,nominal_amount,nominal_rate,num,number,opening_date,opening_time,iattr_6,param_10,param_4,owner_id,local_account_1_id,local_account_2_id,local_account_2_closing_date,local_account_3_id,local_account_3_closing_date,local_account_4_id,local_account_4_closing_date,local_account_1_closing_date,local_bank_1_id,local_bank_2_id,local_bank_2_swift_code,local_bank_3_id,local_bank_3_swift_code,local_bank_4_id,local_bank_4_swift_code,local_bank_1_swift_code,local_bank_1_unit_id,owner_unit_id,p_date_basis,package_id,package_main_type_id,package_type_id,param_18,param_3,param_5,param_6,param_7,param_8,param_19,parent_number,payment_amount,payment_currency_id,payment_date,sattr_13,portfolio_id,premium_amount,premium_currency_id,premium_date,premium_price,premium_type,rate_type,param_0,reference_number,param_2,sattr_0,sattr_10,sattr_11,sattr_12,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,secondary_instrument_id,param_16,actual_amount,other_client_id,other_client_unit_id,actual_currency_id,settlement_date,local_client_id,local_client_unit_id,sign_id,since_when,spread,status,subtype_id,broker_id,top_portfolio_id,trader_id,trading_unit,sattr_2,transaction_currency_id,transaction_currency_2_id,transaction_date,transaction_maturity_date,transaction_opening_date,param_12,param_14,param_15,param_17,state_id,transaction_type_id,transaction_value_date,type_id,units,until_when,value_date +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format affect_number] +Name=AFFECT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=66 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_amount] +Name=ACTUAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_currency_id] +Name=ACTUAL_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_group] +Name=LEG_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=ACTIVE_SINCE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_until] +Name=ACTIVE_UNTIL +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format since_when] +Name=SINCE_WHEN +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format until_when] +Name=UNTIL_WHEN +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format base_currency_id] +Name=BASE_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_class_id] +Name=CURRENCY_CLASS_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_currency_id] +Name=TRANSACTION_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_currency_2_id] +Name=TRANSACTION_CURRENCY_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subtype_id] +Name=SUBTYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_id] +Name=LEG_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +Name=NOMINAL_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=98 +width_in_characters=19 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_quote] +Name=FIXING_QUOTE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=76 +width_in_characters=15 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format factor] +Name=FACTOR +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_type] +Name=RATE_TYPE +Expression= +report_total= +date_format= +time_format= +Justify= +width=109 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate_id] +Name=FIXING_RATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_id] +Name=FIXING_PERIOD_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cap] +Name=CAP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format floor] +Name=FLOOR +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name=DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_rate] +Name=FX_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_spot_rate] +Name=FX_SPOT_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_book_rate] +Name=FX_BOOK_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_type_name] +Name=CASHFLOW_TYPE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=PAYMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format payment_amount] +Name=PAYMENT_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_currency_id] +Name=PAYMENT_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +Name=SPREAD +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_value_date] +Name=CASHFLOW_VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=120 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cashflow_flags] +Name=CASHFLOW_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_flags_2] +Name=CASHFLOW_FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_type_id] +Name=INSTRUMENT_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_id] +Name=LOCAL_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_unit_id] +Name=LOCAL_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_id] +Name=LOCAL_BANK_1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_unit_id] +Name=LOCAL_BANK_1_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_1_id] +Name=LOCAL_ACCOUNT_1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_swift_code] +Name=LOCAL_BANK_1_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_1_closing_date] +Name=LOCAL_ACCOUNT_1_CLOSING_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_id] +Name=LOCAL_BANK_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_2_id] +Name=LOCAL_ACCOUNT_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_swift_code] +Name=LOCAL_BANK_2_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_2_closing_date] +Name=LOCAL_ACCOUNT_2_CLOSING_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_id] +Name=LOCAL_BANK_3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_3_id] +Name=LOCAL_ACCOUNT_3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_swift_code] +Name=LOCAL_BANK_3_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_3_closing_date] +Name=LOCAL_ACCOUNT_3_CLOSING_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_id] +Name=LOCAL_BANK_4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_4_id] +Name=LOCAL_ACCOUNT_4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_swift_code] +Name=LOCAL_BANK_4_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_4_closing_date] +Name=LOCAL_ACCOUNT_4_CLOSING_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_id] +Name=OTHER_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_unit_id] +Name=OTHER_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_id] +Name=OTHER_BANK_1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_unit_id] +Name=OTHER_BANK_1_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_1_id] +Name=OTHER_ACCOUNT_1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_swift_code] +Name=OTHER_BANK_1_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_id] +Name=OTHER_BANK_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_2_id] +Name=OTHER_ACCOUNT_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_swift_code] +Name=OTHER_BANK_2_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_id] +Name=OTHER_BANK_3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_3_id] +Name=OTHER_ACCOUNT_3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_swift_code] +Name=OTHER_BANK_3_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_id] +Name=OTHER_BANK_4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_4_id] +Name=OTHER_ACCOUNT_4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_swift_code] +Name=OTHER_BANK_4_SWIFT_CODE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount] +Name=NOMINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=21 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +Name=DEAL_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_maturity_date] +Name=CASHFLOW_MATURITY_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=75 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name=PACKAGE_MAIN_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +Name=PACKAGE_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_id] +Name=PACKAGE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format top_portfolio_id] +Name=TOP_PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format num] +Name=NUM +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name=COMMENT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_1] +Name=COMMENT_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_2] +Name=COMMENT_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_3] +Name=COMMENT_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_4] +Name=COMMENT_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=152 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_offset] +Name=FIXING_OFFSET +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_type] +Name=PREMIUM_TYPE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format p_date_basis] +Name=P_DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_date] +Name=TRANSACTION_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format transaction_opening_date] +Name=TRANSACTION_OPENING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format transaction_value_date] +Name=TRANSACTION_VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format transaction_maturity_date] +Name=TRANSACTION_MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format market_id] +Name=MARKET_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=180 +width_in_characters=36 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name=ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name=SIGN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name=KIND_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +Name=AUDIT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=54 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +Name=REFERENCE_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +Name=LOGICAL_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +Name=PARENT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name=BATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format settlement_date] +Name=SETTLEMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format value_date] +Name=VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format expiry_date] +Name=EXPIRY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format opening_date] +Name=OPENING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format opening_time] +Name=OPENING_TIME +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format closing_date] +Name=CLOSING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format deal_price] +Name=DEAL_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +Name=PREMIUM_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +Name=PREMIUM_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +Name=PREMIUM_DATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +Name=PREMIUM_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +Name=MARGIN1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +Name=MARGIN2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name=UNITS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +Name=AGREED_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name=TRADING_UNIT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name=BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify= +width=88 +width_in_characters=17 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name=ACCRUED_INTEREST +Expression= +report_total= +date_format= +time_format= +Justify=right +width=140 +width_in_characters=28 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=88 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name=OWNER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name=OWNER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +Name=TRADER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name=SECONDARY_INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +Name=COLLATERAL_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name=BROKER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=CP_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=CP_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +Name=CP_TRADER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name=CP_PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +Name=SATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +Name=SATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +Name=SATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +Name=SATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +Name=SATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +Name=SATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +Name=SATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +Name=SATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +Name=SATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +Name=SATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +Name=SATTR_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +Name=SATTR_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +Name=SATTR_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +Name=SATTR_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +Name=SATTR_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +Name=SATTR_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +Name=SATTR_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +Name=SATTR_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +Name=SATTR_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +Name=SATTR_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +Name=DATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +Name=DATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +Name=DATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +Name=DATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +Name=DATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +Name=DATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +Name=DATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +Name=DATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +Name=DATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +Name=DATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +Name=IATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +Name=IATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +Name=IATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +Name=IATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +Name=IATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +Name=IATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +Name=IATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +Name=IATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +Name=IATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +Name=IATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +Name=MATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +Name=MATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_2] +Name=MATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +Name=MATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +Name=MATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +Name=FATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +Name=FATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +Name=FATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +Name=FATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +Name=FATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=STATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name=CONTEXTS +Expression= +report_total= +date_format= +time_format= +Justify= +width=105 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +Name=CREATOR_TAG +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name=STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=42 +width_in_characters=8 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +Name=FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +Name=FLAGS_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_type_id] +Name=TRANSACTION_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=PARAM_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=PARAM_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=PARAM_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=PARAM_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=PARAM_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=PARAM_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=PARAM_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=PARAM_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=PARAM_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=PARAM_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=PARAM_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=PARAM_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=PARAM_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=PARAM_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=PARAM_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=PARAM_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=PARAM_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=PARAM_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=PARAM_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=PARAM_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.yml new file mode 100644 index 0000000..1d203e9 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CASHFLOW.yml @@ -0,0 +1,16 @@ + + +parameters: + from_date: + value: "t-10d" + hidden: false + + to_date: + value: "t" + hidden: false + + portfolio_id: + value: ACC-AVG-TOP-ALL + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.fkr new file mode 100644 index 0000000..d56873f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.fkr @@ -0,0 +1,470 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=client-account +Name=Client Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Client Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=client_id,kind,rule_id,currency_id,priority,not_rule_id,comment,name,active_since,active_until,delivery_vs_payment,prefer_bank,prefer_account,last_link,own_account,full_chain,bank1,account1,bank2,account2,bank3,account3,bank4,account4 +Selected=account1,account2,account3,account4,active_since,active_until,bank1,bank2,bank3,bank4,client_id,comment,currency_id,delivery_vs_payment,full_chain,kind,last_link,name,not_rule_id,own_account,prefer_account,prefer_bank,priority,rule_id +Grouping= +Sorting=client_id,priority,rule_id,kind +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= +kind= +client_id= +bank_id= +currency_id= + + +[Format client_id] +Name=client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind] +Name=kind +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_id] +Name=rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format priority] +Name=priority +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_id] +Name=not_rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name=comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=active_since +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=active_until +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_vs_payment] +Name=delivery_vs_payment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format prefer_bank] +Name=prefer_bank +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format prefer_account] +Name=prefer_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format last_link] +Name=last_link +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format own_account] +Name=own_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format full_chain] +Name=full_chain +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank1] +Name=bank1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account1] +Name=account1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank2] +Name=bank2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account2] +Name=account2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank3] +Name=bank3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account3] +Name=account3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank4] +Name=bank4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account4] +Name=account4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.yml new file mode 100644 index 0000000..d5c908e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTACCOUNT.yml @@ -0,0 +1,8 @@ + + +parameters: + Fields: + value: "kind,client_id,bank_id,currency_id" + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.fkr new file mode 100644 index 0000000..036c3b0 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.fkr @@ -0,0 +1,1481 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=settlement-instructions +Name=Settlement Instructions Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Settlement Instructions Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=client_id,client_name,instruction_name,currency_id,priority,kind,active_since,active_until,client_unit_id,comment,bank_1,account_1_number,account_1_currency,account_1_type,account_1_name,account_1_target_closing_date,account_1_secondary_number,account_1_group,account_1_parent_account,account_1_comment,bank_2,account_2_number,account_2_currency,account_2_type,account_2_name,account_2_target_closing_date,account_2_secondary_number,account_2_group,account_2_parent_account,account_2_comment,bank_3,account_3_number,account_3_currency,account_3_type,account_3_name,account_3_target_closing_date,account_3_secondary_number,account_3_group,account_3_parent_account,account_3_comment,bank_4,account_4_number,account_4_currency,account_4_type,account_4_name,account_4_target_closing_date,account_4_secondary_number,account_4_group,account_4_parent_account,account_4_comment,rule_cf_type_name,rule_id,rule_name,rule_number,rule_owner,rule_portfolio,rule_currency,rule_currency_2,rule_instrument_group,rule_instrument,rule_counterparty,rule_transaction_sign,rule_transaction_type,rule_transaction_kind,rule_cpty_main_group,rule_counterparty_group,rule_cashflow_category,rule_cf_type_id,rule_cf_subtype_id,not_rule_id,not_rule_name,not_rule_number,not_rule_owner,not_rule_portfolio,not_rule_currency,not_rule_currency_2,not_rule_instrument_group,not_rule_instrument,not_rule_counterparty,not_rule_transaction_sign,not_rule_transaction_type,not_rule_transaction_kind,not_rule_cpty_main_group,not_rule_counterparty_group,not_rule_cashflow_category,not_rule_cf_type_id,not_rule_cf_subtype_id +Selected=account_1_comment,account_1_currency,account_1_group,account_1_name,account_1_number,account_1_parent_account,account_1_secondary_number,account_1_target_closing_date,account_1_type,account_2_comment,account_2_currency,account_2_group,account_2_name,account_2_number,account_2_parent_account,account_2_secondary_number,account_2_target_closing_date,account_2_type,account_3_comment,account_3_currency,account_3_group,account_3_name,account_3_number,account_3_parent_account,account_3_secondary_number,account_3_target_closing_date,account_3_type,account_4_comment,account_4_currency,account_4_group,account_4_name,account_4_number,account_4_parent_account,account_4_secondary_number,account_4_target_closing_date,account_4_type,active_since,active_until,bank_1,bank_2,bank_3,bank_4,client_id,client_name,client_unit_id,comment,currency_id,instruction_name,kind,not_rule_cashflow_category,not_rule_cf_subtype_id,not_rule_cf_type_id,not_rule_counterparty,not_rule_counterparty_group,not_rule_cpty_main_group,not_rule_currency,not_rule_currency_2,not_rule_id,not_rule_instrument,not_rule_instrument_group,not_rule_name,not_rule_number,not_rule_owner,not_rule_portfolio,not_rule_transaction_kind,not_rule_transaction_sign,not_rule_transaction_type,priority,rule_cashflow_category,rule_cf_subtype_id,rule_cf_type_id,rule_cf_type_name,rule_counterparty,rule_counterparty_group,rule_cpty_main_group,rule_currency,rule_currency_2,rule_id,rule_instrument,rule_instrument_group,rule_name,rule_number,rule_owner,rule_portfolio,rule_transaction_kind,rule_transaction_sign,rule_transaction_type +Grouping= +Sorting=client_id,priority,rule_id,kind +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= +client_id= +client_roles= +client_group_id= +active_only_p= +include_rule_p= +include_not_rule_p= +currency_id= + + +[Format client_id] +Name=client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_name] +Name=client_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instruction_name] +Name=instruction_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format priority] +Name=priority +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind] +Name=kind +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=active_since +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=active_until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name=client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name=comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_1] +Name=bank_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_number] +Name=account_1_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_currency] +Name=account_1_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_type] +Name=account_1_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_name] +Name=account_1_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_target_closing_date] +Name=account_1_target_closing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_secondary_number] +Name=account_1_secondary_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_group] +Name=account_1_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_parent_account] +Name=account_1_parent_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_1_comment] +Name=account_1_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_2] +Name=bank_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_number] +Name=account_2_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_currency] +Name=account_2_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_type] +Name=account_2_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_name] +Name=account_2_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_target_closing_date] +Name=account_2_target_closing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_secondary_number] +Name=account_2_secondary_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_group] +Name=account_2_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_parent_account] +Name=account_2_parent_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_2_comment] +Name=account_2_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_3] +Name=bank_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_number] +Name=account_3_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_currency] +Name=account_3_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_type] +Name=account_3_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_name] +Name=account_3_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_target_closing_date] +Name=account_3_target_closing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_secondary_number] +Name=account_3_secondary_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_group] +Name=account_3_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_parent_account] +Name=account_3_parent_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_3_comment] +Name=account_3_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_4] +Name=bank_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_number] +Name=account_4_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_currency] +Name=account_4_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_type] +Name=account_4_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_name] +Name=account_4_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_target_closing_date] +Name=account_4_target_closing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_secondary_number] +Name=account_4_secondary_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_group] +Name=account_4_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_parent_account] +Name=account_4_parent_account +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_4_comment] +Name=account_4_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_cf_type_name] +Name=rule_cf_type_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_id] +Name=rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_name] +Name=rule_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_number] +Name=rule_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_owner] +Name=rule_owner +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_portfolio] +Name=rule_portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_currency] +Name=rule_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_currency_2] +Name=rule_currency_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_instrument_group] +Name=rule_instrument_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_instrument] +Name=rule_instrument +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_counterparty] +Name=rule_counterparty +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_transaction_sign] +Name=rule_transaction_sign +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_transaction_type] +Name=rule_transaction_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_transaction_kind] +Name=rule_transaction_kind +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_cpty_main_group] +Name=rule_cpty_main_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_counterparty_group] +Name=rule_counterparty_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_cashflow_category] +Name=rule_cashflow_category +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_cf_type_id] +Name=rule_cf_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rule_cf_subtype_id] +Name=rule_cf_subtype_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_id] +Name=not_rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_name] +Name=not_rule_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_number] +Name=not_rule_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_owner] +Name=not_rule_owner +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_portfolio] +Name=not_rule_portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_currency] +Name=not_rule_currency +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_currency_2] +Name=not_rule_currency_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_instrument_group] +Name=not_rule_instrument_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_instrument] +Name=not_rule_instrument +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_counterparty] +Name=not_rule_counterparty +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_transaction_sign] +Name=not_rule_transaction_sign +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_transaction_type] +Name=not_rule_transaction_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_transaction_kind] +Name=not_rule_transaction_kind +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_cpty_main_group] +Name=not_rule_cpty_main_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_counterparty_group] +Name=not_rule_counterparty_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_cashflow_category] +Name=not_rule_cashflow_category +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_cf_type_id] +Name=not_rule_cf_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_rule_cf_subtype_id] +Name=not_rule_cf_subtype_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.yml new file mode 100644 index 0000000..0c8eea7 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CLIENTINSTRUCTION.yml @@ -0,0 +1,20 @@ + + +parameters: + Fields: + value: "client_id,client_roles,client_group_id,active_only_p,include_rule_p,include_not_rule_p,currency_id" + hidden: true + + active_only_p: + value: 0 + hidden: true + + include_rule_p: + value: 1 + hidden: true + + include_not_rule_p: + value: 1 + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.fkr new file mode 100644 index 0000000..a5f39d3 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rules +Name=Rules Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Rules Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=id like "PORT ONLY REST BLACKOUT PERIOD" and instrument_id <> "DONT_DELETE_ME" and issuer_id<>"" +MergeCellsVertically= +MergeCellsHorizontally= +Fields=sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,number,priority,portfolio_id,instrument_path_id,status,not_status,cf_leg_path_id,cf_flags,cf_not_flags,flags,name,client_id,client_main_group_id,client_group_id,user_id,currency_id,currency_2_id,instrument_id,secondary_instrument_id,package_main_type_id,package_type_id,issuer_id,issuer_main_group_id,issuer_group_id,cp_client_id,cp_client_main_group_id,cp_client_group_id,cp_portfolio_id,broker_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,contexts,not_contexts,transaction_flags,transaction_not_flags,sign_id,type_id,kind_id,state_id,cf_status,cf_not_status,cf_currency_id,cf_payment_currency_id,cf_sign,cf_type_id,cf_subtype_id,cf_category_id,cf_subcategory_id,cf_leg_group,cf_leg_id,cf_branch0_id,cf_branch1_id,cf_branch2_id,cf_branch3_id,cf_branch4_id,cf_flags_2,cf_not_flags_2,cf_kinds,cf_not_kinds,isin,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,cf_branch10_id,cf_branch11_id,cf_branch12_id,cf_branch13_id,cf_branch14_id,cf_branch15_id,cf_branch16_id,cf_branch17_id,cf_branch18_id,cf_branch19_id,cf_branch20_id,cf_branch21_id,cf_branch22_id,cf_branch23_id,cf_branch24_id,cf_branch25_id,cf_branch26_id,cf_branch27_id,cf_branch28_id,cf_branch29_id,cf_branch30_id,cf_branch31_id,cf_branch32_id,cf_branch33_id,cf_branch34_id,cf_branch35_id,cf_branch36_id,cf_branch37_id,cf_branch38_id,cf_branch39_id,cf_branch5_id,cf_branch6_id,cf_branch7_id,cf_branch8_id,cf_branch9_id,client_unit_id,cp_client_unit_id,issuer_unit_id,cf_type_name +Selected=issuer_id,name +Grouping= +Sorting=id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=12363955, 8999536, 0 +HeaderRowColors=11056316, 8999536 +DataRowColors=15724270, 0, 0 +TotalRowColors=12363955, 8999536, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=116 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=72 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format isin] +Name=ISIN +Expression=get_security_identifier(instrument_id, "ISIN") +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CPBLOCKEDISSUERS.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.fkr new file mode 100644 index 0000000..0a14e78 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.fkr @@ -0,0 +1,477 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=custody-balance +Name=Custody Balance Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Custody Balance Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,client_id,client_unit_id,custody_id,custody_unit_id,account_id,date,instrument_id,currency_id,issuer_id,issuer_unit_id,maturity_date,balance_amount,sellable_amount,balance_face_amount,sellable_face_amount,balance_units,sellable_units,comment,flags,instrument_group,isin_code,cusip_code,sedol_code +Selected=id,client_id,client_unit_id,custody_id,custody_unit_id,account_id,date,instrument_id,currency_id,issuer_id,issuer_unit_id,maturity_date,balance_amount,sellable_amount,balance_face_amount,sellable_face_amount,balance_units,sellable_units,comment,flags,instrument_group,isin_code,cusip_code,sedol_code +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=balance_period_start,balance_period_end,custody_id,custody_unit_id,client_id,client_unit_id,account_id,instrument_id,instrument_group,security_code_standard,security_code +balance_period_start= +balance_period_end= +custody_id= +custody_unit_id= +client_id= +client_unit_id= +account_id= +instrument_id= +instrument_group= +security_code_standard= +security_code= + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format client_id] +Name=CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name=CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format custody_id] +Name=CUSTODY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format custody_unit_id] +Name=CUSTODY_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format account_id] +Name=ACCOUNT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date] +Name=DATE_ +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name=ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format balance_amount] +Name=BALANCE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sellable_amount] +Name=SELLABLE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format balance_face_amount] +Name=BALANCE_FACE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sellable_face_amount] +Name=SELLABLE_FACE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format balance_units] +Name=BALANCE_UNITS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format sellable_units] +Name=SELLABLE_UNITS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format comment] +Name=COMMENT_ +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin_code] +Name=ISIN_CODE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cusip_code] +Name=CUSIP_CODE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format sedol_code] +Name=SEDOL_CODE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.yml new file mode 100644 index 0000000..c76b5c2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_CUSTODYBALANCE.yml @@ -0,0 +1,17 @@ + + +parameters: + balance_period_start: + value: "t-10d" + hidden: false + + balance_period_end: + value: "t" + hidden: false + + client_id: + value: [AGRIPARAM, BCLXLUX, BDFEPAR, BGALLIS, BITAROM, BNGRATH, DEUTFRAAM, ECBFFRA, ESPBMAD, FLORAMS, INGIHAGAM, IRCEDUB, MARKFRA, NABAVIE, NBBEBRU, NBSBBTS, SBOSLONAM, SPFBHEL] + hidden: true + + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.fkr new file mode 100644 index 0000000..d21f1e4 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.fkr @@ -0,0 +1,1235 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-instrument-bond-cashflow +Name=ECB Instrument Bond Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=ECB Instrument Bond Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,umi_id,currency_id,issuer_id,settlement_flags,amount_rounding,accrued_interest,minimum_price_denom,default_price_denom,sign_id,coupon_rate,issuer_unit_id,key_id,cashflow_type_id,subtype_id,category_id,value_date,date_basis,fixing_price,kinds,cf_flags,cf_flags_2,cf_keep_flags,reference_id,referee_id,amount,actual_currency_id,actual_fx_rate,actual_amount,subcategory_id,sign,since_when,until_when,payment_date,realize_date,origin_id,origin_value,ex_date,reference_date,spread,cashflow_group,accrual_date_basis,fixing_rate_id,fixing_period_id,fixing_period_2_id,fixing_param,fixing_subscenario_id,fixing_date,fixing_from,fixing_to,fixing_price_2,fixing_quote,fixing_type,fixing_divider,fixing_cap,fixing_floor,expression,freeze_date,effective_fixing_date +Selected=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,umi_id,currency_id,issuer_id,settlement_flags,amount_rounding,accrued_interest,minimum_price_denom,default_price_denom,sign_id,coupon_rate,issuer_unit_id,key_id,cashflow_type_id,subtype_id,category_id,value_date,date_basis,fixing_price,kinds,cf_flags,cf_flags_2,cf_keep_flags,reference_id,referee_id,amount,actual_currency_id,actual_fx_rate,actual_amount,subcategory_id,sign,since_when,until_when,payment_date,realize_date,origin_id,origin_value,ex_date,reference_date,spread,cashflow_group,accrual_date_basis,fixing_rate_id,fixing_period_id,fixing_period_2_id,fixing_param,fixing_subscenario_id,fixing_date,fixing_from,fixing_to,fixing_price_2,fixing_quote,fixing_type,fixing_divider,fixing_cap,fixing_floor,expression,freeze_date,effective_fixing_date +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=umi_id +umi_id= + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name=DOMAIN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name=NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format layout_id] +Name=LAYOUT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=ACTIVE_SINCE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=ACTIVE_UNTIL +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format buy_label] +Name=BUY_LABEL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sell_label] +Name=SELL_LABEL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format path_id] +Name=PATH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format result_type_id] +Name=RESULT_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name=TEMPLATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_id] +Name=UMI_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_flags] +Name=SETTLEMENT_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount_rounding] +Name=AMOUNT_ROUNDING +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name=ACCRUED_INTEREST +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format minimum_price_denom] +Name=MINIMUM_PRICE_DENOM +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format default_price_denom] +Name=DEFAULT_PRICE_DENOM +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name=SIGN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format coupon_rate] +Name=COUPON_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name=ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_id] +Name=KEY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_type_id] +Name=CASHFLOW_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subtype_id] +Name=SUBTYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format category_id] +Name=CATEGORY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name=VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name=DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price] +Name=FIXING_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kinds] +Name=KINDS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags] +Name=CF_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags_2] +Name=CF_FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_keep_flags] +Name=CF_KEEP_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_id] +Name=REFERENCE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format referee_id] +Name=REFEREE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_currency_id] +Name=ACTUAL_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_fx_rate] +Name=ACTUAL_FX_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_amount] +Name=ACTUAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subcategory_id] +Name=SUBCATEGORY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign] +Name=SIGN +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format since_when] +Name=SINCE_WHEN +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format until_when] +Name=UNTIL_WHEN +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=PAYMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realize_date] +Name=REALIZE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_id] +Name=ORIGIN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_value] +Name=ORIGIN_VALUE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ex_date] +Name=EX_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_date] +Name=REFERENCE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +Name=SPREAD +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_group] +Name=CASHFLOW_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrual_date_basis] +Name=ACCRUAL_DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate_id] +Name=FIXING_RATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_id] +Name=FIXING_PERIOD_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_2_id] +Name=FIXING_PERIOD_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_param] +Name=FIXING_PARAM +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_subscenario_id] +Name=FIXING_SUBSCENARIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_date] +Name=FIXING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_from] +Name=FIXING_FROM +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_to] +Name=FIXING_TO +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price_2] +Name=FIXING_PRICE_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_quote] +Name=FIXING_QUOTE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_type] +Name=FIXING_TYPE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_divider] +Name=FIXING_DIVIDER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_cap] +Name=FIXING_CAP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_floor] +Name=FIXING_FLOOR +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expression] +Name=EXPRESSION +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format freeze_date] +Name=FREEZE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_fixing_date] +Name=EFFECTIVE_FIXING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.fkr new file mode 100644 index 0000000..4d4cdbb --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.fkr @@ -0,0 +1,320 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-performance-report +Name=Performance report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Performance report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=date,market_value_start,market_value_end,cashflows,return,,portfolio,trading_portfolio,figure_date,mkt_v_st,cshf,mkt_v_end,ret +Selected=figure_date,portfolio,trading_portfolio,mkt_v_st,cshf,mkt_v_end,ret +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_start] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=132 +width_in_characters=26 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_end] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=128 +width_in_characters=25 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflows] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format return] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=83 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=90 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format crncy] +Name=Currency +Expression=as_text( left(currency , 3)) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=as_datetime( date) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format mkt_v_st] +Name=market_value_start +Expression=as_money( market_value_start) +report_total= +date_format= +time_format= +Justify=right +width=196 +width_in_characters=39 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format cshf] +Name=cashflows +Expression=if( cashflows='')(0)( cashflows) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format mkt_v_end] +Name=market_value_end +Expression=as_money(market_value_end) +report_total= +date_format= +time_format= +Justify=right +width=161 +width_in_characters=32 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format ret] +Name=return +Expression=if( return='')( 0)( return) +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.yml new file mode 100644 index 0000000..6dcaa39 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.yml @@ -0,0 +1,691 @@ + +parameters: + + FROM_DATE: + value: t-5d + hidden: false + virtual: true + replace_parameter: page + + TO_DATE: + value: t-1d + hidden: false + virtual: true + replace_parameter: page + + + + page: + hidden: true + column(data_set_id/number(12,0)): [143, 147, 610, 611, 614, 615, 138, 140, 144, 148] + value: + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + ECBUSDPERF_RE + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + ECBUSDPERF_RE + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + AATBUSDDURTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + AATBUSDCRVTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + AATBUSDCRVTOP + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + + AATBUSDDURTOP + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + + AASBUSDPERF + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + + AASBUSDPERF + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + + USDCOMPERF + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + + USDCOMPERF + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.fkr new file mode 100644 index 0000000..028bc58 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.fkr @@ -0,0 +1,336 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-performance-report +Name=Performance report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Performance report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=date,market_value_start,market_value_end,cashflows,return,currency,crncy,portfolio,trading_portfolio,figure_date,mkt_v_st,cshf,mkt_v_end,ret +Selected=figure_date,portfolio,trading_portfolio,crncy,mkt_v_st,cshf,mkt_v_end,ret +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_start] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=132 +width_in_characters=26 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_end] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=128 +width_in_characters=25 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflows] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format return] +Name=return +Expression= +report_total= +date_format= +time_format= +Justify=right +width=83 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=125 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=90 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format crncy] +Name=Currency +Expression=as_text( left(currency , 3)) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_] +Name=date +Expression=as_datetime( date ) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format mkt_v_st] +Name=market_value_start +Expression=as_money( market_value_start) +report_total= +date_format= +time_format= +Justify=right +width=196 +width_in_characters=39 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format cshf] +Name=cashflows +Expression=if( cashflows='')(0)( cashflows) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format mkt_v_end] +Name=market_value_end +Expression=as_money(market_value_end) +report_total= +date_format= +time_format= +Justify=right +width=161 +width_in_characters=32 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format ret] +Name=return +Expression=if( return='')( 0)( return) +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=as_datetime( date) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.yml new file mode 100644 index 0000000..cba3cbf --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.yml @@ -0,0 +1,2008 @@ + +parameters: + + FROM_DATE: + value: t-5d + hidden: false + virtual: true + replace_parameter: page + + TO_DATE: + value: t-1d + hidden: false + virtual: true + replace_parameter: page + + + + page: + hidden: true + column(data_set_id/number(12,0)): [122, 124, 126, 128, 132, 136, 218, 259, 418, 493, 519, 547, 551, 556, 131, 135, 340, 494, 608, 609, 612, 613, 616, 617, 637, 638, 641, 642] + value: + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + EURCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + EURCOM + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + AASBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + AASBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + JPYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + JPYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPCBPPALL + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPPSPPALL + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPCSPP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + AASBCNY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + CNYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPPEPPGOVNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPPEPPCORPNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + currency + no + no + + + + MPPEPPCOVNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + ECBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + ECBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + ACC-HTM-ABSPP-AVG + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + ECBCNY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBCNYCRVTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBCNYDURTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBJPYCRVTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBJPYDURTOP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBJPYCRVTOP + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AATBJPYDURTOP + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + OFBISIP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + AASBEURBISIP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + OFEURINV + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + currency + no + no + + + + SBEURINV + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.fkr new file mode 100644 index 0000000..8a6c912 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.fkr @@ -0,0 +1,304 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-performance-report +Name=Performance report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Performance report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=date,market_value_start,market_value_end,cashflows,return,instrument,portfolio,trading_portfolio,figure_date,mkt_v_st,cshf,mkt_v_end,ret +Selected=figure_date,portfolio,trading_portfolio,instrument,mkt_v_st,cshf,mkt_v_end,ret +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_start] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=132 +width_in_characters=26 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_end] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=128 +width_in_characters=25 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflows] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format return] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=83 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=as_datetime(date) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=74 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format mkt_v_st] +Name=market_value_start +Expression=as_money( market_value_start) +report_total= +date_format= +time_format= +Justify=right +width=196 +width_in_characters=39 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format cshf] +Name=cashflows +Expression=if( cashflows='')(0)( cashflows) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format mkt_v_end] +Name=market_value_end +Expression=as_money(market_value_end) +report_total= +date_format= +time_format= +Justify=right +width=161 +width_in_characters=32 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format ret] +Name=return +Expression=if( return='')( 0)( return) +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.yml new file mode 100644 index 0000000..3830acc --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.yml @@ -0,0 +1,453 @@ + +parameters: + + FROM_DATE: + value: t-5d + hidden: false + virtual: true + replace_parameter: page + + TO_DATE: + value: t-1d + hidden: false + virtual: true + replace_parameter: page + + + + page: + hidden: true + column(data_set_id/number(12,0)): [142, 146, 137, 139, 141, 145] + value: + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + ECBUSDPERF_RE + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + ECBUSDPERF_RE + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + AASBUSDPERF + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + AASBUSDPERF + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + USDCOMPERF + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + + USDCOMPERF + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.fkr new file mode 100644 index 0000000..515772a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.fkr @@ -0,0 +1,335 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-performance-report +Name=Performance report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Performance report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=date,figure_date,market_value_start,market_value_end,cashflows,return,trading_portfolio,portfolio,currency,crncy,instrument,figure_date,mkt_v_st,cshf,mkt_v_end,ret +Selected=figure_date,trading_portfolio,portfolio,instrument,crncy,mkt_v_st,cshf,mkt_v_end,ret +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_start] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=132 +width_in_characters=26 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value_end] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=128 +width_in_characters=25 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflows] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format return] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=83 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + +[Format portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=90 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format crncy] +Name=Currency +Expression=as_text( left(currency , 3)) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=as_datetime( date) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=74 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format mkt_v_st] +Name=market_value_start +Expression=as_money( market_value_start) +report_total= +date_format= +time_format= +Justify=right +width=196 +width_in_characters=39 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format cshf] +Name=cashflows +Expression=if( cashflows='')(0)( cashflows) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format mkt_v_end] +Name=market_value_end +Expression=as_money(market_value_end) +report_total= +date_format= +time_format= +Justify=right +width=161 +width_in_characters=32 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format ret] +Name=return +Expression=if( return='')( 0)( return) +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_portfolio] +Name=portfolio +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.yml new file mode 100644 index 0000000..21b44a2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.yml @@ -0,0 +1,1701 @@ + +parameters: + + FROM_DATE: + value: t-5d + hidden: false + virtual: true + replace_parameter: page + + TO_DATE: + value: t-1d + hidden: false + virtual: true + replace_parameter: page + + + + page: + hidden: true + column(data_set_id/number(12,0)): [130, 134, 339, 495, 639, 640, 643, 644, 121, 123, 125, 127, 129, 133, 217, 258, 419, 492, 518, 546, 550, 557] + value: + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + ECBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + ECBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + ACC-HTM-ABSPP-AVG + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + ECBCNY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + OFBISIP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + AASBEURBISIP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + OFEURINV + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + SBEURINV + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + EURCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + EURCOM + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + AASBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + AASBJPY + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + JPYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + JPYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + value_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPCBPPALL + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPPSPPALL + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPCSPP + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + AASBCNY + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + CNYCOM + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPPEPPGOVNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPPEPPCORPNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + + - | + + + FREEZE + + FROM_DATE + TO_DATE + + + key_figure + no + no + + + + + date + no + no + + + trading_portfolio + no + no + + + instrument + no + no + + + currency + no + no + + + + MPPEPPCOVNCB + + + + + + market_value_start + market_value_end + cashflows + return + + trade_date + 0.00 + no + no + no + no + default + 0.00 + periodic + day + 30 + + + + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.fkr new file mode 100644 index 0000000..3ceb9dc --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.fkr @@ -0,0 +1,590 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_liquidity_type,by_position_date,figure_date,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,operation,instrument_group,ispositioninunits +Selected=operation,figure_date,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(prev(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.yml new file mode 100644 index 0000000..38fa42b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.yml @@ -0,0 +1,358 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [468, 481, 482, 483, 484, 570, 571, 635, 636, 73, 74, 75, 76, 90, 91, 92, 444, 445, 447, 449, 450] + value: + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.fkr new file mode 100644 index 0000000..ac319bd --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.fkr @@ -0,0 +1,608 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date, figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_portfolio,by_maturity_period,by_instrument,operation,instrument_group,ispositioninunits +Selected=figure_date,operation,by_portfolio,by_maturity_period,by_instrument,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=88 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=118 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_portfolio_compare] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.yml new file mode 100644 index 0000000..d1d6bf0 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.yml @@ -0,0 +1,306 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 57, 58, 59, 60, 465, 466] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.fkr new file mode 100644 index 0000000..ddddcec --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.fkr @@ -0,0 +1,623 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_instrument,by_currency,by_portfolio,by_transaction_counterparty,operation,instrument_group,ispositioninunits +Selected=operation,figure_date,by_instrument,by_transaction_counterparty,by_portfolio,by_currency,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=88 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=144 +width_in_characters=28 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=118 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_portfolio_compare] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_transaction_counterparty] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=80 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.yml new file mode 100644 index 0000000..b01909a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.yml @@ -0,0 +1,173 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + position: + hidden: true + column(data_set_id/number(12,0)): [656, 658, 659, 664, 665, 666, 667] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.fkr new file mode 100644 index 0000000..11d8fee --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.fkr @@ -0,0 +1,607 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_instrument,operation,instrument_group,ispositioninunits +Selected=figure_date,operation,by_instrument,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=118 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_portfolio_compare] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=89 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.yml new file mode 100644 index 0000000..ba52a9e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.yml @@ -0,0 +1,244 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [11, 358, 439, 440, 441, 442, 459, 514, 568, 569, 630, 631] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.fkr new file mode 100644 index 0000000..cf3bcbd --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.fkr @@ -0,0 +1,558 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_portfolio,by_instrument,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,operation,instrument_group,ispositioninunits +Selected=figure_date,operation,by_portfolio,by_instrument,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.yml new file mode 100644 index 0000000..c92da13 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.yml @@ -0,0 +1,529 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + position: + hidden: true + column(data_set_id/number(12,0)): [2, 7, 8, 9, 10, 14, 15, 16, 17, 18, 19, 20, 22, 23, 297, 298, 420, 458, 460, 474, 475, 476, 477, 520, 521, 548, 549, 552, 628, 629, 647] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.fkr new file mode 100644 index 0000000..8f0b15e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.fkr @@ -0,0 +1,607 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_portfolio_compare,operation,instrument_group,ispositioninunits +Selected=figure_date,operation,by_portfolio_compare,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=100 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_portfolio_compare] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=73 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.yml new file mode 100644 index 0000000..d3a9625 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.yml @@ -0,0 +1,258 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + position: + hidden: true + column(data_set_id/number(12,0)): [104, 105, 106, 107, 108, 111, 112, 472, 473, 516, 517, 527, 632] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.fkr new file mode 100644 index 0000000..6e61ff2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.fkr @@ -0,0 +1,558 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_maturity_period,operation,ispositioninunits +Selected=operation,figure_date,by_maturity_period,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name=accrued_book_value__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=121 +width_in_characters=24 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=93 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.yml new file mode 100644 index 0000000..0896d02 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.yml @@ -0,0 +1,110 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [462, 515, 528] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.fkr new file mode 100644 index 0000000..843e948 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.fkr @@ -0,0 +1,575 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_portfolio,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_maturity_period,operation,instrument_group,ispositioninunits +Selected=operation,figure_date,by_portfolio,by_maturity_period,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.yml new file mode 100644 index 0000000..dabbc17 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.yml @@ -0,0 +1,226 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [24, 26, 28, 30, 32, 37, 39, 461, 478, 479, 522] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.fkr new file mode 100644 index 0000000..4137671 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.fkr @@ -0,0 +1,576 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_portfolio,by_liquidity_type,by_position_date,figure_date,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,operation,instrument_group,ispositioninunits +Selected=operation,by_portfolio,figure_date,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(prev(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.yml new file mode 100644 index 0000000..91d288d --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.yml @@ -0,0 +1,799 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [68, 69, 70, 71, 72, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 96, 97, 98, 99, 100, 101, 102, 103, 421, 446, 448, 467, 469, 470, 471, 480, 485, 486, 487, 488, 489, 490, 491, 523, 524, 525, 526, 553, 554, 555, 633, 634, 646] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.fkr new file mode 100644 index 0000000..cbed8ad --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.fkr @@ -0,0 +1,592 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_portfolio,by_liquidity_type,by_position_date,figure_date,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_currency,operation,instrument_group,ispositioninunits +Selected=operation,by_currency,by_portfolio,figure_date,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=146 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=104 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=80 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(prev(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.yml new file mode 100644 index 0000000..cbf502a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.yml @@ -0,0 +1,80 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [277] + value: + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.fkr new file mode 100644 index 0000000..450985d --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.fkr @@ -0,0 +1,591 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-position-report +Name=Position report Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Position report Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=by_position_date,figure_date,by_liquidity_type,nominal_amount__normal,amount__normal,units__normal,market_value__normal,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk_ir__normal,value_at_risk_fx__normal,value_at_risk__normal,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,value_at_risk_ir__delta_reference_vector,accrued_interest__normal,accrued_book_value__normal,accrued_profit__normal,maturity__normal,effective_convexity__normal,commited_mv,commited_mv_total,accrued_book_value_interest,nominal_amount,by_maturity_period,by_instrument,operation,instrument_group,ispositioninunits +Selected=figure_date,operation,by_maturity_period,by_instrument,accrued_book_value__normal,accrued_book_value_interest,accrued_interest__normal,accrued_profit__normal,nominal_amount,market_value__normal,commited_mv_total,underlying_market_value__normal,ir_exposure_1__normal,modified_duration__normal,effective_duration__contribution_to_total,value_at_risk__normal,value_at_risk_fx__normal,value_at_risk_ir__normal,maturity__normal,value_at_risk_ir__delta_reference_vector,value_at_risk_ir__delta_reference_vector_relative_to_total_mv,amount__normal,effective_convexity__normal +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format by_portfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_liquidity_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_market_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ir_exposure_1__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration__normal] +Name=modified_duration__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=86 +width_in_characters=17 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_duration__contribution_to_total] +Name=effective_duration__contribution_to_total +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_fx__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector_relative_to_total_mv] +Name=value_at_risk_ir__delta_reference_vector_relative_to_total_mv +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=6 +max_precision=6 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_at_risk_ir__delta_reference_vector] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=120 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit__normal] +Name=accrued_profit__normal +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_convexity__normal] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format commited_mv] +Name=Commited mv +Expression=if ( by_liquidity_type = 'committed')( market_value__normal) (0) +report_total= +date_format= +time_format= +Justify=right +width=79 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format commited_mv_total] +Name=commited_mv_total +Expression=prev( @commited_mv) +report_total= +date_format= +time_format= +Justify=right +width=151 +width_in_characters=30 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format accrued_book_value_interest] +Name=accrued_book_value_interest +Expression=if ( accrued_interest__normal!='NaN') ( if (accrued_book_value__normal != 'NaN') ( accrued_interest__normal + accrued_book_value__normal ) ( accrued_interest__normal ) ) (accrued_book_value__normal) +report_total= +date_format= +time_format= +Justify=right +width=173 +width_in_characters=34 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression=if( by_instrument!="")(get_field("UMPathFlat", get_field("UMI", by_instrument, "path_id"), 0, "upath"))("") +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ispositioninunits] +Name=isPositionInUnits +Expression=if( instrument_group =' /EQ/FUND/EU/BOND_FUND' or instrument_group =' /EQ/DRVT/EU/EIFT' or instrument_group =' /EQ/FUND/EU/ETF' )(1 )( 0 ) +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format nominal_amount] +Name=nominal_amount +Expression=if ( ispositioninunits=1 and units__normal != 'NaN' and units__normal != 0 ) ( units__normal ) ( nominal_amount__normal ) +report_total= +date_format= +time_format= +Justify=right +width=133 +width_in_characters=26 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_maturity_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=100 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format by_currency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression=if( by_liquidity_type!="total")("IGNORE" )("") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + +[Format by_position_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=^Z +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name=figure_date +Expression=if(by_position_date <> "") (as_datetime(by_position_date)) (as_datetime(first(@by_position_date))) +report_total= +date_format= +time_format= +Justify=right +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.yml new file mode 100644 index 0000000..0e78d52 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.yml @@ -0,0 +1,125 @@ + +parameters: + + VALUATION_DATE: + value: + - select("dual", "to_char(sysdate - 5, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 4, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 3, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 2, 'YYYY-MM-DD')") + - select("dual", "to_char(sysdate - 1, 'YYYY-MM-DD')") + hidden: false + virtual: true + replace_parameter: position + + + view: + hidden: true + value: | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]]> + + + position: + hidden: true + column(data_set_id/number(12,0)): [51, 52, 463, 464] + value: + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + + - | + + + + + + + + + + + + + ]]> + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.fkr new file mode 100644 index 0000000..5e5a494 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.fkr @@ -0,0 +1,362 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=effective-permission-report +Name=Effective Permissions Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Effective Permissions Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,name,admin_center_id,time_zone_id,menu_name,web_home_group_id,comment,flags,access_profile_id,access_profile_name,access_profile_type_id,access_profile_type_name,entity_id,permission_id,resource_rule,active_from,active_to +Selected=id,name,admin_center_id,time_zone_id,menu_name,web_home_group_id,comment,flags,access_profile_id,access_profile_name,access_profile_type_id,access_profile_type_name,entity_id,permission_id,resource_rule,active_from,active_to +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=16504743, 7956305, 7956305 +HeaderRowColors=13026239, 7956305 +DataRowColors=16381424, 0, 0 +TotalRowColors=16504743, 7956305, 7956305 +DecimalSeparator=. +ThousandsSeparator=, +DigitGrouping=03 + + +[Parameters] +Fields=user_id,admin_center_id,eff_date,exclude_p,access_profile_type_id,expand_object_hierarchy_p +user_id= +admin_center_id= +eff_date=today +exclude_p=0 +access_profile_type_id= +expand_object_hierarchy_p=1 + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format admin_center_id] +Name=admin_center_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name=time_zone_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format menu_name] +Name=menu_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format web_home_group_id] +Name=web_home_group_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format comment] +Name=comment +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_id] +Name=access_profile_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_name] +Name=access_profile_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_type_id] +Name=access_profile_type_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_type_name] +Name=access_profile_type_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format entity_id] +Name=entity_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format permission_id] +Name=permission_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format resource_rule] +Name=resource_rule +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_from] +Name=active_from +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_to] +Name=active_to +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.yml new file mode 100644 index 0000000..9165e17 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEPERMISSIONS.yml @@ -0,0 +1,8 @@ + + +parameters: + + user_id: + value: + hidden: false + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.b64 b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.b64 new file mode 100644 index 0000000..80d43bb --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.b64 @@ -0,0 +1,2 @@ +IyBlbmNvZGluZzogVVRGLTgKW2ZrIHJlcG9ydF0KVmVyc2lvbj0yLjMKVmVyc2lvbldhcm5pbmc9CgoKW01haW5dClR5cGU9ZWZmZWN0aXZlLXJvbGUtcHJvZmlsZS1yZXBvcnQKTmFtZT1FZmZlY3RpdmUgUm9sZSBQcm9maWxlIFJlcG9ydApFeGNlbFdvcmtib29rPQpIZWFkZXJMZWZ0PVtcbl0lcGFnZWJyZWFrCkhlYWRlckNlbnRlcj1FZmZlY3RpdmUgUm9sZSBQcm9maWxlIFJlcG9ydApIZWFkZXJSaWdodD0lZGF0ZXRpbWVbXG5dUGFnZSAlcGFnZW51bS8lcGFnZWNvdW50CkZvb3RlckxlZnQ9CkZvb3RlckNlbnRlcj0KRm9vdGVyUmlnaHQ9CkNyaXRlcmlhPQpUb3RhbHNPbmx5PTAKTWVyZ2VDZWxsc1ZlcnRpY2FsbHk9Ck1lcmdlQ2VsbHNIb3Jpem9udGFsbHk9CkZpZWxkcz1pZCxhZG1pbl9jZW50ZXJfaWQsYWNjZXNzX3Byb2ZpbGVfdHlwZV9pZCxhY2Nlc3NfcHJvZmlsZV90eXBlX25hbWUsbmFtZSx0aW1lX3pvbmVfaWQsbWVudV9uYW1lLHdlYl9ob21lX2dyb3VwX2lkLGNvbW1lbnQsZmxhZ3MsYWNjZXNzX3JvbGVfaWQsYWNjZXNzX3JvbGVfbmFtZSxkaXJlY3RfcCxhY3RpdmVfZnJvbSxhY3RpdmVfdG8sYWNjZXNzX3Byb2ZpbGVfaWQsYWNjZXNzX3Byb2ZpbGVfbmFtZQpTZWxlY3RlZD1pZCxuYW1lLGFkbWluX2NlbnRlcl9pZCx0aW1lX3pvbmVfaWQsbWVudV9uYW1lLHdlYl9ob21lX2dyb3VwX2lkLGZsYWdzLGFjY2Vzc19yb2xlX2lkLGFjY2Vzc19yb2xlX25hbWUsZGlyZWN0X3AsYWN0aXZlX2Zyb20sYWN0aXZlX3RvLGFjY2Vzc19wcm9maWxlX2lkLGFjY2Vzc19wcm9maWxlX25hbWUsYWNjZXNzX3Byb2ZpbGVfdHlwZV9pZCxhY2Nlc3NfcHJvZmlsZV90eXBlX25hbWUKR3JvdXBpbmc9ClNvcnRpbmc9ClNvcnREZXNjZW5kaW5nPQpHcm91cFNvcnRMZXZlbD0wCkdyb3VwU29ydENvbHVtbnM9Ckdyb3VwU29ydERlc2NlbmRpbmc9ClBhZ2VicmVhaz0wClpvb21QZXJjZW50PTEwMApGb250R3JpZEhlYWRpbmc9VGFob21hLC0xMSw3MDAsCkZvbnQwPVRhaG9tYSwtMTEsNDAwLGkKRm9udDE9VGFob21hLC0xMSw0MDAsCkZvbnQyPVRhaG9tYSwtMTEsNzAwLApGb250R3JpZEhlYWRpbmdEcmlsbERvd249VGFob21hLC0xMSw3MDAsCkZvbnQxRHJpbGxEb3duPVRhaG9tYSwtMTEsNDAwLHUKRm9udDJEcmlsbERvd249VGFob21hLC0xMSw3MDAsCkdyaWRIZWFkaW5nQ29sb3JzPTE0MDA1OTI3LCA2NDQxMDE4LCA2NDQxMDE4CkhlYWRlclJvd0NvbG9ycz0xMzgxMjE1MiwgNjQ0MTAxOApEYXRhUm93Q29sb3JzPTE2MTgyNzYxLCAwLCAwClRvdGFsUm93Q29sb3JzPTE0MDA1OTI3LCA2NDQxMDE4LCA2NDQxMDE4CkRlY2ltYWxTZXBhcmF0b3I9LgpUaG91c2FuZHNTZXBhcmF0b3I9LAoKCltQYXJhbWV0ZXJzXQpGaWVsZHM9dXNlcl9pZCxhZG1pbl9jZW50ZXJfaWQsZWZmX2RhdGUKdXNlcl9pZD0KYWRtaW5fY2VudGVyX2lkPQplZmZfZGF0ZT0KCgpbRm9ybWF0IGlkXQpOYW1lPWlkCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCBhZG1pbl9jZW50ZXJfaWRdCk5hbWU9YWRtaW5fY2VudGVyX2lkCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCBhY2Nlc3NfcHJvZmlsZV90eXBlX2lkXQpOYW1lPWFjY2Vzc19wcm9maWxlX3R5cGVfaWQKRXhwcmVzc2lvbj0KcmVwb3J0X3RvdGFsPQpkYXRlX2Zvcm1hdD0KdGltZV9mb3JtYXQ9Ckp1c3RpZnk9bGVmdAp3aWR0aD0xMDcKd2lkdGhfaW5fY2hhcmFjdGVycz0yMQpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCBhY2Nlc3NfcHJvZmlsZV90eXBlX25hbWVdCk5hbWU9YWNjZXNzX3Byb2ZpbGVfdHlwZV9uYW1lCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9MTY0CndpZHRoX2luX2NoYXJhY3RlcnM9MzIKZGl2aWRlcj0xCnByZWNpc2lvbj0wCm1heF9wcmVjaXNpb249MApzaG93X3plcm89MApOb051bWJlckZvcm1hdHRpbmc9MAoKCltGb3JtYXQgbmFtZV0KTmFtZT1uYW1lCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCB0aW1lX3pvbmVfaWRdCk5hbWU9dGltZV96b25lX2lkCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCBtZW51X25hbWVdCk5hbWU9bWVudV9uYW1lCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW0Zvcm1hdCB3ZWJfaG9tZV9ncm91cF9pZF0KTmFtZT13ZWJfaG9tZV9ncm91cF9pZApFeHByZXNzaW9uPQpyZXBvcnRfdG90YWw9CmRhdGVfZm9ybWF0PQp0aW1lX2Zvcm1hdD0KSnVzdGlmeT1sZWZ0CndpZHRoPTYwCndpZHRoX2luX2NoYXJhY3RlcnM9MTIKZGl2aWRlcj0xCnByZWNpc2lvbj0wCm1heF9wcmVjaXNpb249MApzaG93X3plcm89MApOb051bWJlckZvcm1hdHRpbmc9MAoKCltGb3JtYXQgY29tbWVudF0KTmFtZT1jb21tZW50XwpFeHByZXNzaW9uPQpyZXBvcnRfdG90YWw9CmRhdGVfZm9ybWF0PQp0aW1lX2Zvcm1hdD0KSnVzdGlmeT1sZWZ0CndpZHRoPTE4Ngp3aWR0aF9pbl9jaGFyYWN0ZXJzPTM3CmRpdmlkZXI9MQpwcmVjaXNpb249MAptYXhfcHJlY2lzaW9uPTAKc2hvd196ZXJvPTAKTm9OdW1iZXJGb3JtYXR0aW5nPTAKCgpbRm9ybWF0IGZsYWdzXQpOYW1lPWZsYWdzCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PXJpZ2h0CndpZHRoPTYwCndpZHRoX2luX2NoYXJhY3RlcnM9MTIKZGl2aWRlcj0xCnByZWNpc2lvbj0wCm1heF9wcmVjaXNpb249MApzaG93X3plcm89MApOb051bWJlckZvcm1hdHRpbmc9MAoKCltGb3JtYXQgYWNjZXNzX3JvbGVfaWRdCk5hbWU9YWNjZXNzX3JvbGVfaWQKRXhwcmVzc2lvbj0KcmVwb3J0X3RvdGFsPQpkYXRlX2Zvcm1hdD0KdGltZV9mb3JtYXQ9Ckp1c3RpZnk9bGVmdAp3aWR0aD02MAp3aWR0aF9pbl9jaGFyYWN0ZXJzPTEyCmRpdmlkZXI9MQpwcmVjaXNpb249MAptYXhfcHJlY2lzaW9uPTAKc2hvd196ZXJvPTAKTm9OdW1iZXJGb3JtYXR0aW5nPTAKCgpbRm9ybWF0IGFjY2Vzc19yb2xlX25hbWVdCk5hbWU9YWNjZXNzX3JvbGVfbmFtZQpFeHByZXNzaW9uPQpyZXBvcnRfdG90YWw9CmRhdGVfZm9ybWF0PQp0aW1lX2Zvcm1hdD0KSnVzdGlmeT1sZWZ0CndpZHRoPTE2OQp3aWR0aF9pbl9jaGFyYWN0ZXJzPTMzCmRpdmlkZXI9MQpwcmVjaXNpb249MAptYXhfcHJlY2lzaW9uPTAKc2hvd196ZXJvPTAKTm9OdW1iZXJGb3JtYXR0aW5nPTAKCgpbRm9ybWF0IGRpcmVjdF9wXQpOYW1lPWRpcmVjdF9wCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PXJpZ2h0CndpZHRoPTYwCndpZHRoX2luX2NoYXJhY3RlcnM9MTIKZGl2aWRlcj0xCnByZWNpc2lvbj0wCm1heF9wcmVjaXNpb249MApzaG93X3plcm89MApOb051bWJlckZvcm1hdHRpbmc9MAoKCltGb3JtYXQgYWN0aXZlX2Zyb21dCk5hbWU9YWN0aXZlX2Zyb20KRXhwcmVzc2lvbj0KcmVwb3J0X3RvdGFsPQpkYXRlX2Zvcm1hdD1kZC9NTS95eXl5CnRpbWVfZm9ybWF0PUhIOm1tOnNzCkp1c3RpZnk9bGVmdAp3aWR0aD02MAp3aWR0aF9pbl9jaGFyYWN0ZXJzPTEyCmRpdmlkZXI9MQpwcmVjaXNpb249MAptYXhfcHJlY2lzaW9uPTAKc2hvd196ZXJvPTAKTm9OdW1iZXJGb3JtYXR0aW5nPTAKCgpbRm9ybWF0IGFjdGl2ZV90b10KTmFtZT1hY3RpdmVfdG8KRXhwcmVzc2lvbj0KcmVwb3J0X3RvdGFsPQpkYXRlX2Zvcm1hdD1kZC9NTS95eXl5CnRpbWVfZm9ybWF0PUhIOm1tOnNzCkp1c3RpZnk9bGVmdAp3aWR0aD02MAp3aWR0aF9pbl9jaGFyYWN0ZXJzPTEyCmRpdmlkZXI9MQpwcmVjaXNpb249MAptYXhfcHJlY2lzaW9uPTAKc2hvd196ZXJvPTAKTm9OdW1iZXJGb3JtYXR0aW5nPTAKCgpbRm9ybWF0IGFjY2Vzc19wcm9maWxlX2lkXQpOYW1lPWFjY2Vzc19wcm9maWxlX2lkCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9MTc2CndpZHRoX2luX2NoYXJhY3RlcnM9MzUKZGl2aWRlcj0xCnByZWNpc2lvbj0wCm1heF9wcmVjaXNpb249MApzaG93X3plcm89MApOb051bWJlckZvcm1hdHRpbmc9MAoKCltGb3JtYXQgYWNjZXNzX3Byb2ZpbGVfbmFtZV0KTmFtZT1hY2Nlc3NfcHJvZmlsZV9uYW1lCkV4cHJlc3Npb249CnJlcG9ydF90b3RhbD0KZGF0ZV9mb3JtYXQ9CnRpbWVfZm9ybWF0PQpKdXN0aWZ5PWxlZnQKd2lkdGg9NjAKd2lkdGhfaW5fY2hhcmFjdGVycz0xMgpkaXZpZGVyPTEKcHJlY2lzaW9uPTAKbWF4X3ByZWNpc2lvbj0wCnNob3dfemVybz0wCk5vTnVtYmVyRm9ybWF0dGluZz0wCgoKW1BhcGVyXQpUeXBlPTkKV2lkdGg9Mjk3MDAKSGVpZ2h0PTIxMDAwCkxlZnQ9MjAwMApSaWdodD0yMDAwClRvcD0yMDAwCkJvdHRvbT0yMDAwCgoKW0V4cG9ydF0KRXhwb3J0Q292ZXJQYWdlPTEKRXhwb3J0UmVwb3J0TmFtZT0wCkV4cG9ydEhlYWRlclJvd3M9MApFeHBvcnRQbGFpblJvd3M9MQpFeHBvcnRGb290ZXJSb3dzPTEKRXhwb3J0RW1wdHlSZXBvcnQ9MQoKCltQcmludF0KUHJpbnRWZXJ0R3JpZExpbmVzPTEKUHJpbnRIb3J6R3JpZExpbmVzPTEKUHJpbnRGcmFtZT0xClByaW50Um93TnVtYmVycz0xClByaW50SW5Db2xvcj0wClByaW50Q2VudGVySG9yaXpvbnRhbGx5PTAKUHJpbnRDZW50ZXJWZXJ0aWNhbGx5PTAKUHJpbnRUcmFuc3BhcmVudEJhY2tncm91bmQ9MApQcmludENvdmVyUGFnZT0xClByaW50SGVhZGVyUm93cz0xClByaW50UGxhaW5Sb3dzPTEKUHJpbnRGb290ZXJSb3dzPTEKCg== + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.fkr new file mode 100644 index 0000000..8e0fd9a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.fkr @@ -0,0 +1,357 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=effective-role-profile-report +Name=Effective Role Profile Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Effective Role Profile Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,admin_center_id,access_profile_type_id,access_profile_type_name,name,time_zone_id,menu_name,web_home_group_id,comment,flags,access_role_id,access_role_name,direct_p,active_from,active_to,access_profile_id,access_profile_name +Selected=id,name,admin_center_id,time_zone_id,menu_name,web_home_group_id,flags,access_role_id,access_role_name,direct_p,active_from,active_to,access_profile_id,access_profile_name,access_profile_type_id,access_profile_type_name +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=user_id,admin_center_id,eff_date +user_id= +admin_center_id= +eff_date= + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format admin_center_id] +Name=admin_center_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_type_id] +Name=access_profile_type_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=107 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_type_name] +Name=access_profile_type_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=164 +width_in_characters=32 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name=time_zone_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format menu_name] +Name=menu_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format web_home_group_id] +Name=web_home_group_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format comment] +Name=comment_ +Expression= +report_total= +date_format= +time_format= +Justify=left +width=186 +width_in_characters=37 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_role_id] +Name=access_role_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_role_name] +Name=access_role_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=169 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format direct_p] +Name=direct_p +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_from] +Name=active_from +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_to] +Name=active_to +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_id] +Name=access_profile_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=176 +width_in_characters=35 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format access_profile_name] +Name=access_profile_name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_EFFECTIVEROLEPROFILE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.fkr new file mode 100644 index 0000000..f612fd0 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.fkr @@ -0,0 +1,1270 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=finmessage-log +Name=FIN Message Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=FIN Message Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_date,log_user,log_action,log_type,log_serial,id,sender_swift_code,receiver_swift_code,type,state_id,flags,log_caller,number,settlement_id,operation,reference,related_reference,status_description,sent_date_time,date,fixing_id,input_time,message_input_reference,output_time,release_batch_id,source_system_id,system_id,target_system,call_movement_id,confirmation_reference,confirmation_time,interface_info,leg_id,service_type_id,session_id,session_sequence,uetr,uti,version_id,warning,sending,sent,urgent,failed,stp,delivery,portfolio_id,instrument_id,owner_id,owner_unit_id,cp_client_id,cp_client_unit_id,currency_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19 +Selected=call_movement_id,confirmation_reference,confirmation_time,cp_client_id,cp_client_unit_id,currency_id,date,delivery,failed,fixing_id,flags,id,input_time,instrument_id,interface_info,leg_id,log_action,log_caller,log_date,log_serial,log_type,log_user,message_input_reference,number,operation,output_time,owner_id,owner_unit_id,param_0,param_1,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,portfolio_id,receiver_swift_code,reference,related_reference,release_batch_id,sender_swift_code,sending,sent,sent_date_time,service_type_id,session_id,session_sequence,settlement_id,source_system_id,state_id,status_description,stp,system_id,target_system,type,uetr,urgent,uti,version_id,warning +Grouping= +Sorting=id,log_serial +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from,to,id,number,portfolio_id,instrument_type_id,instrument_group_id,instrument_id,user_id,cp_client_id,currency_id,currency_class_id,currency_group_id,settlement_id,sender_swift_code,receiver_swift_code,type,operation,reference,related_reference +from= +to= +id= +number= +portfolio_id= +instrument_type_id= +instrument_group_id= +instrument_id= +user_id= +cp_client_id= +currency_id= +currency_class_id= +currency_group_id= +settlement_id= +sender_swift_code= +receiver_swift_code= +type= +operation= +reference= +related_reference= + + +[Format log_date] +Name=log_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=150 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format log_user] +Name=log_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_action] +Name=log_action +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_type] +Name=log_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_serial] +Name=log_serial +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sender_swift_code] +Name=sender_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format receiver_swift_code] +Name=receiver_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type] +Name=type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=state_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_caller] +Name=log_caller +Expression= +report_total= +date_format= +time_format= +Justify= +width=98 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_id] +Name=settlement_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format operation] +Name=operation +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference] +Name=reference +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format related_reference] +Name=related_reference +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status_description] +Name=status_description +Expression= +report_total= +date_format= +time_format= +Justify= +width=157 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sent_date_time] +Name=sent_date_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format fixing_id] +Name=fixing_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format input_time] +Name=input_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format message_input_reference] +Name=message_input_reference +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format output_time] +Name=output_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=142 +width_in_characters=28 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format release_batch_id] +Name=release_batch_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format source_system_id] +Name=source_system_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format system_id] +Name=system_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format target_system] +Name=target_system +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format call_movement_id] +Name=call_movement_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format confirmation_reference] +Name=confirmation_reference +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format confirmation_time] +Name=confirmation_time +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format interface_info] +Name=interface_info +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_id] +Name=leg_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format service_type_id] +Name=service_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format session_id] +Name=session_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format session_sequence] +Name=session_sequence +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format uetr] +Name=uetr +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format uti] +Name=uti +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format version_id] +Name=version_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format warning] +Name=warning +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sending] +Name=sending +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sent] +Name=sent +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format urgent] +Name=urgent +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format failed] +Name=failed +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format stp] +Name=stp +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery] +Name=delivery +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=portfolio_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=instrument_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name=owner_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name=owner_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=cp_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=cp_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=param_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=param_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=param_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=param_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=param_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=param_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=param_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=param_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=param_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=param_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=param_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=param_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=param_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=param_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=param_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=param_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=param_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=param_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=param_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=param_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.yml new file mode 100644 index 0000000..c37ce8d --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_FINMESSAGELOG.yml @@ -0,0 +1,12 @@ + + +parameters: + from: + value: "t-1d" + hidden: false + + to: + value: "t" + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.fkr new file mode 100644 index 0000000..d357587 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.fkr @@ -0,0 +1,279 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=history-log +Name=History Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=History Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=object_id,action,user_id,date,key_1_string,field_name,key_2_string,key_3_string,key_4_string,from_value,to_value,domain_id +Selected=object_id,action,user_id,date,key_1_string,field_name,key_2_string,key_3_string,key_4_string,from_value,to_value,domain_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from,to,user_id,action,object_id +from= +to= +user_id= +action= +object_id= + + +[Format object_id] +Name=OBJECT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format action] +Name=ACTION +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format user_id] +Name=USER_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date] +Name=DATE_ +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=136 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format key_1_string] +Name=KEY_1_STRING +Expression= +report_total= +date_format= +time_format= +Justify=left +width=208 +width_in_characters=41 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format field_name] +Name=FIELD_NAME +Expression= +report_total= +date_format= +time_format= +Justify=left +width=74 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format key_2_string] +Name=KEY_2_STRING +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format key_3_string] +Name=KEY_3_STRING +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format key_4_string] +Name=KEY_4_STRING +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format from_value] +Name=FROM_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=155 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format to_value] +Name=TO_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format domain_id] +Name=DOMAIN_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.yml new file mode 100644 index 0000000..e924604 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_HISTORY_LOG.yml @@ -0,0 +1,12 @@ + + +parameters: + from: + value: "t-1d" + hidden: false + + to: + value: "t-1d" + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.fkr new file mode 100644 index 0000000..90e6aa2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.fkr @@ -0,0 +1,1331 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=instrument-bond-cashflow +Name=Instrument Bond Cashflow Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Bond Cashflow Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=instrument_id,instrument_name,instrument_type_id,instrument_template_id,instrument_result_type_id,instrument_domain_id,instrument_layout_id,instrument_active_since,instrument_active_until,instrument_buy_label,instrument_sell_label,instrument_flags,instrument_group,instrument_currency_id,instrument_issuer_id,instrument_issuer_unit_id,instrument_amount_rounding,instrument_accrued_interest,instrument_settlement_flags,instrument_minimum_price_denom,instrument_default_price_denom,instrument_sign_id,instrument_coupon_rate,key_id,umi_id,active_since,active_until,type_id,subtype_id,category_id,value_date,currency_id,date_basis,fixing_price,kinds,cf_flags,cf_flags_2,cf_keep_flags,flags,reference_id,referee_id,amount,amount_rounding,actual_currency_id,actual_fx_rate,actual_amount,subcategory_id,sign,since_when,until_when,payment_date,realize_date,origin_id,origin_value,ex_date,reference_date,spread,cashflow_group,accrual_date_basis,fixing_rate_id,fixing_period_id,fixing_period_2_id,fixing_param,fixing_subscenario_id,fixing_date,fixing_from,fixing_to,fixing_price_2,fixing_quote,fixing_type,fixing_divider,fixing_cap,fixing_floor,expression,freeze_date,effective_fixing_date,fixing_processing_date,type_name +Selected=accrual_date_basis,active_since,active_until,actual_amount,actual_currency_id,actual_fx_rate,amount,amount_rounding,cashflow_group,category_id,cf_flags,cf_flags_2,cf_keep_flags,currency_id,date_basis,effective_fixing_date,ex_date,expression,fixing_cap,fixing_date,fixing_divider,fixing_floor,fixing_from,fixing_param,fixing_period_2_id,fixing_period_id,fixing_price,fixing_price_2,fixing_processing_date,fixing_quote,fixing_rate_id,fixing_subscenario_id,fixing_to,fixing_type,flags,freeze_date,instrument_accrued_interest,instrument_active_since,instrument_active_until,instrument_amount_rounding,instrument_buy_label,instrument_coupon_rate,instrument_currency_id,instrument_default_price_denom,instrument_domain_id,instrument_flags,instrument_group,instrument_id,instrument_issuer_id,instrument_issuer_unit_id,instrument_layout_id,instrument_minimum_price_denom,instrument_name,instrument_result_type_id,instrument_sell_label,instrument_settlement_flags,instrument_sign_id,instrument_template_id,instrument_type_id,key_id,kinds,origin_id,origin_value,payment_date,realize_date,referee_id,reference_date,reference_id,sign,since_when,spread,subcategory_id,subtype_id,type_id,type_name,umi_id,until_when,value_date +Grouping= +Sorting=instrument_id,type_name,since_when,until_when +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= +umi_id= + + +[Format instrument_id] +Name=instrument_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=138 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_name] +Name=instrument_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_type_id] +Name=instrument_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_template_id] +Name=instrument_template_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_result_type_id] +Name=instrument_result_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_domain_id] +Name=instrument_domain_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_layout_id] +Name=instrument_layout_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_active_since] +Name=instrument_active_since +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_active_until] +Name=instrument_active_until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=95 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_buy_label] +Name=instrument_buy_label +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_sell_label] +Name=instrument_sell_label +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_flags] +Name=instrument_flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=instrument_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_currency_id] +Name=instrument_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_issuer_id] +Name=instrument_issuer_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_issuer_unit_id] +Name=instrument_issuer_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_amount_rounding] +Name=instrument_amount_rounding +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_accrued_interest] +Name=instrument_accrued_interest +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_settlement_flags] +Name=instrument_settlement_flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_minimum_price_denom] +Name=instrument_minimum_price_denom +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_default_price_denom] +Name=instrument_default_price_denom +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_sign_id] +Name=instrument_sign_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_coupon_rate] +Name=instrument_coupon_rate +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_id] +Name=key_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_id] +Name=umi_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=active_since +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=active_until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subtype_id] +Name=subtype_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format category_id] +Name=category_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name=value_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name=date_basis +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price] +Name=fixing_price +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kinds] +Name=kinds +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags] +Name=cf_flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags_2] +Name=cf_flags_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_keep_flags] +Name=cf_keep_flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_id] +Name=reference_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format referee_id] +Name=referee_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=104 +width_in_characters=20 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount_rounding] +Name=amount_rounding +Expression= +report_total= +date_format= +time_format= +Justify= +width=79 +width_in_characters=15 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_currency_id] +Name=actual_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_fx_rate] +Name=actual_fx_rate +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_amount] +Name=actual_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subcategory_id] +Name=subcategory_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign] +Name=sign +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format since_when] +Name=since_when +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=83 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format until_when] +Name=until_when +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=payment_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realize_date] +Name=realize_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_id] +Name=origin_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_value] +Name=origin_value +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ex_date] +Name=ex_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_date] +Name=reference_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +Name=spread +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_group] +Name=cashflow_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrual_date_basis] +Name=accrual_date_basis +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate_id] +Name=fixing_rate_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_id] +Name=fixing_period_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_2_id] +Name=fixing_period_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_param] +Name=fixing_param +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_subscenario_id] +Name=fixing_subscenario_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_date] +Name=fixing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_from] +Name=fixing_from +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_to] +Name=fixing_to +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price_2] +Name=fixing_price_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_quote] +Name=fixing_quote +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_type] +Name=fixing_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_divider] +Name=fixing_divider +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_cap] +Name=fixing_cap +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_floor] +Name=fixing_floor +Expression= +report_total= +date_format= +time_format= +Justify= +width=107 +width_in_characters=21 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expression] +Name=expression +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format freeze_date] +Name=freeze_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_fixing_date] +Name=effective_fixing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_processing_date] +Name=fixing_processing_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_name] +Name=type_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.fkr new file mode 100644 index 0000000..b4b3ed4 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.fkr @@ -0,0 +1,211 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=instrument-bond-schedule +Name=Instrument Bond Schedule Report +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Bond Schedule Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=instrument_id LIKE '%FRN' and id == 1 +Fields=accrual_date_basis,accrual_date_basis_dep,accrual_date_basis_flags,instrument_accrued_interest,instrument_active_since,instrument_active_until,actual_amount,actual_amount_dep,actual_amount_flags,actual_currency_id,actual_currency_id_dep,actual_currency_id_flags,actual_fx_rate,actual_fx_rate_dep,actual_fx_rate_flags,adjust_value_date,adjust_value_date_dep,adjust_value_date_flags,amount_rounding,amount_rounding_dep,amount_rounding_flags,instrument_buy_label,calculation_method_id,calculation_method_id_dep,calculation_method_id_flags,calculation_rounding,calculation_rounding_dep,calculation_rounding_flags,calendar_id,calendar_id_dep,calendar_id_flags,cap,cap_dep,cap_flags,cashflow_group,cashflow_group_dep,cashflow_group_flags,category_id,category_id_dep,category_id_flags,cf_flags,cf_flags_2,cf_flags_2_dep,cf_flags_2_flags,cf_flags_dep,cf_flags_flags,convention_id,convention_id_dep,convention_id_flags,instrument_coupon_rate,instrument_currency_id,currency_id_dep,currency_id_flags,date_basis,date_basis_dep,date_basis_flags,instrument_default_price_denom,description,description_dep,description_flags,divider,divider_dep,divider_flags,instrument_domain_id,end_date,end_date_dep,end_date_flags,exclude_period,exclude_period_dep,exclude_period_flags,expression,expression_dep,expression_flags,factor,factor_dep,factor_flags,first_date,first_date_dep,first_date_flags,fixing_calendar_id,fixing_calendar_id_dep,fixing_calendar_id_flags,fixing_convention_id,fixing_convention_id_dep,fixing_convention_id_flags,fixing_holidays_id,fixing_holidays_id_dep,fixing_holidays_id_flags,fixing_max_offset,fixing_max_offset_dep,fixing_max_offset_flags,fixing_offset,fixing_offset_dep,fixing_offset_flags,fixing_period_id,fixing_period_id_dep,fixing_period_id_flags,fixing_rate_id,fixing_rate_id_dep,fixing_rate_id_flags,fixing_subscenario_id,fixing_subscenario_id_dep,fixing_subscenario_id_flags,flags,flags_dep,flags_flags,floor,floor_dep,floor_flags,frequency,frequency_dep,frequency_flags,frn_convention,frn_convention_dep,frn_convention_flags,gap_id,gap_id_dep,gap_id_flags,gap_set_id,gap_set_id_dep,gap_set_id_flags,group_id,group_id_dep,group_id_flags,holidays_id,holidays_id_dep,holidays_id_flags,instrument_id,id_dep,id_flags,in_advance,in_advance_dep,in_advance_flags,in_arrears,in_arrears_dep,in_arrears_flags,instrument_group,instrument_issuer_id,item_flags,kind_id,kind_id_dep,kind_id_flags,instrument_layout_id,long_stub,long_stub_dep,long_stub_flags,max_offset,max_offset_dep,max_offset_flags,method_id,method_id_dep,method_id_flags,min_stub_length,min_stub_length_dep,min_stub_length_flags,instrument_minimum_price_denom,instrument_name,need_fixing,need_fixing_dep,need_fixing_flags,offset,offset_dep,offset_flags,payment_client_id,payment_client_id_dep,payment_client_id_flags,payment_offset,payment_offset_dep,payment_offset_flags,penultimate_date,penultimate_date_dep,penultimate_date_flags,pseudo,pseudo_dep,pseudo_flags,rate,rate_dep,rate_flags,rate_offset,rate_offset_dep,rate_offset_flags,rate_type,rate_type_dep,rate_type_flags,reference_id,reference_id_dep,reference_id_flags,roll_from_start,roll_from_start_dep,roll_from_start_flags,instrument_sell_label,instrument_settlement_flags,sign,sign_dep,sign_flags,instrument_sign_id,spread,spread_dep,spread_flags,start_date,start_date_dep,start_date_flags,subcategory_id,subcategory_id_dep,subcategory_id_flags,subtype_id,subtype_id_dep,subtype_id_flags,instrument_template_id,instrument_type_id,type_id_dep,type_id_flags,instrument_result_type_id,schedule_template_id,fixed_roll_date,fixed_roll_date_dep,fixed_roll_date_flags,instrument_flags,instrument_issuer_unit_id,instrument_amount_rounding,umi_id,id,currency_id,type_id,max_number_of_dates,payment_client_unit_id,max_number_of_dates_dep,max_number_of_dates_flags,payment_client_unit_id_dep,payment_client_unit_id_flags +Selected=instrument_active_until,instrument_id,frequency,fixing_rate_id,spread,floor,cap,factor +Grouping= +Sorting=instrument_id +GroupSortLevel=0 +GroupSortColumns= +Pagebreak=0 +ZoomPercent=100 +DecimalSeparator=. +ThousandsSeparator=, +ExcelWorkbook= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +SortDescending= +GroupSortDescending= +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 + + +[Parameters] +Fields=umi_id +umi_id= + + +[Format instrument_active_until] +Name= +width=86 +width_in_characters=17 +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format instrument_id] +Name= +width=114 +width_in_characters=22 +report_total= +date_format= +time_format= +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format frequency] +Name= +width=69 +width_in_characters=13 +report_total= +date_format= +time_format= +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format fixing_rate_id] +Name= +width=106 +width_in_characters=21 +report_total= +date_format= +time_format= +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format spread] +Name= +width=60 +width_in_characters=12 +report_total= +date_format= +time_format= +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format floor] +Name= +width=60 +width_in_characters=12 +report_total= +date_format= +time_format= +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format cap] +Name= +width=60 +width_in_characters=12 +report_total= +date_format= +time_format= +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= + + +[Format factor] +Name=Factor +width=99 +width_in_characters=19 +report_total= +date_format= +time_format= +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify=right + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.fkr new file mode 100644 index 0000000..afc4d68 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.fkr @@ -0,0 +1,977 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=instrument +Name=Instrument Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,security_identifier_type,security_identifier,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id +Selected=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,security_identifier_type,security_identifier,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format layout_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name= +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name= +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format buy_label] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sell_label] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format path_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format result_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch0_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch3_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch4_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch5_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch6_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch7_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch8_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch9_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch10_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch11_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch12_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch13_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch14_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch15_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch16_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch17_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch18_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch19_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_INSTRUMENT_REPORT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.fkr new file mode 100644 index 0000000..222b755 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.fkr @@ -0,0 +1,482 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=market-info +Name=Market Info Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Market Info Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,source,producer,item,group_tag,enabled,name_id,type_id,scenario_id,subscenario_id,period_id,period_1_id,period_2_id,bid_tag,ask_tag,date_tag,price_type,change_period,change_percent,valid_from_time,valid_to_time,time_zone_id,calendar_id,divider,flags +Selected=id,source,producer,item,group_tag,enabled,name_id,type_id,scenario_id,subscenario_id,period_id,period_1_id,period_2_id,bid_tag,ask_tag,date_tag,price_type,change_period,change_percent,valid_from_time,valid_to_time,time_zone_id,calendar_id,divider,flags +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format source] +Name=source +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format producer] +Name=producer +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format item] +Name=item +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format group_tag] +Name=group_tag +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format enabled] +Name=enabled +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name_id] +Name=name_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type_id] +Name=type_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format scenario_id] +Name=scenario_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name=subscenario_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_id] +Name=period_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_1_id] +Name=period_1_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_2_id] +Name=period_2_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bid_tag] +Name=bid_tag +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ask_tag] +Name=ask_tag +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_tag] +Name=date_tag +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_type] +Name=price_type +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format change_period] +Name=change_period +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format change_percent] +Name=change_percent +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format valid_from_time] +Name=valid_from_time +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format valid_to_time] +Name=valid_to_time +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name=time_zone_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format calendar_id] +Name=calendar_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format divider] +Name=divider +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.yml new file mode 100644 index 0000000..3cbdfc0 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MARKET_INFO.yml @@ -0,0 +1,23 @@ + + +parameters: + + date_from: + value: t-1d + hidden: false + + date_to: + value: t-1d + hidden: false + + scenario_id: + value: + - " " + - ECBRR + - FIXING + - FREEZE + - FRM + - FRMJPY + - SWAPMARGIN + + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.fkr new file mode 100644 index 0000000..e68746e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.fkr @@ -0,0 +1,650 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb_prices +Name=ECB Rate Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=ECB Rate Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=target_date,type_id,name_id,id,scenario_id,subscenario_id,source_id,item,mi_group,currency_id,period_from,period_to,period_id,period_1_id,period_2_id,price_type,date_basis,date,date_time,price_1,price_2,price_3,price_4,price_5,price_6,price_7,price_8,price_9,price_10,bid,ask,maturity_date,security_identifier,security_identifier_type,instrument_group +Selected=target_date,type_id,name_id,id,scenario_id,subscenario_id,source_id,item,mi_group,currency_id,period_from,period_to,period_id,period_1_id,period_2_id,price_type,date_basis,date,date_time,price_1,price_2,bid,ask,maturity_date,security_identifier,security_identifier_type,instrument_group +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=date_from,date_to,group_tag,scenario_id,subscenario_id,instrument_group,instrument_id,rate_type_id +date_from=t-5d +date_to=t +group_tag= +scenario_id= +subscenario_id= +instrument_group= +instrument_id= +rate_type_id= + + +[Format target_date] +Name=TARGET_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type_id] +Name=TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name_id] +Name=NAME_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format scenario_id] +Name=SCENARIO_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name=SUBSCENARIO_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format source_id] +Name=SOURCE_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format item] +Name=ITEM +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format mi_group] +Name=MI_GROUP +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_from] +Name=PERIOD_FROM +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_to] +Name=PERIOD_TO +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_id] +Name=PERIOD_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_1_id] +Name=PERIOD_1_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_2_id] +Name=PERIOD_2_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_type] +Name=PRICE_TYPE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_basis] +Name=DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date] +Name=DATE_ +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=83 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_time] +Name=DATE_TIME +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=127 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_1] +Name=PRICE_1 +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_2] +Name=PRICE_2 +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_3] +Name=PRICE_3 +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bid] +Name=BID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ask] +Name=ASK +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format security_identifier] +Name=SECURITY_IDENTIFIER +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name=SECURITY_IDENTIFIER_TYPE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.yml new file mode 100644 index 0000000..feacf3c --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_MOPDBPRICES.yml @@ -0,0 +1,23 @@ + + +parameters: + + date_from: + value: t-5d + hidden: false + + date_to: + value: t + hidden: false + + scenario_id: + value: + - " " + - ECBRR + - FIXING + - FREEZE + - FRM + - FRMJPY + - SWAPMARGIN + + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.fkr new file mode 100644 index 0000000..2b33453 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.fkr @@ -0,0 +1,162 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=parameter +Name=Parameter Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Parameter Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=param,type_id,number,order_number,name +Selected=type_id,number,order_number,param,name +Grouping= +Sorting=type_id,number,order_number +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= + + +[Format param] +Name=param +Expression= +report_total= +date_format= +time_format= +Justify=left +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format type_id] +Name=type_id +Expression= +report_total= +date_format= +time_format= +Justify=left +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format number] +Name=number_ +Expression= +report_total= +date_format= +time_format= +Justify=right +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format order_number] +Name=order_number +Expression= +report_total= +date_format= +time_format= +Justify=right +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify=left +width=129 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PARAMETER.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.fkr new file mode 100644 index 0000000..05c479e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.fkr @@ -0,0 +1,284 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=activity-log +Name=Activity Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Activity Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=id like '%EOD PERF%SAVE%' and test=0 +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id,count,test +Selected=id,due_date,finish_date +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=365 +width_in_characters=73 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format prerequisite_id] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format due_date] +Name=Due Date +Expression= +total_count= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=133 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format start_date] +Name=Start Date +Expression= +total_count= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format finish_date] +Name=Finish Date +Expression= +total_count= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=128 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format description] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user_id] +Name= +Expression= +total_count= +report_total= +date_format= +time_format= +Justify= +width=78 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format count] +Name=count +Expression=1 +total_count=sum(@count) +report_total= +date_format= +time_format= +Justify=right +width=68 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format test] +Name=test +Expression=10000*year( param_from )+100*month( param_from )+day(param_from) -( 10000*year( due_date)+100*month( due_date)+day(due_date)) +report_total= +date_format=%c +time_format= +Justify=right +width=144 +width_in_characters=28 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.yml new file mode 100644 index 0000000..fe195de --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.yml @@ -0,0 +1,14 @@ + +parameters: + + from: + value: select("dual", "to_char(sysdate - 1, 'YYYYMMDD')|| ' - 000000'") + hidden: false + + to: + value: select("dual", "to_char(sysdate - 1, 'YYYYMMDD')|| ' - 235959'") + hidden: false + + state: + value: 5 + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.fkr new file mode 100644 index 0000000..c4251bb --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.fkr @@ -0,0 +1,769 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb_prices +Name=Rate Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Rate Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=enabled_flag<>0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=target_date,type_id,name_id,id,scenario_id,subscenario_id,source_id,item,mi_group,currency_id,period_from,period_to,period_id,period_1_id,period_2_id,price_type,date_basis,date,date_time,price_1,price_2,price_3,price_4,price_5,price_6,price_7,price_8,price_9,price_10,bid,ask,maturity_date,security_identifier,security_identifier_type,instrument_group,mid,isin,bloomberg_tick_item,reuters_item,price_type_csd,instr_group,enabled_flag +Selected=name_id,period_id,bid,ask,mid,price_type_csd,currency_id,date,date_time,isin,reuters_item,instr_group,scenario_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=14005927, 6441018, 0 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format target_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=141 +width_in_characters=28 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format scenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format source_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format default_item] +Name=Reuters Code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mi_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_from] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_to] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=150 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_type_csd] +Name=Price Type CSD +Expression=if(type_id='FX-RATE')( if( period_id='SPOT' )( 'FX Spot Rate' ) ( 'FX Forward Points' ) )( if( type_id='IR-RATE' )( 'Interest Rate' ) ( if(price_type='price@price%')('Price %')(if(price_type='yield@trading')('Yield')(price_type)) ) ) +report_total= +date_format= +time_format= +Justify= +width=150 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=123 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_time] +Name=Date Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=176 +width_in_characters=35 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bid] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ask] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name=Security Identifier +Expression= +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bloomberg_tick_item] +Name=BLOOMBERG-TICK item +Expression=get_field('ECBMarketInfoByString', 'BLOOMBERG-TICK', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format reuters_item] +Name=REUTERS item +Expression=get_field('ECBMarketInfoByString', 'REUTERS', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=left +width=132 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin] +Name=Bloomberg Identifier +Expression=if(bloomberg_tick_item <> '') (bloomberg_tick_item) (if( left(name_id,5)= "EUBIS" or left(name_id,5) like "USBIS%" )( get_security_identifier( name_id, "SEDOL") ) ( if(get_security_identifier( name_id, "ISIN") <> '')(get_security_identifier( name_id, "ISIN")) (get_security_identifier( name_id, "TICKER")) )) +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=Instrument Group +Expression= +report_total= +date_format= +time_format= +Justify=right +width=247 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format mid] +Name=Mid +Expression=(ask+bid)/2 +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=3 +max_precision=3 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format instr_group] +Name=Instr. Group +Expression=if(type_id='UM-PRICE')(get_field("UMPathFlat", get_field("UMI", name_id, "path_id"), 0, "upath")) +report_total= +date_format= +time_format= +Justify=right +width=159 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format enabled_flag] +Name=enabled flag +Expression=as_int( get_field('ECBMarketInfoByString', 'REUTERS', name_id, period_id, 'enabled_p')+get_field('ECBMarketInfoByString', 'BLOOMBERG-TICK', name_id, period_id, 'enabled_p') ) +report_total= +date_format= +time_format= +Justify=right +width=81 +width_in_characters=10 +divider=1 +precision=9 +max_precision=9 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.yml new file mode 100644 index 0000000..feacf3c --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PRICE_RATE_REPORT.yml @@ -0,0 +1,23 @@ + + +parameters: + + date_from: + value: t-5d + hidden: false + + date_to: + value: t + hidden: false + + scenario_id: + value: + - " " + - ECBRR + - FIXING + - FREEZE + - FRM + - FRMJPY + - SWAPMARGIN + + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.fkr new file mode 100644 index 0000000..c113c3a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.fkr @@ -0,0 +1,242 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=property +Name=Property Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Property Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=object_id,key_id,key_2_id,key_3_id,order_number,type_id,value_id,name,flags,validate_hook +Selected=object_id,key_id,key_2_id,key_3_id,order_number,type_id,value_id,name,flags,validate_hook +Grouping= +Sorting=object_id,key_id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= + + +[Format object_id] +Name=object_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_id] +Name=key_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_2_id] +Name=key_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_3_id] +Name=key_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format order_number] +Name=order_number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_id] +Name=value_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format validate_hook] +Name=validate_hook +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PROPERTY.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PSPPBLOCKEDINSTRUMENT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PSPPBLOCKEDINSTRUMENT.fkr new file mode 100644 index 0000000..eb6743f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_PSPPBLOCKEDINSTRUMENT.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rules +Name=Rules Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Rules Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=id like "PORT ONLY REST BLACKOUT PERIOD" and instrument_id <> "DONT_DELETE_ME" and issuer_id="" +MergeCellsVertically= +MergeCellsHorizontally= +Fields=sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,number,priority,portfolio_id,instrument_path_id,status,not_status,cf_leg_path_id,cf_flags,cf_not_flags,flags,name,client_id,client_main_group_id,client_group_id,user_id,currency_id,currency_2_id,instrument_id,secondary_instrument_id,package_main_type_id,package_type_id,issuer_id,issuer_main_group_id,issuer_group_id,cp_client_id,cp_client_main_group_id,cp_client_group_id,cp_portfolio_id,broker_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,contexts,not_contexts,transaction_flags,transaction_not_flags,sign_id,type_id,kind_id,state_id,cf_status,cf_not_status,cf_currency_id,cf_payment_currency_id,cf_sign,cf_type_id,cf_subtype_id,cf_category_id,cf_subcategory_id,cf_leg_group,cf_leg_id,cf_branch0_id,cf_branch1_id,cf_branch2_id,cf_branch3_id,cf_branch4_id,cf_flags_2,cf_not_flags_2,cf_kinds,cf_not_kinds,isin,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,cf_branch10_id,cf_branch11_id,cf_branch12_id,cf_branch13_id,cf_branch14_id,cf_branch15_id,cf_branch16_id,cf_branch17_id,cf_branch18_id,cf_branch19_id,cf_branch20_id,cf_branch21_id,cf_branch22_id,cf_branch23_id,cf_branch24_id,cf_branch25_id,cf_branch26_id,cf_branch27_id,cf_branch28_id,cf_branch29_id,cf_branch30_id,cf_branch31_id,cf_branch32_id,cf_branch33_id,cf_branch34_id,cf_branch35_id,cf_branch36_id,cf_branch37_id,cf_branch38_id,cf_branch39_id,cf_branch5_id,cf_branch6_id,cf_branch7_id,cf_branch8_id,cf_branch9_id,client_unit_id,cp_client_unit_id,issuer_unit_id,cf_type_name +Selected=instrument_id,isin,name +Grouping= +Sorting=id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=12363955, 8999536, 0 +HeaderRowColors=11056316, 8999536 +DataRowColors=15724270, 0, 0 +TotalRowColors=12363955, 8999536, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=116 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=115 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format isin] +Name=ISIN +Expression=get_security_identifier(instrument_id, "ISIN") +report_total= +date_format= +time_format= +Justify=right +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.fkr new file mode 100644 index 0000000..ec28691 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.fkr @@ -0,0 +1,606 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=client +Name=Client Report +HeaderLeft=[\n]%pagebreak +HeaderCenter=Client Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +Fields=client_id,domain_id,country_id,roles,groups,flags,template_id,name,parent_id,short_name,postal_address,street_address,payment_address,city,risk_country_id,sdm_id,sdm_state_name,sdm_state_id,sdm_edit_allowed,sdm_state_chg_allowed,sdm_reject_state_id,sdm_accept_state_id,sdm_daccept_state_id,sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,swift_code,role_number,flags_number +Selected=client_id,domain_id,country_id,roles,name,parent_id,postal_address,street_address,city,risk_country_id,swift_code,flags +Grouping= +GroupSortLevel=0 +Pagebreak=0 +ZoomPercent=100 +DecimalSeparator=. +ThousandsSeparator=, +footerleft= +footercenter= +footerright= +ExcelWorkbook= +Criteria= +TotalsOnly=0 +Sorting= +GroupSortColumns= +MergeCellsVertically= +MergeCellsHorizontally= +SortDescending= +GroupSortDescending= +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=13414297, 8411193, 8411193 +HeaderRowColors=15849921, 8411193 +DataRowColors=16577257, 0, 0 +TotalRowColors=13414297, 8411193, 8411193 + + +[Format client_id] +Name=Client +Justify=left +width=131 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format domain_id] +Name=Domain +Justify=left +width=65 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format country_id] +Name=Country +Justify=left +width=60 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format roles] +Name=Roles +Justify=left +width=83 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format groups] +Name=Groups +Justify=right +width=62 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format flags] +Name=Flags +Justify=right +width=274 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=54 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format template_id] +Name=Template +Justify=left +width=72 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format name] +Name=Name +Justify=left +width=627 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=125 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format parent_id] +Name=Parent +Justify=left +width=111 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format short_name] +Name=Short Name +Justify=left +width=70 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format postal_address] +Name=Postal Address +Justify=left +width=643 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=128 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format street_address] +Name=Street Address +Justify=left +width=387 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=77 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format payment_address] +Name=Payment Address +Justify=left +width=59 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=11 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format city] +Name=City +Justify=left +width=53 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Format risk_country_id] +Name=Risk Country +Justify=left +width=67 +total_country_id= +report_total= +date_format= +time_format= +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=500 +Right=500 +Top=1000 +Bottom=1000 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=1 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Parameters] + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Format sdm_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_name] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_edit_allowed] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_chg_allowed] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_reject_state_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_accept_state_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_daccept_state_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_object_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_key_id] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_create_date] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_access_level] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_active] +Name= +Expression= +total_country_id= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format swift_code] +Name=Swift Code +Expression=get_client_swift_code(client_id,"") +total_country_id= +report_total= +date_format= +time_format= +Justify=left +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format role_number] +Name=role_number +Expression=get_field("client",client_id,"roles") +report_total= +date_format= +time_format= +Justify=right +width=69 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags_number] +Name=Flags_number +Expression=get_field("client",client_id,"flags") +report_total= +date_format= +time_format= +Justify=right +width=96 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.fkr new file mode 100644 index 0000000..a416e42 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.fkr @@ -0,0 +1,261 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-client-group-map +Name=ECB Client Group Map Report +HeaderLeft=[\n]%pagebreak +HeaderCenter=ECB Client Group Map Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=!(sdm_key_id=14281) +Fields=client_id,main_group_id,group_id,group_x,sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,main_group_name,group_name +Selected=client_id,main_group_id,main_group_name,group_id,group_name +Grouping= +Sorting= +GroupSortLevel=0 +GroupSortColumns= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 +ExcelWorkbook= +MergeCellsVertically= +MergeCellsHorizontally= +SortDescending= +GroupSortDescending= +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 + + +[Parameters] + + +[Format client_id] +Name= +total_client_id= +report_total= +date_format= +time_format= +width=128 +width_in_characters=25 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format main_group_id] +Name= +total_client_id= +report_total= +date_format= +time_format= +width=139 +width_in_characters=27 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Justify=left +Expression= +divider=1 + + +[Format group_id] +Name= +total_client_id= +report_total= +date_format= +time_format= +width=110 +width_in_characters=22 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format group_x] +Name=Groupx +Expression=if (group_id ='')( 5 )( 1 ) +total_client_id= +report_total= +date_format= +time_format= +Justify=right +width=58 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +divider=1 + + +[Format sdm_object_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_key_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_create_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_access_level] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_active] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Format main_group_name] +Name=main group name +Expression=get_field("ClientGroupGroup", main_group_id, "name") +report_total= +date_format= +time_format= +Justify=right +width=291 +width_in_characters=58 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format group_name] +Name=group name +Expression=get_field("ClientGroup", main_group_id, group_id, "name") +report_total= +date_format= +time_format= +Justify=right +width=253 +width_in_characters=50 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_CLIENTGROUPMAP.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.fkr new file mode 100644 index 0000000..0dc24d3 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.fkr @@ -0,0 +1,209 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-ecb-instrumentts +Name=RAR ECB Instrument TS Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR ECB Instrument TS Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,data_set_id,figure_date,instrument_id,ctd_instrument_id,modified_duration,convexity,ytm +Selected=instrument_id,figure_date,ctd_instrument_id,modified_duration,ytm,convexity +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format data_set_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format figure_date] +Name= +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ctd_instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format modified_duration] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format convexity] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ytm] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=16 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.yml new file mode 100644 index 0000000..464690f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_ECBINSTRUMENTS.yml @@ -0,0 +1,20 @@ + +parameters: + + valuation_date: + value: t-1d + hidden: false + + scenario_id: + value: FREEZE + hidden: true + + valuation_method_id: + value: NORMAL + hidden: true + + re_instrument_group_a: + value: 120 + hidden: true + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.fkr new file mode 100644 index 0000000..c86a12e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.fkr @@ -0,0 +1,161 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=calendar +Name=Calendar Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Calendar Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=id="FRM-HOL" +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,name,holiday_date,holiday_reason,default_holidays +Selected=id,name,holiday_date,holiday_reason,default_holidays +Grouping= +Sorting=holiday_date +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=194 +width_in_characters=38 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format holiday_date] +Name=Holiday Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format holiday_reason] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=83 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format default_holidays] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FRM_HOLIDAYS.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.fkr new file mode 100644 index 0000000..0359cc6 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.fkr @@ -0,0 +1,145 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-umi-future-basket +Name=RAR UMI Future Basket Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR UMI Future Basket Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,instrument_id,conversion_factor,flags +Selected=id,instrument_id,conversion_factor,flags +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format conversion_factor] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=71 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=72 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_FUTURE_BONDS.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.fkr new file mode 100644 index 0000000..69ac185 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.fkr @@ -0,0 +1,1377 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=instrument +Name=Instrument Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,security_identifier_type,security_identifier,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,maturity_date,issue_date,coupons_per_year,date_basis,coupon_rate,issuer,issuer_disc,yield_curve,issuer_bond,issuer_future,instrument_group_name,currency_id,currency_bond,currency_fut,currency_disc,currency_depo,currency_repo,currency_eqty,currency_gen,issuer_equity +Selected=id,type_id,name,instrument_group,instrument_group_name,issue_date,issuer,maturity_date,currency_id,coupon_rate,date_basis,coupons_per_year,active_since,active_until,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id +Grouping= +Sorting=issuer +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=110 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=189 +width_in_characters=37 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=210 +width_in_characters=42 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=405 +width_in_characters=81 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format layout_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name=Active Since +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=128 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name=Active Until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=96 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format buy_label] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sell_label] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format path_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format result_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=72 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=194 +width_in_characters=38 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=121 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=173 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch0_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=81 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=104 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch3_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=171 +width_in_characters=34 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch4_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=96 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch5_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=109 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch6_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch7_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=89 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch8_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch9_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch10_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch11_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch12_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch13_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch14_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch15_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch16_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch17_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch18_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch19_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format maturity_date] +Name=Maturity Date +Expression=as_datetime( get_field("UMIDateSetup", id, "maturity_date") ) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=140 +width_in_characters=28 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issue_date] +Name=Issue date +Expression=as_datetime( get_field("UMIDateSetup", id, "issue_date") ) +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=right +width=192 +width_in_characters=38 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format coupons_per_year] +Name=Coupons Per Year +Expression=get_field("ECBcouponsPerYear",id,"coupons_per_year") +report_total= +date_format= +time_format= +Justify=right +width=147 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_basis] +Name=Date Basis +Expression=as_text(get_field("ECBUMIBondSchedule",id,2,"DATE_BASIS")) +report_total= +date_format= +time_format= +Justify=right +width=172 +width_in_characters=34 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format coupon_rate] +Name=Coupon Rate +Expression=as_float( get_field("UMIBond",id,"coupon_rate")) +report_total= +date_format= +time_format= +Justify=right +width=201 +width_in_characters=40 +divider=1 +precision=10 +max_precision=10 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer] +Name=Issuer +Expression=issuer_bond+ issuer_disc+ issuer_future+ issuer_equity +report_total= +date_format= +time_format= +Justify=right +width=186 +width_in_characters=37 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_disc] +Name=issuer discount +Expression=as_text( get_field("UMIDiscount",id,"issuer_id")) +report_total= +date_format= +time_format= +Justify=right +width=137 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format yield_curve] +Name=Yield Curve +Expression=as_text(get_field("UMIBasePrice",37344,"name_id")) +report_total= +date_format= +time_format= +Justify=right +width=385 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_bond] +Name=issuer bond +Expression=as_text( get_field("UMIBond",id,"issuer_id")) +report_total= +date_format= +time_format= +Justify=right +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_future] +Name=issuer future +Expression=as_text( get_field("UMIFuture",id,"issuer_id")) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_group_name] +Name=Instrument Group Name +Expression=as_text( get_field("UMPathNode",path_id,"name")) +report_total= +date_format= +time_format= +Justify=right +width=341 +width_in_characters=68 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cusip] +Name=CUSIP +Expression=get_security_identifier( id, "CUSIP") +report_total= +date_format= +time_format= +Justify=right +width=130 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin] +Name=ISIN +Expression=get_security_identifier( id, "ISIN") +report_total= +date_format= +time_format= +Justify=right +width=124 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format sedol] +Name=SEDOL +Expression=get_security_identifier( id, "SEDOL") +report_total= +date_format= +time_format= +Justify=right +width=110 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ticker] +Name=TICKER +Expression=get_security_identifier( id, "TICKER") +report_total= +date_format= +time_format= +Justify=right +width=93 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bb_unique] +Name=BB_UNIQUE +Expression=get_security_identifier( id, "BB_UNIQUE") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=Currency +Expression=currency_bond+ currency_fut+ currency_disc + currency_depo+ currency_repo + currency_eqty + currency_gen +report_total= +date_format= +time_format= +Justify=right +width=126 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_bond] +Name=currency bond +Expression=get_field("UMIBond",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_fut] +Name=currency future +Expression=get_field("UMIFuture",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_disc] +Name=currency disc +Expression=get_field("UMIDiscount",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_depo] +Name=currency depo +Expression=get_field("UMIDeposit",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_repo] +Name=currency repo +Expression=get_field("UMIRepo",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_eqty] +Name=currency equity +Expression=get_field("UMIEquity",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_gen] +Name=currency gen +Expression=get_field("UMIGeneric",id,"currency_id") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_equity] +Name=issuer equity +Expression=as_text( get_field("UMIEquity",id,"issuer_id")) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_INSTRUMENT_REPORT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.fkr new file mode 100644 index 0000000..863ca23 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.fkr @@ -0,0 +1,465 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=limit +Name=Limit Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Limit Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=sdm_id,sdm_state_name,sdm_state_id,sdm_edit_allowed,sdm_state_chg_allowed,sdm_reject_state_id,sdm_accept_state_id,sdm_daccept_state_id,sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,domain_id,category_id,usage_logging_type,flags,name,template_id,global_client_query_id,limit_user_group_id,comment,usage_log_type +Selected=id,domain_id,category_id,usage_log_type,name,global_client_query_id,limit_user_group_id,comment,flags,template_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format sdm_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_edit_allowed] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_chg_allowed] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_reject_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_accept_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_daccept_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_object_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_key_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_create_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_access_level] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_active] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=147 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format category_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=119 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format usage_logging_type] +Name=Usage Logging Type_1 +Expression= +report_total= +date_format= +time_format= +Justify=left +width=132 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=Flags +Expression= +report_total= +date_format= +time_format= +Justify=right +width=63 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=214 +width_in_characters=42 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=72 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format global_client_query_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=123 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_user_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=121 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=93 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format usage_log_type] +Name=Usage Logging Type +Expression=if( usage_logging_type = "By Interval" )(2 )(0) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMIT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.fkr new file mode 100644 index 0000000..67f0c0b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-limit-access +Name=RAR Limit Access Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Limit Access Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=limit_id,user_id,ecbusrbdts +Selected=limit_id,user_id,ecbusrbdts +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format limit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ecbusrbdts] +Name=Ecbusrbdts +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=131 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITACCESS.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.fkr new file mode 100644 index 0000000..80bb15d --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.fkr @@ -0,0 +1,1809 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-limit-log +Name=RAR Limit Log Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Limit Log Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_id,id,date,time,sublimit_id,limit_currency_id,amount,old_amount,violation_p,violation_type,minimum_amount,maximum_amount,item_id,number,portfolio_id,position_type,package_main_type_id,package_type_id,package_id,market_id,instrument_path,instrument_id,currency_id,currency_class_id,cp_client_id,credit_client_id,issuer_id,underlying_issuer_id,leg_group,leg_path_id,leg_id,maturity_gap_id,underlying_maturity_gap_id,risk_gap_id,adjusted_risk_gap_id,delivery_gap_id,payment_date,bank_id,account_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,causing_number,server_id,affect_date,affect_gap_id,client_id,parent_id,client_main_group_id,client_group_id,delivery_date,owner_id,client_rating_id,client_rating_code_id,umi_rating_id,umi_rating_code_id,risk_country_id,underlying_instrument_id,guarantee_level,guarantee_type_id,guarantee_param_0,guarantee_param_1,guarantee_param_2,guarantee_param_3,guarantee_param_4,guarantee_param_5,guarantee_param_6,guarantee_param_7,guarantee_param_8,guarantee_param_9,time_utc +Selected=id,date,time_utc,sublimit_id,amount,old_amount,violation_type,violation_p,minimum_amount,maximum_amount,item_id,number,portfolio_id,package_type_id,package_id,market_id,instrument_id,currency_id,currency_class_id,cp_client_id,credit_client_id,issuer_id,underlying_issuer_id,leg_id,payment_date,bank_id,account_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,causing_number,server_id,delivery_gap_id,maturity_gap_id,risk_gap_id,instrument_path,affect_date,affect_gap_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,client_group_id,client_id,client_main_group_id,client_rating_code_id,client_rating_id,delivery_date,guarantee_level,guarantee_param_0,guarantee_param_1,guarantee_param_2,guarantee_param_3,guarantee_param_4,guarantee_param_5,guarantee_param_6,guarantee_param_7,guarantee_param_8,guarantee_param_9,guarantee_type_id,limit_currency_id,log_id,owner_id,package_main_type_id,parent_id,risk_country_id,umi_rating_code_id,umi_rating_id,underlying_instrument_id,underlying_maturity_gap_id,adjusted_risk_gap_id,leg_group,leg_path_id,position_type +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format log_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=94 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format time] +Name=Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=148 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format sublimit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format old_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format violation_p] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format violation_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format minimum_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maximum_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format position_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_path] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_class_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format credit_client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_path_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_maturity_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format risk_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format adjusted_risk_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=Payment Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bank_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch0_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch3_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch4_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch5_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch6_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch7_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch8_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch9_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch10_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch11_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch12_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch13_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch14_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch15_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch16_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch17_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch18_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch19_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format causing_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format server_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format affect_date] +Name=Affect Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format affect_gap_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_main_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_rating_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_rating_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format risk_country_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_level] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format time_utc] +Name=Time +Expression=time +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=right +width=135 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.yml new file mode 100644 index 0000000..bc776f7 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_LIMITLOG.yml @@ -0,0 +1,12 @@ + + +parameters: + from: + value: "t" + hidden: false + + to: + value: "t" + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.fkr new file mode 100644 index 0000000..7be2ddc --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.fkr @@ -0,0 +1,659 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=market-info +Name=Price Check Market Info +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Market Info Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,source,producer,item,group_tag,enabled,name_id,type_id,scenario_id,subscenario_id,period_id,period_1_id,period_2_id,bid_tag,ask_tag,date_tag,price_type,change_period,change_percent,valid_from_time,valid_to_time,time_zone_id,calendar_id,divider,flags,isin,active_to,inst_group,path,maturity,display,active_from +Selected=item,enabled,name_id,scenario_id,subscenario_id,type_id,period_id,period_1_id,period_2_id,flags,source,producer,group_tag,bid_tag,ask_tag,date_tag,change_period,change_percent,time_zone_id,valid_from_time,valid_to_time,calendar_id,divider,price_type +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=12301740, 3947063, 0 +HeaderRowColors=14867151, 3947063 +DataRowColors=16511720, 0, 0 +TotalRowColors=12301740, 3947063, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=125 +width_in_characters=25 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format source] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=66 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format producer] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=157 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format group_tag] +Name=Group Tag +Expression= +total_source= +report_total= +date_format= +time_format= +Justify=left +width=89 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format enabled] +Name=Enabled +Expression= +total_source= +report_total= +date_format= +time_format= +Justify=right +width=50 +width_in_characters=10 +divider=1 +precision=1 +max_precision=1 +show_zero=1 +NoNumberFormatting=0 + + +[Format name_id] +Name=Name +Expression= +total_source= +report_total= +date_format= +time_format= +Justify=left +width=180 +width_in_characters=36 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format scenario_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=68 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_1_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_2_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bid_tag] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ask_tag] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_tag] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_type] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=124 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format change_period] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format change_percent] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format valid_from_time] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format valid_to_time] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format calendar_id] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format divider] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +total_source= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format isin] +Name=ISIN +Expression=get_security_identifier( name_id, "ISIN") +total_source= +report_total= +date_format= +time_format= +Justify=left +width=171 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_to] +Name=Active to +Expression=get_field("UMI", name_id, "active_until") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=78 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=Instrument Group +Expression=get_field("UMI", name_id, "path_id") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=74 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_type] +Name=Instrument Type +Expression=get_field("UMI", name_id, "type_id") +total_source= +report_total= +date_format= +time_format= +Justify=left +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format inst_group] +Name=Instrument Group +Expression=get_field("UMPathFlat",path, 0, "upath") +total_source= +report_total= +date_format= +time_format= +Justify=left +width=146 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format path] +Name=Path +Expression=get_field("UMI", name_id, "path_id") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity] +Name=Maturity Date +Expression=get_field("UMIDateSetup" , name_id, "maturity_date") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=90 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format display] +Name=Display +Expression=if( ((maturity < today or active_to < today) and enabled = 1) or (active_to >= today and active_from <= today) )("Yes")("No") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_from] +Name=Active from +Expression=get_field("UMI", name_id, "active_since") +total_source= +report_total= +date_format= +time_format= +Justify=right +width=68 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_MARKETINFO.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.fkr new file mode 100644 index 0000000..8a01962 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.fkr @@ -0,0 +1,513 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=portfolio +Name=Portfolio Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Portfolio Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=sdm_id,sdm_state_name,sdm_state_id,sdm_edit_allowed,sdm_state_chg_allowed,sdm_reject_state_id,sdm_accept_state_id,sdm_daccept_state_id,sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,client_id,domain_id,opening_date,flags,valuate_flags,name,template_id,currency_id,closing_date,user_id,risk_rate,client_unit_id,flag2 +Selected=id,client_id,domain_id,opening_date,valuate_flags,flags,name,template_id,currency_id,closing_date,user_id,risk_rate,client_unit_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format sdm_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_edit_allowed] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_state_chg_allowed] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_reject_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_accept_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_daccept_state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_object_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_key_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_create_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_access_level] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_active] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_date] +Name=Opening Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=Flags +Expression= +report_total= +date_format= +time_format= +Justify=left +width=151 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format valuate_flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=151 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format closing_date] +Name=Closing Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=80 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format user_id] +Name=User +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format risk_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flag2] +Name=Flags +Expression=get_field("Portfolio",id,"flags") +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIO.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.fkr new file mode 100644 index 0000000..03d33e0 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-portfolio-access +Name=RAR Portfolio Access Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Portfolio Access Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=portfolio_id,user_id,ecbusrbdts +Selected=portfolio_id,user_id,ecbusrbdts +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=69 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ecbusrbdts] +Name=Ecbusrbdts +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=73 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOACCESS.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.fkr new file mode 100644 index 0000000..fc98295 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-portfolio-tree +Name=RAR Portfolio tree Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Portfolio tree Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=parent_id,child_id,factor +Selected=parent_id,child_id,factor +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format parent_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format child_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format factor] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PORTFOLIOTREE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.fkr new file mode 100644 index 0000000..0348e38 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.fkr @@ -0,0 +1,769 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-prices +Name=RAR Prices Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Prices Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,scenario_id,subscenario_id,period_id,period_1_id,period_2_id,date,date_time,source_id,currency_id,period_from,period_to,price_type,date_basis,price_1,price_2,price_3,price_4,price_5,price_6,price_7,price_8,price_9,price_10,target_date,type_id,name_id,item,mi_group,bid,ask,maturity_date,security_identifier,security_identifier_type,instrument_group,mid,price_type_csd,inst_group,bloomberg_tick_item,isin,reuters_item,scenario_code +Selected=name_id,period_id,price_1,price_2,mid,price_type_csd,currency_id,date,date_time,date_basis,isin,inst_group,reuters_item,scenario_id,source_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format scenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_time] +Name=Date Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=116 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format source_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_from] +Name=Period From +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=85 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format period_to] +Name=Period To +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=83 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=74 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_1] +Name=bid +Expression= +report_total= +date_format= +time_format= +Justify=right +width=84 +width_in_characters=16 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_2] +Name=ask +Expression= +report_total= +date_format= +time_format= +Justify=right +width=106 +width_in_characters=21 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format target_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item] +Name=item +Expression= +report_total= +date_format= +time_format= +Justify=left +width=89 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format mi_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bid] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ask] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=122 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mid] +Name=mid +Expression=(ask+bid)/2 +report_total= +date_format= +time_format= +Justify=right +width=103 +width_in_characters=20 +divider=1 +precision=10 +max_precision=10 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_type_csd] +Name=Price Type CSD +Expression=if(type_id='FX-RATE')( if( period_id='SPOT' )( 'FX Spot Rate' ) ( 'FX Forward Points' ) )( if( type_id='IR-RATE' )( 'Interest Rate' ) ( if(price_type='price@price%')('Price %')(if(price_type='yield@trading')('Yield')(price_type)) ) ) +report_total= +date_format= +time_format= +Justify=right +width=103 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format inst_group] +Name=Inst. Group +Expression=if(type_id='UM-PRICE')(get_field("UMPathFlat", get_field("UMI", name_id, "path_id"), 0, "upath")) +report_total= +date_format= +time_format= +Justify=right +width=108 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bloomberg_tick_item] +Name=BLOOMBERG-TICK item +Expression=get_field('ECBMarketInfoByString', 'BLOOMBERG-TICK', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=right +width=109 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin] +Name=Bloomberg Identifier +Expression=if(bloomberg_tick_item <> '') (bloomberg_tick_item) (if( left(name_id,5)= "EUBIS" or left(name_id,5) like "USBIS%" )( get_security_identifier( name_id, "SEDOL") ) ( if(get_security_identifier( name_id, "ISIN") <> '')(get_security_identifier( name_id, "ISIN")) (get_security_identifier( name_id, "TICKER")) )) +report_total= +date_format= +time_format= +Justify=right +width=98 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format reuters_item] +Name=REUTERS item +Expression=get_field('ECBMarketInfoByString', 'REUTERS', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format scenario_code] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.yml new file mode 100644 index 0000000..feacf3c --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_PRICES.yml @@ -0,0 +1,23 @@ + + +parameters: + + date_from: + value: t-5d + hidden: false + + date_to: + value: t + hidden: false + + scenario_id: + value: + - " " + - ECBRR + - FIXING + - FREEZE + - FRM + - FRMJPY + - SWAPMARGIN + + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.fkr new file mode 100644 index 0000000..10eac10 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-collateral-inventory +Name=RAR Collateral Inventory Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Collateral Inventory Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=valuation_date,number,umi_id +Selected=valuation_date,number,umi_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format valuation_date] +Name=Valuation Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=81 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=109 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.yml new file mode 100644 index 0000000..9764386 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARCOLLATERALINVENTORY.yml @@ -0,0 +1,8 @@ + + +parameters: + + date_from: + value: t + hidden: false + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.fkr new file mode 100644 index 0000000..304a19f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.fkr @@ -0,0 +1,129 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-transaction-property +Name=RAR Transaction Property Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR Transaction Property Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=number,value,type_id +Selected=number,value,type_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.fkr new file mode 100644 index 0000000..7251843 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.fkr @@ -0,0 +1,2737 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=transaction-limits +Name=Transaction Condition Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Transaction Condition Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=condition_set_id,condition_set_name,condition_set_type,condition_set_rule,condition_set_not_rule,condition_set_action,condition_set_since,condition_set_until,condition_description,condition_set_message,condition_set_user,condition_set_date,condition_set_breached,condition_set_active,condition_set_applicable,not_limit_attribute,condition_id,condition_type,condition_message,condition_reference_rate_id,condition_reference_value,condition_breached,condition_active,condition_applicable,not_condition,state_id,counterparty_name,instrument_group,lower_boundary,upper_boundary,rate_boundary_type,lower_value,upper_value,column_value,gap_set,transaction_gap,collateral_instrument,scenario,condition_log_date,condition_log_time,current_transaction_state,number,market_id,instrument_id,issuer_id,issuer_unit_id,currency_id,currency_2_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,maturity_date,expiry_date,opening_date,opening_time,closing_date,deal_rate,deal_price,nominal_rate,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,amount,agreed_amount,trading_unit,book_value,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,secondary_instrument_id,collateral_number,broker_id,cp_client_id,cp_client_unit_id,cp_trader_id,cp_portfolio_id,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,contexts,creator_tag,status,flags,flags_2,flags_3 +Selected=number,instrument_id,condition_set_id,condition_description,condition_set_breached,condition_reference_value,rate_boundary_type,lower_value,upper_value,amount,sign_id,date,condition_message,condition_reference_rate_id +Grouping= +Sorting=number +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format condition_set_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=226 +width_in_characters=45 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_rule] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_not_rule] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_action] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_since] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_until] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_description] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=460 +width_in_characters=92 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_message] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_user] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=85 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_breached] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_active] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_set_applicable] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_limit_attribute] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_message] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=576 +width_in_characters=115 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_reference_rate_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_reference_value] +Name=Condition Reference Value +Expression= +report_total= +date_format= +time_format= +Justify=right +width=90 +width_in_characters=18 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_breached] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_active] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_applicable] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_condition] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format counterparty_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format lower_boundary] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format upper_boundary] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_boundary_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format lower_value] +Name=Lower Value +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format upper_value] +Name=Upper Value +Expression= +report_total= +date_format= +time_format= +Justify=right +width=68 +width_in_characters=13 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format column_value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=81 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format gap_set] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_gap] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_instrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format scenario] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_log_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format condition_log_time] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format current_transaction_state] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=134 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=120 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format settlement_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expiry_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_time] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format closing_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.yml new file mode 100644 index 0000000..24921de --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_RARTRANSLIMITCOND.yml @@ -0,0 +1,9 @@ + + +parameters: + + log_date: + value: t + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_REINSTRUMENTDELIVERY.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_REINSTRUMENTDELIVERY.fkr new file mode 100644 index 0000000..0359cc6 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_REINSTRUMENTDELIVERY.fkr @@ -0,0 +1,145 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-umi-future-basket +Name=RAR UMI Future Basket Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR UMI Future Basket Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,instrument_id,conversion_factor,flags +Selected=id,instrument_id,conversion_factor,flags +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=82 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format conversion_factor] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=71 +width_in_characters=14 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=72 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.fkr new file mode 100644 index 0000000..80f78bc --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.fkr @@ -0,0 +1,2161 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-limit-report +Name=Limit Setup Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Limit Setup Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,domain_id,category_id,usage_logging_type,name,global_client_query_id,limit_user_group_id,comment,flags,template_id,limit_id,limit_currency_id,templ_id,active_from,active_to,gap_set_id,maturity_gap_from_id,maturity_gap_to_id,risk_gap_from_id,risk_gap_to_id,delivery_gap_from_id,delivery_gap_to_id,instrument_id,underlying_id,delivery_id,leg_id,currency_id,currency_class_id,portfolio_owner_id,portfolio_id,package_type_id,client_id,bank_id,account_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,minimum_expression,maximum_expression,rating_id,parent_id,client_group_id,client_main_group_id,risk_country_id,guarantee_level,guarantee_param_0,guarantee_param_1,guarantee_param_2,guarantee_param_3,guarantee_param_4,guarantee_param_5,guarantee_param_6,guarantee_param_7,guarantee_param_8,guarantee_param_9,guarantee_type_id,instrument_path_id,max_rating_code_id,min_rating_code_id,umi_max_rating_code_id,umi_min_rating_code_id,umi_rating_id,affect_gap_from_id,affect_gap_to_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,package_main_type_id,under_maturity_gap_from_id,under_maturity_gap_to_id,expression,active_from_time,active_to_time,time_zone_id,grouping,grouping_branch,grouping_branch_2,byportfolio,byinstrumentgroup,byinstrument,byaffectdate,bytransaction,bylegdeliveryinstrument,bypaymentdate,bydeliverygap,byclient,byclientgroup,byriskcountry,sublimit_id,limit_complete,adjusted_risk_gap_from_id,adjusted_risk_gap_to_id,leg_group,leg_path_id,position_type,id_sublimit +Selected=id_sublimit,limit_id,limit_currency_id,templ_id,active_from,active_to,gap_set_id,maturity_gap_from_id,maturity_gap_to_id,risk_gap_from_id,risk_gap_to_id,delivery_gap_from_id,delivery_gap_to_id,instrument_id,underlying_id,delivery_id,leg_id,currency_id,currency_class_id,portfolio_owner_id,portfolio_id,package_type_id,client_id,bank_id,account_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,minimum_expression,maximum_expression,rating_id,parent_id,client_group_id,client_main_group_id,risk_country_id,guarantee_level,guarantee_param_0,guarantee_param_1,guarantee_param_2,guarantee_param_3,guarantee_param_4,guarantee_param_5,guarantee_param_6,guarantee_param_7,guarantee_param_8,guarantee_param_9,guarantee_type_id,instrument_path_id,max_rating_code_id,min_rating_code_id,umi_max_rating_code_id,umi_min_rating_code_id,umi_rating_id,flags,affect_gap_from_id,affect_gap_to_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,package_main_type_id,under_maturity_gap_from_id,under_maturity_gap_to_id,adjusted_risk_gap_from_id,adjusted_risk_gap_to_id,leg_group,leg_path_id,position_type +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=12363955, 8999536, 0 +HeaderRowColors=11056316, 8999536 +DataRowColors=15724270, 0, 0 +TotalRowColors=12363955, 8999536, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=134 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format category_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=106 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format usage_logging_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format global_client_query_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=110 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_user_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=75 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=116 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format templ_id] +Name=Templ +Expression= +report_total= +date_format= +time_format= +Justify=left +width=72 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format active_from] +Name=Active From +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_to] +Name=Active To +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=67 +width_in_characters=13 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format gap_set_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format risk_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format risk_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format underlying_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_class_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_owner_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch0_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch3_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch4_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch5_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch6_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch7_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch8_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch9_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch10_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch11_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch12_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch13_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch14_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch15_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch16_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch17_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch18_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch19_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format minimum_expression] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=right +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maximum_expression] +Name= +Expression= +report_total= +date_format= +time_format= +Justify=right +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rating_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_main_group_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format risk_country_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_level] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_param_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantee_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_path_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format max_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format min_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_max_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_min_rating_code_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_rating_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format affect_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format affect_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format under_maturity_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format under_maturity_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expression] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=425 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_from_time] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_to_time] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format grouping] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=67 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format grouping_branch] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format grouping_branch_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byportfolio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byinstrumentgroup] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byinstrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byaffectdate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bytransaction] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bylegdeliveryinstrument] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bypaymentdate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=110 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bydeliverygap] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byclient] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byclientgroup] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format byriskcountry] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format sublimit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=251 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format limit_complete] +Name=Limit + Sublimit together +Expression=as_text(limit_id) + as_text(sublimit_id) +report_total= +date_format= +time_format= +Justify=left +width=419 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format adjusted_risk_gap_from_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format adjusted_risk_gap_to_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_path_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format position_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id_sublimit] +Name=ID +Expression=sublimit_id +report_total= +date_format= +time_format= +Justify=right +width=267 +width_in_characters=53 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_SUBLIMIT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.fkr new file mode 100644 index 0000000..0efdf8f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.fkr @@ -0,0 +1,2321 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=transactions-log +Name=Transaction Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Transaction Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_id,log_date,log_user,log_action,log_type,log_serial,log_caller,tags,number,market_id,instrument_id,issuer_id,issuer_unit_id,currency_id,currency_2_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,expiry_date,maturity_date,opening_date,opening_time,closing_date,deal_rate,deal_price,nominal_rate,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,amount,agreed_amount,trading_unit,book_value,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,broker_id,cp_client_id,cp_client_unit_id,cp_portfolio_id,cp_trader_id,secondary_instrument_id,collateral_number,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,type_name,state_name,instrument_group,log_date_utc,opening_time_utc,stamp +Selected=number,broker_id,cp_client_id,owner_id,portfolio_id,log_action,log_date_utc,param_2,currency_2_id,currency_id,param_1,param_5,param_7,param_15,param_8,accrued_interest,agreed_amount,book_value,closing_date,collateral_number,deal_price,deal_rate,expiry_date,param_19,param_18,param_17,log_user,maturity_date,param_6,param_9,nominal_rate,opening_date,opening_time_utc,param_4,parent_number,param_16,settlement_date,amount,date,param_3,param_10,param_11,param_12,param_13,param_14,param_0,state_id,type_id,trader_id,value_date,log_caller,kind_id,instrument_id,stamp,log_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format log_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_date] +Name=Log Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=165 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_user] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_action] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=130 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_serial] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_caller] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format tags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=123 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=81 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=136 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_date] +Name=Settlement Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=78 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name=Value Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expiry_date] +Name=Expiry Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=Maturity Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_date] +Name=Opening Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=96 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_time] +Name=Opening Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=118 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format closing_date] +Name=Closing Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=79 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +Name=Deal Rate +Expression= +report_total= +date_format= +time_format= +Justify=right +width=78 +width_in_characters=15 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +Name=Nominal Rate +Expression= +report_total= +date_format= +time_format= +Justify=right +width=76 +width_in_characters=15 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_name] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format log_date_utc] +Name=Log Date +Expression=log_date +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=right +width=154 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format opening_time_utc] +Name=Opening Time +Expression=opening_time +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=right +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format stamp] +Name=Stamp +Expression=log_id + 70000000000000000 +report_total= +date_format= +time_format= +Justify=right +width=169 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.yml new file mode 100644 index 0000000..b570b6b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_TRANSACTIONSLOG.yml @@ -0,0 +1,17 @@ + + +parameters: + + from_date: + value: t + hidden: false + + to_date: + value: t + hidden: false + + + date_type: + value: 1 + hidden: true + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.fkr new file mode 100644 index 0000000..86ac5da --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.fkr @@ -0,0 +1,1373 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb-instrument-originator +Name=Instrument Bond Report - Originator +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Bond Report - Originator +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=cover_percent<>0 and cover_percent<>"" +TotalsOnly=1 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,umi_id,currency_id,issuer_id,settlement_flags,amount_rounding,accrued_interest,minimum_price_denom,default_price_denom,sign_id,coupon_rate,issuer_unit_id,key_id,subtype_id,category_id,value_date,date_basis,fixing_price,kinds,cf_flags,cf_flags_2,cf_keep_flags,reference_id,referee_id,amount,actual_currency_id,actual_fx_rate,actual_amount,subcategory_id,sign,since_when,until_when,payment_date,realize_date,origin_id,origin_value,ex_date,reference_date,spread,cashflow_group,accrual_date_basis,fixing_rate_id,fixing_period_id,fixing_period_2_id,fixing_param,fixing_subscenario_id,fixing_date,fixing_from,fixing_to,fixing_price_2,fixing_quote,fixing_type,fixing_divider,fixing_cap,fixing_floor,expression,freeze_date,effective_fixing_date,fixing_processing_date,order_number,client_id,cover_percent,merged_col +Selected=flags,umi_id,client_id,cover_percent,order_number +Grouping=merged_col +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format domain_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=Flags +Expression= +total_merged_col=last(@flags) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format name] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format layout_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_since] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_until] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format buy_label] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sell_label] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format path_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format result_type_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format template_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_id] +Name=Umi +Expression= +total_merged_col=last(@umi_id) +report_total= +date_format= +time_format= +Justify=left +width=115 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_flags] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount_rounding] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format minimum_price_denom] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format default_price_denom] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format coupon_rate] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format key_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subtype_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format category_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kinds] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags_2] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_keep_flags] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format referee_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_currency_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_fx_rate] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format actual_amount] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subcategory_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format since_when] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format until_when] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realize_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format origin_value] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ex_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_group] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrual_date_basis] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_2_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_param] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_subscenario_id] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_from] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_to] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_price_2] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_quote] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_type] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_divider] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_cap] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_floor] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expression] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format freeze_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format effective_fixing_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_processing_date] +Name= +Expression= +total_merged_col= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format order_number] +Name=Order Number +Expression= +total_merged_col=last(@order_number) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format client_id] +Name=Client +Expression= +total_merged_col=last(@client_id) +report_total= +date_format= +time_format= +Justify=left +width=77 +width_in_characters=15 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format cover_percent] +Name=Cover Percent +Expression= +total_merged_col=last(@cover_percent) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format merged_col] +Name=mc +Expression=flags+ umi_id+client_id+ cover_percent+ order_number +total_merged_col= +report_total= +date_format= +time_format= +Justify=right +width=355 +width_in_characters=71 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMICREDITCLIENT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.fkr new file mode 100644 index 0000000..264da2e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.fkr @@ -0,0 +1,193 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rar-umi-security-code +Name=RAR UMISecurity Code Report (Webservice) +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=RAR UMISecurity Code Report (Webservice) +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=key_id,umi_id,standard_id,code,flags,stamp1,stampstring +Selected=key_id,umi_id,standard_id,code,flags,stamp1 +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format key_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format umi_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format standard_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format code] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format stamp1] +Name=Stamp +Expression= +report_total= +date_format= +time_format= +Justify=right +width=169 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format stampstring] +Name=Stamp test +Expression=truncate( stamp1,0 ) +report_total= +date_format= +time_format= +Justify=right +width=196 +width_in_characters=39 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYCODE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.fkr new file mode 100644 index 0000000..902f657 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.fkr @@ -0,0 +1,5533 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=instrument-extended +Name=Instrument Report Extended +HeaderLeft=[\n]%pagebreak +HeaderCenter=Instrument Report Extended +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=UMISecurityInfo_flags=0 +Fields=id,type_id,domain_id,flags,name,layout_id,active_since,active_until,buy_label,sell_label,path_id,result_type_id,template_id,instrument_group,security_identifier_type,security_identifier,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,UMIBond_currency_id,UMIBond_issuer_id,UMIBond_settlement_flags,UMIBond_flags,UMIBond_amount_rounding,UMIBond_accrued_interest,UMIBond_minimum_price_denom,UMIBond_default_price_denom,UMIBond_sign_id,UMIBond_coupon_rate,issuer_name_calc,risk_country,ultimate_parent,UMISecurityInfo_active_since,UMISecurityInfo_active_until,UMISecurityInfo_flags,UMISecurityInfo_outstanding_size,UMIBond_issuer_unit_id,par_value +Selected=id,UMISecurityInfo_active_since,UMISecurityInfo_active_until,UMISecurityInfo_flags,par_value,UMISecurityInfo_outstanding_size +Grouping= +Sorting= +GroupSortLevel=0 +GroupSortColumns= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=13160660, 0, 0 +HeaderRowColors=13160657, 0 +DataRowColors=15726583, 0, 0 +TotalRowColors=14410723, 0, 0 +DecimalSeparator=. +ThousandsSeparator=, +ExcelWorkbook= +MergeCellsVertically= +MergeCellsHorizontally= +SortDescending= +GroupSortDescending= +TotalsOnly=0 + + +[Parameters] +Fields=umi_id,instrument_group_id,instrument_type_id,instrument_issuer_id,umi_branch_code3,umi_active_from,umi_active_to,umi_active_from_from,umi_active_from_to,umi_active_to_from,umi_active_to_to,proc,tables +umi_id= +instrument_group_id= +instrument_type_id= +instrument_issuer_id= +umi_branch_code3= +umi_active_from= +umi_active_to= +umi_active_from_from= +umi_active_from_to= +umi_active_to_from= +umi_active_to_to= +proc=InstrumentReport +tables=UMIBond,UMISecurityInfo + + +[Format id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=150 +width_in_characters=30 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format type_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=118 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format domain_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format name] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=311 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format layout_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format active_since] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=85 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format active_until] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format buy_label] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format sell_label] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format path_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format result_type_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format template_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format instrument_group] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=171 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format security_identifier_type] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format security_identifier] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=104 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch0_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch1_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch2_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch3_id] +Name=Eligibility +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=81 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify=left +divider=1 + + +[Format branch4_id] +Name=Limit Bucket +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=89 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify=left +divider=1 + + +[Format branch5_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=114 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch6_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch7_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch8_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch9_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch10_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch11_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch12_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch13_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch14_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch15_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch16_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch17_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch18_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format branch19_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_calendar_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_holiday_calendar_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_gap_set_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_spot_days] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_spot_time_zone_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_spot_time] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_value_gap_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_value_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_maturity_gap_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_maturity_date] +Name=Maturity Date +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=85 +width_in_characters=17 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Expression= +divider=1 + + +[Format UMIDateSetup_expiry_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_expiry_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_issue_date] +Name=Issue Date +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=97 +width_in_characters=19 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Expression= +divider=1 + + +[Format UMIDateSetup_premium_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_premium_date_type_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIDateSetup_tenor_handler] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_currency_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_issuer_id] +Name=Issuer ID +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Expression= +divider=1 + + +[Format UMIBond_settlement_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_amount_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_accrued_interest] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=104 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_minimum_price_denom] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_default_price_denom] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_sign_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +Expression= +Justify= +divider=1 + + +[Format UMIBond_coupon_rate] +Name=Coupon Rate +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=85 +width_in_characters=10 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Expression= +divider=1 + + +[Format UMIDiscount_principal_subtype_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_date_basis] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_currency_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_amount_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_issuer_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_interest_type] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_price_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_rate_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIDiscount_sign_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer] +Name=Issuer +Expression=UMIBond_issuer_id + UMIDiscount_issuer_id +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format UMITradingYield_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingYield_yield_id] +Name=Yield/Price Method +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=186 +width_in_characters=37 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Justify=left + + +[Format UMITradingYield_price_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingYield_rate_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_category_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags_2] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_item_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_schedule_template_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_description] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_group_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_reference_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_currency_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_amount_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_rounding] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cashflow_group] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_date_basis] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_accrual_date_basis] +Name=Accrual Date Basis +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=left +width=81 +width_in_characters=16 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subcategory_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_type_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subtype_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_kind_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_start_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_end_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_set_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_method_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frequency] +Name=Coupons per Year +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=57 +width_in_characters=11 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Justify=right + + +[Format UMIBondSchedule_min_stub_length] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_first_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_penultimate_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_max_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_convention_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calendar_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_holidays_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_sign] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_method_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_client_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_currency_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_fx_rate] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_type] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=12 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_spread] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_factor] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_divider] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cap] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_floor] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_rate_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_period_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_subscenario_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_max_offset] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_convention_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_calendar_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_holidays_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_expression] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_pseudo] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_adjust_value_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_arrears] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_advance] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_roll_from_start] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_exclude_period] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frn_convention] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_long_stub] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_need_fixing] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_accrual_date_basis_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_accrual_date_basis_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_amount_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_amount_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_currency_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_currency_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_fx_rate_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_actual_fx_rate_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_adjust_value_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_adjust_value_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_amount_rounding_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_amount_rounding_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_method_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_method_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_rounding_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calculation_rounding_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calendar_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_calendar_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cap_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cap_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cashflow_group_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cashflow_group_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_category_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_category_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags_2_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_cf_flags_2_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_convention_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_convention_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_currency_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_currency_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_date_basis_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_date_basis_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_description_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_description_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_divider_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_divider_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_end_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_end_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_exclude_period_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_exclude_period_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_expression_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_expression_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_factor_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_factor_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_first_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_first_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_calendar_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_calendar_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_convention_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_convention_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_holidays_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_holidays_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_max_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_max_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_period_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_period_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_rate_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_rate_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_subscenario_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixing_subscenario_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_flags_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_flags_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_floor_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_floor_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frequency_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frequency_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frn_convention_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_frn_convention_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_set_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_gap_set_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_group_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_group_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_holidays_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_holidays_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_advance_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_advance_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_arrears_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_in_arrears_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_kind_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_kind_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_long_stub_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_long_stub_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_max_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_max_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_method_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_method_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_min_stub_length_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_min_stub_length_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_need_fixing_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_need_fixing_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_client_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_client_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_payment_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_penultimate_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_penultimate_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_pseudo_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_pseudo_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_offset_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_offset_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_type_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_rate_type_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_reference_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_reference_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_roll_from_start_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_roll_from_start_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_sign_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_sign_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_spread_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_spread_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_start_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_start_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subcategory_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subcategory_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subtype_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_subtype_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_type_id_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_type_id_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixed_roll_date] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixed_roll_date_dep] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBondSchedule_fixed_roll_date_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_key_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_contexts] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_active_since] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_active_until] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_valuation_usage] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_discount_usage] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_mtom_usage] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_fx_method_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_method_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_method_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_modes] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBaseValuationSetup_base_price] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format pricerounding] +Name=Price Rounding +Expression=UMIDiscount_price_rounding + UMITradingYield_price_rounding +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=90 +width_in_characters=18 +precision=9 +max_precision=9 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency] +Name=Currency +Expression=UMIDiscount_currency_id + UMIBond_currency_id +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format UMIQuoted_key_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIQuoted_active_since] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIQuoted_active_until] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIQuoted_price_type] +Name=Price Type +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=73 +width_in_characters=14 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 +Justify=left + + +[Format UMIQuoted_price_handler] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIQuoted_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIQuoted_currency_id] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_flags] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_minimum_bid_size] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_trading_unit] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_rounding_method] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_tick_size] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMITradingUnit_tick_value] +Name= +total_instrument_group= +total_currency= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFeeRule_flags] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFeeRule_commission_rule_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_issuer_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_currency_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_flags] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_cp_client_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_underlying_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_first_delivery_date] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_last_delivery_date] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_price_type] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_currency_2_id] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_date_basis] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_default_price_denom] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIFuture_minimum_price_denom] +Name= +total_instrument_group= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_name_calc] +Name=Issuer Name +Expression=get_field("Client", get_field("UMIBond", id, "issuer_id"), "name") +report_total= +date_format= +time_format= +Justify=left +width=187 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Format risk_country] +Name=Country +Expression=get_client_country( UMIBond_issuer_id ) +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format ultimate_parent] +Name=MP Ultimate Parent +Expression=get_client_group( UMIBond_issuer_id, "MP ULTIMATE PARENT") +report_total= +date_format= +time_format= +Justify=right +width=126 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format UMISecurityInfo_active_since] +Name=Active Since +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=114 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMISecurityInfo_active_until] +Name=Active Until +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=112 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMISecurityInfo_flags] +Name=Flags +Expression= +report_total= +date_format= +time_format= +Justify=right +width=92 +width_in_characters=18 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMISecurityInfo_outstanding_size] +Name=Outstanding size +Expression= +report_total= +date_format= +time_format= +Justify=right +width=114 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format UMIBond_issuer_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format par_value] +Name=Par Value +Expression=as_numeric(get_field("ECBsecurityInfo",id,"par_value")) +report_total= +date_format= +time_format= +Justify=right +width=83 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RAR_UMISECURITYINFO.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.fkr new file mode 100644 index 0000000..d42df38 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.fkr @@ -0,0 +1,508 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=reconciliation +Name=Reconciliation Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Reconciliation Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=payment_statement_id,case_id,item_type,currency_id,client_id,bank_id,account_id,recon_status,recon_group,recon_group_comment,recon_group_log,payment_date,amount,client_unit_id,bank_unit_id,cp_client_id,cp_client_unit_id,portfolio_id,case_reason_code,case_comment,rate,number,balance,interest,currency_class_id,instrument_id +Selected=account_id,amount,balance,bank_id,bank_unit_id,case_comment,case_id,case_reason_code,client_id,client_unit_id,cp_client_id,cp_client_unit_id,currency_class_id,currency_id,instrument_id,interest,item_type,number,payment_date,payment_statement_id,portfolio_id,rate,recon_group,recon_group_comment,recon_group_log,recon_status +Grouping= +Sorting=client_id,account_id,amount +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from_date,to_date,portfolio_id,owner_id,owner_unit_id,bank_id,bank_unit_id,currency_id,currency_class_id,currency_group_id +from_date= +to_date= +portfolio_id= +owner_id= +owner_unit_id= +bank_id= +bank_unit_id= +currency_id= +currency_class_id= +currency_group_id= + + +[Format payment_statement_id] +Name=payment_statement_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format case_id] +Name=case_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item_type] +Name=item_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=100 +width_in_characters=20 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name=client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_id] +Name=bank_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format account_id] +Name=account_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format recon_status] +Name=recon_status +Expression= +report_total= +date_format= +time_format= +Justify= +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format recon_group] +Name=recon_group +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format recon_group_comment] +Name=recon_group_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=148 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format recon_group_log] +Name=recon_group_log +Expression= +report_total= +date_format= +time_format= +Justify= +width=210 +width_in_characters=42 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=payment_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format amount] +Name=amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=112 +width_in_characters=22 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name=client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bank_unit_id] +Name=bank_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=cp_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=cp_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=portfolio_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format case_reason_code] +Name=case_reason_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format case_comment] +Name=case_comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate] +Name=rate +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format balance] +Name=balance +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format interest] +Name=interest +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_class_id] +Name=currency_class_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=instrument_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.yml new file mode 100644 index 0000000..51e1767 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RECONCILIATION.yml @@ -0,0 +1,17 @@ + +parameters: + + from_date: + value: t-10d + hidden: false + + + to_date: + value: t + hidden: false + + + portfolio_id: + value: ACC-AVG-TOP-ALL + hidden: true + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.fkr new file mode 100644 index 0000000..6d4216a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.fkr @@ -0,0 +1,179 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=role-portfolio-profile +Name=Role Portfolio Permission Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Role Portfolio Permission Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=role,inherited_role,access_profile_id,permission_id,access_profile_type_id,entity_id +Selected=role,inherited_role,access_profile_id,access_profile_type_id,permission_id,entity_id +Grouping= +Sorting=role,inherited_role,access_profile_id,access_profile_type_id,permission_id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=access_profile_type_id +access_profile_type_id= + + +[Format role] +Name=ROLE +Expression= +report_total= +date_format= +time_format= +Justify= +width=137 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format inherited_role] +Name=INHERITED_ROLE +Expression= +report_total= +date_format= +time_format= +Justify= +width=168 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format access_profile_id] +Name=ACCESS_PROFILE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=217 +width_in_characters=43 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format permission_id] +Name=PERMISSION_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=133 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format access_profile_type_id] +Name=ACCESS_PROFILE_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=185 +width_in_characters=37 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format entity_id] +Name=ENTITY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=220 +width_in_characters=44 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_ROLEPORTFOLIOPROFILE.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.fkr new file mode 100644 index 0000000..d0ecdc7 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.fkr @@ -0,0 +1,2674 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=rules +Name=Rules Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Rules Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,number,priority,portfolio_id,instrument_path_id,status,not_status,cf_leg_path_id,cf_flags,cf_not_flags,flags,name,client_id,client_main_group_id,client_group_id,user_id,currency_id,currency_2_id,instrument_id,secondary_instrument_id,package_main_type_id,package_type_id,issuer_id,issuer_main_group_id,issuer_group_id,cp_client_id,cp_client_main_group_id,cp_client_group_id,cp_portfolio_id,broker_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,contexts,not_contexts,transaction_flags,transaction_not_flags,sign_id,type_id,kind_id,state_id,cf_status,cf_not_status,cf_currency_id,cf_payment_currency_id,cf_sign,cf_type_id,cf_subtype_id,cf_category_id,cf_subcategory_id,cf_leg_group,cf_leg_id,cf_branch0_id,cf_branch1_id,cf_branch2_id,cf_branch3_id,cf_branch4_id,cf_flags_2,cf_not_flags_2,cf_kinds,cf_not_kinds,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,cf_branch10_id,cf_branch11_id,cf_branch12_id,cf_branch13_id,cf_branch14_id,cf_branch15_id,cf_branch16_id,cf_branch17_id,cf_branch18_id,cf_branch19_id,cf_branch20_id,cf_branch21_id,cf_branch22_id,cf_branch23_id,cf_branch24_id,cf_branch25_id,cf_branch26_id,cf_branch27_id,cf_branch28_id,cf_branch29_id,cf_branch30_id,cf_branch31_id,cf_branch32_id,cf_branch33_id,cf_branch34_id,cf_branch35_id,cf_branch36_id,cf_branch37_id,cf_branch38_id,cf_branch39_id,cf_branch5_id,cf_branch6_id,cf_branch7_id,cf_branch8_id,cf_branch9_id,client_unit_id,cp_client_unit_id,issuer_unit_id,cf_type_name +Selected=sdm_object_id,sdm_key_id,sdm_create_date,sdm_access_level,sdm_active,id,number,priority,portfolio_id,instrument_path_id,status,not_status,cf_leg_path_id,cf_flags,cf_not_flags,flags,name,client_id,client_main_group_id,client_group_id,user_id,currency_id,currency_2_id,instrument_id,secondary_instrument_id,package_main_type_id,package_type_id,issuer_id,issuer_main_group_id,issuer_group_id,cp_client_id,cp_client_main_group_id,cp_client_group_id,cp_portfolio_id,broker_id,branch0_id,branch1_id,branch2_id,branch3_id,branch4_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,contexts,not_contexts,transaction_flags,transaction_not_flags,sign_id,type_id,kind_id,state_id,cf_status,cf_not_status,cf_currency_id,cf_payment_currency_id,cf_sign,cf_type_id,cf_subtype_id,cf_category_id,cf_subcategory_id,cf_leg_group,cf_leg_id,cf_branch0_id,cf_branch1_id,cf_branch2_id,cf_branch3_id,cf_branch4_id,cf_flags_2,cf_not_flags_2,cf_kinds,cf_not_kinds,branch10_id,branch11_id,branch12_id,branch13_id,branch14_id,branch15_id,branch16_id,branch17_id,branch18_id,branch19_id,branch20_id,branch21_id,branch22_id,branch23_id,branch24_id,branch25_id,branch26_id,branch27_id,branch28_id,branch29_id,branch30_id,branch31_id,branch32_id,branch33_id,branch34_id,branch35_id,branch36_id,branch37_id,branch38_id,branch39_id,branch5_id,branch6_id,branch7_id,branch8_id,branch9_id,cf_branch10_id,cf_branch11_id,cf_branch12_id,cf_branch13_id,cf_branch14_id,cf_branch15_id,cf_branch16_id,cf_branch17_id,cf_branch18_id,cf_branch19_id,cf_branch20_id,cf_branch21_id,cf_branch22_id,cf_branch23_id,cf_branch24_id,cf_branch25_id,cf_branch26_id,cf_branch27_id,cf_branch28_id,cf_branch29_id,cf_branch30_id,cf_branch31_id,cf_branch32_id,cf_branch33_id,cf_branch34_id,cf_branch35_id,cf_branch36_id,cf_branch37_id,cf_branch38_id,cf_branch39_id,cf_branch5_id,cf_branch6_id,cf_branch7_id,cf_branch8_id,cf_branch9_id,client_unit_id,cp_client_unit_id,issuer_unit_id,cf_type_name +Grouping= +Sorting=id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= + + +[Format sdm_object_id] +Name=SDM_OBJECT_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format sdm_key_id] +Name=SDM_KEY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_create_date] +Name=SDM_CREATE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=158 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_access_level] +Name=SDM_ACCESS_LEVEL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sdm_active] +Name=SDM_ACTIVE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=NUMBER_ +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format priority] +Name=PRIORITY +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_path_id] +Name=INSTRUMENT_PATH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name=STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_status] +Name=NOT_STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_leg_path_id] +Name=CF_LEG_PATH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags] +Name=CF_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_not_flags] +Name=CF_NOT_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name=NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_id] +Name=CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_main_group_id] +Name=CLIENT_MAIN_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_group_id] +Name=CLIENT_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format user_id] +Name=USER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +Name=CURRENCY_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name=SECONDARY_INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name=PACKAGE_MAIN_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +Name=PACKAGE_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_main_group_id] +Name=ISSUER_MAIN_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_group_id] +Name=ISSUER_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=CP_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_main_group_id] +Name=CP_CLIENT_MAIN_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_group_id] +Name=CP_CLIENT_GROUP_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name=CP_PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name=BROKER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch0_id] +Name=BRANCH0_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch1_id] +Name=BRANCH1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch2_id] +Name=BRANCH2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch3_id] +Name=BRANCH3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch4_id] +Name=BRANCH4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=PARAM_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=PARAM_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=PARAM_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=PARAM_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=PARAM_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=PARAM_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=PARAM_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=PARAM_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=PARAM_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=PARAM_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=PARAM_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=PARAM_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=PARAM_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=PARAM_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=PARAM_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=PARAM_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=PARAM_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=PARAM_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=PARAM_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=PARAM_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name=CONTEXTS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format not_contexts] +Name=NOT_CONTEXTS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_flags] +Name=TRANSACTION_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_not_flags] +Name=TRANSACTION_NOT_FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name=SIGN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name=KIND_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=STATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_status] +Name=CF_STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_not_status] +Name=CF_NOT_STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_currency_id] +Name=CF_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_payment_currency_id] +Name=CF_PAYMENT_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_sign] +Name=CF_SIGN +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_type_id] +Name=CF_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_subtype_id] +Name=CF_SUBTYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_category_id] +Name=CF_CATEGORY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_subcategory_id] +Name=CF_SUBCATEGORY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_leg_group] +Name=CF_LEG_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_leg_id] +Name=CF_LEG_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch0_id] +Name=CF_BRANCH0_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch1_id] +Name=CF_BRANCH1_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch2_id] +Name=CF_BRANCH2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch3_id] +Name=CF_BRANCH3_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch4_id] +Name=CF_BRANCH4_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_flags_2] +Name=CF_FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_not_flags_2] +Name=CF_NOT_FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_kinds] +Name=CF_KINDS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_not_kinds] +Name=CF_NOT_KINDS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch10_id] +Name=BRANCH10_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch11_id] +Name=BRANCH11_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch12_id] +Name=BRANCH12_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch13_id] +Name=BRANCH13_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch14_id] +Name=BRANCH14_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch15_id] +Name=BRANCH15_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch16_id] +Name=BRANCH16_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch17_id] +Name=BRANCH17_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch18_id] +Name=BRANCH18_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch19_id] +Name=BRANCH19_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch20_id] +Name=BRANCH20_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch21_id] +Name=BRANCH21_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch22_id] +Name=BRANCH22_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch23_id] +Name=BRANCH23_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch24_id] +Name=BRANCH24_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch25_id] +Name=BRANCH25_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch26_id] +Name=BRANCH26_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch27_id] +Name=BRANCH27_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch28_id] +Name=BRANCH28_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch29_id] +Name=BRANCH29_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch30_id] +Name=BRANCH30_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch31_id] +Name=BRANCH31_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch32_id] +Name=BRANCH32_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch33_id] +Name=BRANCH33_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch34_id] +Name=BRANCH34_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch35_id] +Name=BRANCH35_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch36_id] +Name=BRANCH36_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch37_id] +Name=BRANCH37_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch38_id] +Name=BRANCH38_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch39_id] +Name=BRANCH39_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch5_id] +Name=BRANCH5_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch6_id] +Name=BRANCH6_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch7_id] +Name=BRANCH7_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch8_id] +Name=BRANCH8_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format branch9_id] +Name=BRANCH9_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch10_id] +Name=CF_BRANCH10_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch11_id] +Name=CF_BRANCH11_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch12_id] +Name=CF_BRANCH12_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch13_id] +Name=CF_BRANCH13_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch14_id] +Name=CF_BRANCH14_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch15_id] +Name=CF_BRANCH15_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch16_id] +Name=CF_BRANCH16_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch17_id] +Name=CF_BRANCH17_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch18_id] +Name=CF_BRANCH18_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch19_id] +Name=CF_BRANCH19_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch20_id] +Name=CF_BRANCH20_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch21_id] +Name=CF_BRANCH21_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch22_id] +Name=CF_BRANCH22_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch23_id] +Name=CF_BRANCH23_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch24_id] +Name=CF_BRANCH24_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch25_id] +Name=CF_BRANCH25_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch26_id] +Name=CF_BRANCH26_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch27_id] +Name=CF_BRANCH27_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch28_id] +Name=CF_BRANCH28_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch29_id] +Name=CF_BRANCH29_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch30_id] +Name=CF_BRANCH30_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch31_id] +Name=CF_BRANCH31_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch32_id] +Name=CF_BRANCH32_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch33_id] +Name=CF_BRANCH33_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch34_id] +Name=CF_BRANCH34_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch35_id] +Name=CF_BRANCH35_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch36_id] +Name=CF_BRANCH36_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch37_id] +Name=CF_BRANCH37_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch38_id] +Name=CF_BRANCH38_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch39_id] +Name=CF_BRANCH39_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch5_id] +Name=CF_BRANCH5_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch6_id] +Name=CF_BRANCH6_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch7_id] +Name=CF_BRANCH7_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch8_id] +Name=CF_BRANCH8_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_branch9_id] +Name=CF_BRANCH9_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format client_unit_id] +Name=CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=CP_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name=ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cf_type_name] +Name=CF_TYPE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_RULES.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.fkr new file mode 100644 index 0000000..15094c7 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.fkr @@ -0,0 +1,364 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=sdm-entity-state +Name=SDM Entity State Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=SDM Entity State Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,key_id,state_id,state_name,state_edit_allowed,no_state_transition,category,status,entity_name,entered_by,updated_by,initiated_by,entity_type,state_log_updated_by,state_log_from_state,state_log_to_state,state_log_update_time +Selected=id,key_id,state_id,state_name,state_edit_allowed,no_state_transition,category,status,entity_name,entered_by,updated_by,initiated_by,entity_type,state_log_updated_by,state_log_from_state,state_log_to_state,state_log_update_time +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, +DigitGrouping=03 + + +[Parameters] +Fields=date_from,date_to,entered_by,initiated_by,updated_by,mode,state,entity_type +date_from=t +date_to=t +entered_by= +initiated_by= +updated_by= +mode=EDITOR +state= +entity_type= + + +[Format id] +Name=ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format key_id] +Name=KEY_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_id] +Name=STATE_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_name] +Name=STATE_NAME +Expression= +report_total= +date_format= +time_format= +Justify=left +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_edit_allowed] +Name=STATE_EDIT_ALLOWED +Expression= +report_total= +date_format= +time_format= +Justify=right +width=117 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format no_state_transition] +Name=NO_STATE_TRANSITION +Expression= +report_total= +date_format= +time_format= +Justify=right +width=139 +width_in_characters=27 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format category] +Name=CATEGORY +Expression= +report_total= +date_format= +time_format= +Justify=left +width=105 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format status] +Name=STATUS +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format entity_name] +Name=ENTITY_NAME +Expression= +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format entered_by] +Name=ENTERED_BY +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format updated_by] +Name=UPDATED_BY +Expression= +report_total= +date_format= +time_format= +Justify=left +width=134 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format initiated_by] +Name=INITIATED_BY +Expression= +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format entity_type] +Name=ENTITY_TYPE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=165 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_log_updated_by] +Name=STATE_LOG_UPDATED_BY +Expression= +report_total= +date_format= +time_format= +Justify=left +width=158 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_log_from_state] +Name=STATE_LOG_FROM_STATE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=178 +width_in_characters=35 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_log_to_state] +Name=STATE_LOG_TO_STATE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=166 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format state_log_update_time] +Name=STATE_LOG_UPDATE_TIME +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=169 +width_in_characters=33 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.yml new file mode 100644 index 0000000..7e301e6 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SDM_ENTITY_STATE.yml @@ -0,0 +1,17 @@ + +parameters: + + date_from: + value: select("w_ODS_TMS_SDM_ENTITY_STATE", "to_char(MAX(STATE_LOG_UPDATE_TIME), 'YYYYMMDD')||' - 000000'") + hidden: false + + + date_to: + value: t-1d + hidden: false + + + mode: + value: EDITOR + hidden: true + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.fkr new file mode 100644 index 0000000..091db8c --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.fkr @@ -0,0 +1,808 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=security-position +Name=Security Positions +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Security Positions +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=date,portfolio_id,instrument_id,contexts,currency_id,package_main_type_id,package_type_id,package_id,inflow_nominal_amount,outflow_nominal_amount,nominal_amount,inflow_units,outflow_units,units,inflow_book_value,outflow_book_value,book_value,redemption_amount,accrued_profit,indexation_profit,discount_premium,accrued_book_value,realized_mtom_profit,realized_accrued_profit,realized_indexation_profit,bvc_mtom_profit,mtom_profit,market_value,accrued_interest,accrued_interest_bought,accrued_interest_sold,yield,price,market_quote,index_value,issuer_id,issue_date,maturity_date,isin_code,accrual_method,accrual_interest_type,accrual_date_basis,input_date,flags,face_amount +Selected=date,portfolio_id,instrument_id,contexts,currency_id,package_main_type_id,package_type_id,package_id,inflow_nominal_amount,outflow_nominal_amount,nominal_amount,inflow_units,outflow_units,units,inflow_book_value,outflow_book_value,book_value,redemption_amount,accrued_profit,indexation_profit,discount_premium,accrued_book_value,realized_mtom_profit,realized_accrued_profit,realized_indexation_profit,bvc_mtom_profit,mtom_profit,market_value,accrued_interest,accrued_interest_bought,accrued_interest_sold,yield,price,market_quote,index_value,issuer_id,issue_date,maturity_date,isin_code,accrual_method,accrual_interest_type,accrual_date_basis,input_date,flags,face_amount +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=portfolio_id,date_from,date_to,instrument_id,instrument_type_id,contexts +portfolio_id= +date_from= +date_to= +instrument_id= +instrument_type_id= +contexts= + + +[Format date] +Name=DATE_ +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format contexts] +Name=CONTEXTS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name=PACKAGE_MAIN_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format package_type_id] +Name=PACKAGE_TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format package_id] +Name=PACKAGE_ID +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format inflow_nominal_amount] +Name=INFLOW_NOMINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format outflow_nominal_amount] +Name=OUTFLOW_NOMINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_amount] +Name=NOMINAL_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format inflow_units] +Name=INFLOW_UNITS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format outflow_units] +Name=OUTFLOW_UNITS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name=UNITS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format inflow_book_value] +Name=INFLOW_BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format outflow_book_value] +Name=OUTFLOW_BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name=BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format redemption_amount] +Name=REDEMPTION_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_profit] +Name=ACCRUED_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format indexation_profit] +Name=INDEXATION_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format discount_premium] +Name=DISCOUNT_PREMIUM +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_book_value] +Name=ACCRUED_BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realized_mtom_profit] +Name=REALIZED_MTOM_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realized_accrued_profit] +Name=REALIZED_ACCRUED_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format realized_indexation_profit] +Name=REALIZED_INDEXATION_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bvc_mtom_profit] +Name=BVC_MTOM_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mtom_profit] +Name=MTOM_PROFIT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_value] +Name=MARKET_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name=ACCRUED_INTEREST +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest_bought] +Name=ACCRUED_INTEREST_BOUGHT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest_sold] +Name=ACCRUED_INTEREST_SOLD +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format yield] +Name=YIELD +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price] +Name=PRICE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format market_quote] +Name=MARKET_QUOTE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format index_value] +Name=INDEX_VALUE +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format issue_date] +Name=ISSUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin_code] +Name=ISIN_CODE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format accrual_method] +Name=ACCRUAL_METHOD +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format accrual_interest_type] +Name=ACCRUAL_INTEREST_TYPE +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format accrual_date_basis] +Name=ACCRUAL_DATE_BASIS +Expression= +report_total= +date_format= +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format input_date] +Name=INPUT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format face_amount] +Name=FACE_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.yml new file mode 100644 index 0000000..c040862 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SECURITYPOSITION.yml @@ -0,0 +1,20 @@ + +parameters: + + date_from: + value: t-10d + hidden: false + + + date_to: + value: t + hidden: false + + + portfolio_id: + value: ACC-AVG-TOP-ALL + hidden: true + + contexts: + value: 2130706432 + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.fkr new file mode 100644 index 0000000..3ac225f --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.fkr @@ -0,0 +1,1466 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=settlement-cashflows +Name=Settlement Cashflows Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Settlement Cashflows Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,settlement_type,delivery_instrument,payment_date,currency_id,class_id,amount,split_source_id,transfer_method,advice_method,state_id,state_id_sort_order,accepted_by,netting_rule_id,number,portfolio_id,local_client_id,local_client_unit_id,local_client_swift_code,local_bank_1_id,local_bank_1_unit_id,local_bank_1_swift_code,local_account_1_id,local_bank_2_id,local_bank_2_swift_code,local_account_2_id,local_bank_3_id,local_bank_3_swift_code,local_account_3_id,local_bank_4_id,local_bank_4_swift_code,local_account_4_id,other_client_id,other_client_unit_id,other_client_swift_code,other_bank_1_id,other_bank_1_unit_id,other_bank_1_swift_code,other_account_1_id,other_bank_2_id,other_bank_2_swift_code,other_account_2_id,other_bank_3_id,other_bank_3_swift_code,other_account_3_id,other_bank_4_id,other_bank_4_swift_code,other_account_4_id,settlement_client_id,settlement_client_swift_code,owner_id,owner_unit_id,cp_client_id,cp_client_unit_id,flags,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,cashflow_amount,cashflow_id,cashflow_currency_id,cashflow_type_name,payment_id,call_movement_trader_id,cashflow_param_0,cashflow_param_1,cashflow_param_2,cashflow_param_3,cashflow_param_4 +Selected=accepted_by,advice_method,amount,call_movement_trader_id,cashflow_amount,cashflow_currency_id,cashflow_id,cashflow_param_0,cashflow_param_1,cashflow_param_2,cashflow_param_3,cashflow_param_4,cashflow_type_name,class_id,cp_client_id,cp_client_unit_id,currency_id,delivery_instrument,flags,id,local_account_1_id,local_account_2_id,local_account_3_id,local_account_4_id,local_bank_1_id,local_bank_1_swift_code,local_bank_1_unit_id,local_bank_2_id,local_bank_2_swift_code,local_bank_3_id,local_bank_3_swift_code,local_bank_4_id,local_bank_4_swift_code,local_client_id,local_client_swift_code,local_client_unit_id,netting_rule_id,number,other_account_1_id,other_account_2_id,other_account_3_id,other_account_4_id,other_bank_1_id,other_bank_1_swift_code,other_bank_1_unit_id,other_bank_2_id,other_bank_2_swift_code,other_bank_3_id,other_bank_3_swift_code,other_bank_4_id,other_bank_4_swift_code,other_client_id,other_client_swift_code,other_client_unit_id,owner_id,owner_unit_id,param_0,param_1,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,payment_date,payment_id,portfolio_id,settlement_client_id,settlement_client_swift_code,settlement_type,split_source_id,state_id,state_id_sort_order,transfer_method +Grouping= +Sorting=id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=from_date,to_date,number,settlement_id,portfolio_id,currency_id,currency_class_id,currency_group_id +from_date= +to_date= +number= +settlement_id= +portfolio_id= +currency_id= +currency_class_id= +currency_group_id= + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_type] +Name=settlement_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_instrument] +Name=delivery_instrument +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=payment_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=199 +width_in_characters=39 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format class_id] +Name=class_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=98 +width_in_characters=19 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format split_source_id] +Name=split_source_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transfer_method] +Name=transfer_method +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format advice_method] +Name=advice_method +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=state_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id_sort_order] +Name=state_id_sort_order +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accepted_by] +Name=accepted_by +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_rule_id] +Name=netting_rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=portfolio_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_id] +Name=local_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_unit_id] +Name=local_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_swift_code] +Name=local_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_id] +Name=local_bank_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_unit_id] +Name=local_bank_1_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_swift_code] +Name=local_bank_1_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_1_id] +Name=local_account_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_id] +Name=local_bank_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_swift_code] +Name=local_bank_2_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_2_id] +Name=local_account_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_id] +Name=local_bank_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_swift_code] +Name=local_bank_3_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_3_id] +Name=local_account_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_id] +Name=local_bank_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_swift_code] +Name=local_bank_4_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_4_id] +Name=local_account_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_id] +Name=other_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_unit_id] +Name=other_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_swift_code] +Name=other_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_id] +Name=other_bank_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_unit_id] +Name=other_bank_1_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_swift_code] +Name=other_bank_1_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_1_id] +Name=other_account_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_id] +Name=other_bank_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_swift_code] +Name=other_bank_2_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_2_id] +Name=other_account_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_id] +Name=other_bank_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_swift_code] +Name=other_bank_3_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_3_id] +Name=other_account_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_id] +Name=other_bank_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_swift_code] +Name=other_bank_4_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_4_id] +Name=other_account_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_client_id] +Name=settlement_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_client_swift_code] +Name=settlement_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name=owner_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name=owner_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=cp_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=cp_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=param_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=param_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=param_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=param_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=param_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=param_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=param_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=param_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=param_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=param_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=param_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=param_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=param_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=param_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=param_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=param_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=param_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=85 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=param_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=param_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=param_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_amount] +Name=cashflow_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_id] +Name=cashflow_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_currency_id] +Name=cashflow_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_type_name] +Name=cashflow_type_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_id] +Name=payment_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format call_movement_trader_id] +Name=call_movement_trader_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_param_0] +Name=cashflow_param_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_param_1] +Name=cashflow_param_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_param_2] +Name=cashflow_param_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_param_3] +Name=cashflow_param_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_param_4] +Name=cashflow_param_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.yml new file mode 100644 index 0000000..c974ea6 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTCASHFLOW.yml @@ -0,0 +1,13 @@ + +parameters: + + from_date: + value: t-10d + hidden: false + + + to_date: + value: t + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.fkr new file mode 100644 index 0000000..e4a981e --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.fkr @@ -0,0 +1,2051 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=settlement-log +Name=Settlement Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Settlement Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_date,log_user,log_action,log_event,log_type,id,owner_id,payment_date,currency_id,amount,sign,local_client_id,other_client_id,settlement_type_id,state_id,flags,log_caller,number,group_id,instrument_id,instrument_path_id,portfolio_id,orig_currency_id,orig_amount,split_source_id,local_bank_1_id,local_bank_1_swift_code,local_bank_2_id,local_bank_2_swift_code,local_bank_3_id,local_bank_3_swift_code,local_bank_4_id,local_bank_4_swift_code,local_account_1_id,local_account_2_id,local_account_3_id,local_account_4_id,other_bank_1_id,other_bank_1_swift_code,other_bank_2_id,other_bank_2_swift_code,other_bank_3_id,other_bank_3_swift_code,other_bank_4_id,other_bank_4_swift_code,other_account_1_id,other_account_2_id,other_account_3_id,other_account_4_id,transfer_type,advice_type,message_type,comment,comment_1,comment_2,comment_3,comment_4,netting_rule_id,face_amount,delivery_units,trading_unit,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,date_time,trmswift_message,trmswift_timestamp,netting_cutoff_time,error,local_client_swift_code,other_client_swift_code,release_cutoff_time,accepted_by,netting_flags,netting_group_id,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,settlement_client_id,settlement_client_swift_code,cashflow_subtype_id,cashflow_type_id,cp_client_id,cp_client_unit_id,local_bank_1_unit_id,local_client_unit_id,netting_criteria_id,on_behalf_account_id,on_behalf_id,on_behalf_swift_code,other_bank_1_unit_id,other_client_unit_id,owner_unit_id,uetr,last_user_id,unique_id,message_id,source_type_id,sender,receiver,status_id,code,description,action,action_date,related_message_type,related_funct_type,related_trmswift_id +Selected=accepted_by,action,action_date,advice_type,amount,cashflow_subtype_id,cashflow_type_id,code,comment,comment_1,comment_2,comment_3,comment_4,cp_client_id,cp_client_unit_id,currency_id,date_time,delivery_units,description,error,face_amount,flags,group_id,id,instrument_id,instrument_path_id,last_user_id,local_account_1_id,local_account_2_id,local_account_3_id,local_account_4_id,local_bank_1_id,local_bank_2_id,local_bank_1_swift_code,local_bank_1_unit_id,local_bank_2_swift_code,local_bank_3_id,local_bank_3_swift_code,local_bank_4_id,local_bank_4_swift_code,local_client_id,local_client_swift_code,local_client_unit_id,log_action,log_caller,log_date,log_event,log_type,log_user,message_id,message_type,netting_criteria_id,netting_cutoff_time,netting_flags,netting_group_id,netting_rule_id,number,on_behalf_account_id,on_behalf_id,on_behalf_swift_code,orig_amount,orig_currency_id,other_account_1_id,other_account_2_id,other_account_3_id,other_account_4_id,other_bank_1_id,other_bank_1_swift_code,other_bank_1_unit_id,other_bank_2_id,other_bank_2_swift_code,other_bank_3_id,other_bank_3_swift_code,other_bank_4_id,other_bank_4_swift_code,other_client_id,other_client_swift_code,other_client_unit_id,owner_id,owner_unit_id,param_0,param_1,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,payment_date,portfolio_id,receiver,related_funct_type,related_message_type,related_trmswift_id,release_cutoff_time,sender,settlement_client_id,settlement_client_swift_code,settlement_type_id,sign,source_type_id,split_source_id,state_id,status_id,trading_unit,transfer_type,trmswift_message,trmswift_timestamp,uetr,unique_id +Grouping= +Sorting=id,log_date +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields=log_period_start,log_period_end,transaction_number,payment_id,payment_date,local_client_id,local_client_unit_id,local_bank_1_id,local_bank_1_unit_id,local_account_1_id,other_client_id,other_client_unit_id,other_bank_1_id,other_bank_1_unit_id,other_account_1_id,transfer_type,user_id +log_period_start= +log_period_end= +transaction_number= +payment_id= +payment_date= +local_client_id= +local_client_unit_id= +local_bank_1_id= +local_bank_1_unit_id= +local_account_1_id= +other_client_id= +other_client_unit_id= +other_bank_1_id= +other_bank_1_unit_id= +other_account_1_id= +transfer_type= +user_id= + + +[Format log_date] +Name=log_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_user] +Name=log_user +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_action] +Name=log_action +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_event] +Name=log_event +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_type] +Name=log_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name=owner_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format payment_date] +Name=payment_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign] +Name=sign +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_id] +Name=local_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_id] +Name=other_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_type_id] +Name=settlement_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=state_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_caller] +Name=log_caller +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=number +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format group_id] +Name=group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=instrument_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_path_id] +Name=instrument_path_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=portfolio_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format orig_currency_id] +Name=orig_currency_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format orig_amount] +Name=orig_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format split_source_id] +Name=split_source_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_id] +Name=local_bank_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_swift_code] +Name=local_bank_1_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_id] +Name=local_bank_1_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_2_swift_code] +Name=local_bank_2_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_id] +Name=local_bank_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_3_swift_code] +Name=local_bank_3_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_id] +Name=local_bank_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_4_swift_code] +Name=local_bank_4_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_1_id] +Name=local_account_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_2_id] +Name=local_account_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_3_id] +Name=local_account_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_account_4_id] +Name=local_account_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_id] +Name=other_bank_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_swift_code] +Name=other_bank_1_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_id] +Name=other_bank_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_2_swift_code] +Name=other_bank_2_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_id] +Name=other_bank_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_3_swift_code] +Name=other_bank_3_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_id] +Name=other_bank_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_4_swift_code] +Name=other_bank_4_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_1_id] +Name=other_account_1_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_2_id] +Name=other_account_2_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_3_id] +Name=other_account_3_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_account_4_id] +Name=other_account_4_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transfer_type] +Name=transfer_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format advice_type] +Name=advice_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format message_type] +Name=message_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name=comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_1] +Name=comment_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_2] +Name=comment_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_3] +Name=comment_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_4] +Name=comment_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_rule_id] +Name=netting_rule_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format face_amount] +Name=face_amount +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format delivery_units] +Name=delivery_units +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name=trading_unit +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=param_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=param_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=param_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=param_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=param_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=param_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=param_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=param_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=param_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=param_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_time] +Name=date_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trmswift_message] +Name=trmswift_message +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trmswift_timestamp] +Name=trmswift_timestamp +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_cutoff_time] +Name=netting_cutoff_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format error] +Name=error +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_swift_code] +Name=local_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_swift_code] +Name=other_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format release_cutoff_time] +Name=release_cutoff_time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accepted_by] +Name=accepted_by +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_flags] +Name=netting_flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_group_id] +Name=netting_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=param_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=param_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=param_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=param_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=param_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=param_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=param_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=param_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=param_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=param_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_client_id] +Name=settlement_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_client_swift_code] +Name=settlement_client_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_subtype_id] +Name=cashflow_subtype_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cashflow_type_id] +Name=cashflow_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=cp_client_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=cp_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_bank_1_unit_id] +Name=local_bank_1_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format local_client_unit_id] +Name=local_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format netting_criteria_id] +Name=netting_criteria_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format on_behalf_account_id] +Name=on_behalf_account_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format on_behalf_id] +Name=on_behalf_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format on_behalf_swift_code] +Name=on_behalf_swift_code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_bank_1_unit_id] +Name=other_bank_1_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format other_client_unit_id] +Name=other_client_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name=owner_unit_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format uetr] +Name=uetr +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format last_user_id] +Name=last_user_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format unique_id] +Name=unique_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format message_id] +Name=message_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format source_type_id] +Name=source_type_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sender] +Name=sender +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format receiver] +Name=receiver +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status_id] +Name=status_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format code] +Name=code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format description] +Name=description +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format action] +Name=action +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format action_date] +Name=action_date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format related_message_type] +Name=related_message_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format related_funct_type] +Name=related_funct_type +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format related_trmswift_id] +Name=related_trmswift_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.yml new file mode 100644 index 0000000..92335c9 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_SETTLEMENTLOG.yml @@ -0,0 +1,13 @@ + +parameters: + + log_period_start: + value: t-1d + hidden: false + + + log_period_end:: + value: t + hidden: false + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.fkr new file mode 100644 index 0000000..c4251bb --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.fkr @@ -0,0 +1,769 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=ecb_prices +Name=Rate Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Rate Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=enabled_flag<>0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=target_date,type_id,name_id,id,scenario_id,subscenario_id,source_id,item,mi_group,currency_id,period_from,period_to,period_id,period_1_id,period_2_id,price_type,date_basis,date,date_time,price_1,price_2,price_3,price_4,price_5,price_6,price_7,price_8,price_9,price_10,bid,ask,maturity_date,security_identifier,security_identifier_type,instrument_group,mid,isin,bloomberg_tick_item,reuters_item,price_type_csd,instr_group,enabled_flag +Selected=name_id,period_id,bid,ask,mid,price_type_csd,currency_id,date,date_time,isin,reuters_item,instr_group,scenario_id +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=14005927, 6441018, 0 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] + + +[Format target_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=141 +width_in_characters=28 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format scenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format subscenario_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format source_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=117 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format item] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format default_item] +Name=Reuters Code +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mi_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_from] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_to] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format period_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=150 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_type_csd] +Name=Price Type CSD +Expression=if(type_id='FX-RATE')( if( period_id='SPOT' )( 'FX Spot Rate' ) ( 'FX Forward Points' ) )( if( type_id='IR-RATE' )( 'Interest Rate' ) ( if(price_type='price@price%')('Price %')(if(price_type='yield@trading')('Yield')(price_type)) ) ) +report_total= +date_format= +time_format= +Justify= +width=150 +width_in_characters=30 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format= +Justify=left +width=123 +width_in_characters=24 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format date_time] +Name=Date Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=176 +width_in_characters=35 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format price_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format price_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format bid] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format ask] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name=Security Identifier +Expression= +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format bloomberg_tick_item] +Name=BLOOMBERG-TICK item +Expression=get_field('ECBMarketInfoByString', 'BLOOMBERG-TICK', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format reuters_item] +Name=REUTERS item +Expression=get_field('ECBMarketInfoByString', 'REUTERS', name_id, period_id, 'item') +report_total= +date_format= +time_format= +Justify=left +width=132 +width_in_characters=26 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin] +Name=Bloomberg Identifier +Expression=if(bloomberg_tick_item <> '') (bloomberg_tick_item) (if( left(name_id,5)= "EUBIS" or left(name_id,5) like "USBIS%" )( get_security_identifier( name_id, "SEDOL") ) ( if(get_security_identifier( name_id, "ISIN") <> '')(get_security_identifier( name_id, "ISIN")) (get_security_identifier( name_id, "TICKER")) )) +report_total= +date_format= +time_format= +Justify=left +width=111 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=Instrument Group +Expression= +report_total= +date_format= +time_format= +Justify=right +width=247 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format mid] +Name=Mid +Expression=(ask+bid)/2 +report_total= +date_format= +time_format= +Justify=right +width=99 +width_in_characters=19 +divider=1 +precision=3 +max_precision=3 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format instr_group] +Name=Instr. Group +Expression=if(type_id='UM-PRICE')(get_field("UMPathFlat", get_field("UMI", name_id, "path_id"), 0, "upath")) +report_total= +date_format= +time_format= +Justify=right +width=159 +width_in_characters=31 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format enabled_flag] +Name=enabled flag +Expression=as_int( get_field('ECBMarketInfoByString', 'REUTERS', name_id, period_id, 'enabled_p')+get_field('ECBMarketInfoByString', 'BLOOMBERG-TICK', name_id, period_id, 'enabled_p') ) +report_total= +date_format= +time_format= +Justify=right +width=81 +width_in_characters=10 +divider=1 +precision=9 +max_precision=9 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.yml new file mode 100644 index 0000000..f017675 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.yml @@ -0,0 +1,18 @@ + + +parameters: + + date_from: + value: t + hidden: false + + date_to: + value: t + hidden: false + + scenario_id: + value: + - " " + - FREEZE + + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.fkr new file mode 100644 index 0000000..fcbd596 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.fkr @@ -0,0 +1,1595 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=transactions-delta +Name=Transactions Delta Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Transactions Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=number > 0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=last_log_id,number,instrument_id,issuer_id,currency_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,maturity_date,expiry_date,opening_date,opening_time,closing_date,deal_price,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,trading_unit,accrued_interest,portfolio_id,owner_id,trader_id,secondary_instrument_id,collateral_amount,collateral_number,cp_client_id,cp_trader_id,cp_portfolio_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,broker_id,instrument_group,package_main_type_id,package_type_id,package_id,comment,comment_2,comment_3,comment_4,fixing_rate,fixing_period,fixing_offset,date_basis,maturity_code,premium_type,signature_date,fixing_rate_2,fixing_period_2,fixing_offset_2,date_basis_2,fixing_calendar,fixing_calendar_2,frequency,frequency_2,method_value,method_value_2,spread,spread_2,leg_1_id,leg_2_id,currency_2_id,amount_2,nominal_rate,rate_type,nominal_rate_2,rate_type_2,cap,floor,top_portfolio_id,security_identifier_type,security_identifier,deal_rate,book_value,amount,agreed_amount,face_amount,pool_factor,transaction_tag,fx_finance_rate,quote_currency_yield,idx,issue_index,index_ratio,fx_fwd_points,fx_rate,deal_price2,isin_code,issuer_unit_id,owner_unit_id,cp_client_unit_id,state_id_sort_order,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,last_log_date +Selected=last_log_id,last_log_date,opening_date,date,value_date,maturity_date,maturity_code,issuer_id,amount,book_value,currency_id,currency_2_id,instrument_group,instrument_id,security_identifier,portfolio_id,number,cp_client_id,deal_price2,deal_rate,type_id,sign_id,fx_rate,fx_fwd_points,state_id,param_0,isin_code,param_13,accrued_interest,agreed_amount,amount_2,audit_number,batch_id,cap,closing_date,collateral_amount,collateral_number,comment,comment_2,comment_3,comment_4,contexts,cp_portfolio_id,cp_trader_id,cp_client_unit_id,creator_tag,date_basis,date_basis_2,deal_price,param_1,expiry_date,fx_finance_rate,mattr_4,face_amount,sattr_1,fixing_calendar,fixing_calendar_2,fixing_offset,fixing_offset_2,fixing_period,fixing_period_2,fixing_rate,fixing_rate_2,flags,flags_2,flags_3,floor,frequency,frequency_2,idx,index_ratio,issue_index,issuer_unit_id,kind_id,leg_1_id,leg_2_id,sattr_4,logical_number,sattr_3,margin1,margin2,param_11,method_value,method_value_2,param_9,nominal_rate,nominal_rate_2,opening_time,iattr_6,param_10,param_4,owner_id,owner_unit_id,package_id,package_main_type_id,package_type_id,param_18,param_3,param_5,param_6,param_7,param_8,param_19,parent_number,sattr_13,pool_factor,premium_amount,premium_currency_id,premium_date,premium_price,premium_type,quote_currency_yield,rate_type,rate_type_2,reference_number,param_2,secondary_instrument_id,security_identifier_type,param_16,settlement_date,signature_date,spread,spread_2,state_id_sort_order,status,broker_id,top_portfolio_id,trader_id,trading_unit,sattr_2,param_12,param_14,param_15,param_17,transaction_tag,units +Grouping= +Sorting=number +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=14005927, 6441018, 0 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 0 +DecimalSeparator=. +ThousandsSeparator=, +TotalsOnly=0 + + +[Parameters] +Fields=portfolio_id,instrument_id,instrument_group,cp_client_id,cp_client_unit_id,collateral_number,opening_date_from,opening_date_to,value_date_from,value_date_to,state_id,number,package_main_type_id,transaction_tag,sattr_1,facility_reporting_group,log_id_from,log_date_from +portfolio_id= +instrument_group= +instrument_id= +cp_client_id= +collateral_number= +opening_date_from= +opening_date_to= +value_date_from= +value_date_to= +state_id= +number= +package_main_type_id= +transaction_tag= +cp_client_unit_id= +sattr_1= +facility_reporting_group= +log_id_from= + + +[Format last_log_id] +Name=Last Log ID +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format guarantor_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date/Time +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_date] +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name=Value Date +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=Maturity Date +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expiry_date] +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_date] +Name=Opening Date +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_time] +Name=Opening Time +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format closing_date] +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_price] +Name=Deal Price +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=1 + + +[Format mattr_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_3] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_4] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_offset] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_code] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_type] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format signature_date] +date_format=dd/MM/yyyy +time_format=HH:mm:ss +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_offset_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_calendar] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_calendar_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format frequency] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format frequency_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format method_value] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format method_value_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread_2] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_1_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_2_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount_2] +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_type] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate_2] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_type_2] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cap] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format floor] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format top_portfolio_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier_type] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format face_amount] +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format pool_factor] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_tag] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_finance_rate] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format quote_currency_yield] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_order] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format idx] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issue_index] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format index_ratio] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_fwd_points] +Name=FX Forward Points +Expression=get_transaction_property(number, "fx_forward_points") +Justify=right +divider=1 +precision=3 +max_precision=3 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_rate] +Name=FX Spot Rate +Expression=1/get_transaction_property(number, "spot_rate") +Justify=right +divider=1 +precision=10 +max_precision=10 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format deal_price2] +Name=Deal Price 2 +Expression=deal_price +Justify=right +divider=1 +precision=5 +max_precision=5 +show_zero=1 +NoNumberFormatting=0 + + +[Format isin_code] +Name=ISIN +Expression=get_security_identifier(instrument_id, 'ISIN') +Justify=right +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format broker_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id_sort_order] +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_amount] +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format last_log_date] +Name=Last Log Date +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 \ No newline at end of file diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.yml new file mode 100644 index 0000000..6fdf901 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION.yml @@ -0,0 +1,14 @@ + +parameters: + + min_log_date: + value: select("ct_mrds.a_workflow_history", + "to_char(max(workflow_start) + interval '1' second, 'YYYYMMDD - HH24MISS')", + "workflow_name = 'w_ODS_TMS_TRANSACTION' and workflow_successful = 'Y'") + hidden: false + + + max_log_date: + value: select("dual", "to_char(sysdate - 5/(24*60), 'YYYYMMDD - HH24MISS')") + hidden: false + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.fkr new file mode 100644 index 0000000..675fd07 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.fkr @@ -0,0 +1,2286 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=transactions-log +Name=Transaction Log Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Transaction Log Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=log_id,log_date,log_user,log_action,log_type,log_serial,log_caller,tags,number,market_id,instrument_id,issuer_id,issuer_unit_id,currency_id,currency_2_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,expiry_date,maturity_date,opening_date,opening_time,closing_date,deal_rate,deal_price,nominal_rate,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,amount,agreed_amount,trading_unit,book_value,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,broker_id,cp_client_id,cp_client_unit_id,cp_portfolio_id,cp_trader_id,secondary_instrument_id,collateral_number,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,type_name,state_name,instrument_group +Selected=log_id,log_date,log_user,log_action,log_type,log_serial,log_caller,tags,number,market_id,instrument_id,issuer_id,issuer_unit_id,currency_id,currency_2_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,expiry_date,maturity_date,opening_date,opening_time,closing_date,deal_rate,deal_price,nominal_rate,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,amount,agreed_amount,trading_unit,book_value,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,broker_id,cp_client_id,cp_client_unit_id,cp_portfolio_id,cp_trader_id,secondary_instrument_id,collateral_number,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,type_name,state_name,instrument_group +Grouping= +Sorting=log_id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, +DigitGrouping=03 + + +[Parameters] +Fields=date_type,from,to,number,portfolio_id,instrument_id,instrument_group,user_id,cp_client_id,cp_client_unit_id,collateral_number +date_type=1 +from= +to= +number= +portfolio_id= +instrument_id= +instrument_group= +user_id= +cp_client_id= +cp_client_unit_id= +collateral_number= + + +[Format log_id] +Name=LOG_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_date] +Name=LOG_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_user] +Name=LOG_USER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_action] +Name=LOG_ACTION +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_type] +Name=LOG_TYPE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_serial] +Name=LOG_SERIAL +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_caller] +Name=LOG_CALLER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format tags] +Name=TAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format number] +Name=NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format market_id] +Name=MARKET_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name=INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name=ISSUER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name=ISSUER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name=CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +Name=CURRENCY_2_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name=SIGN_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name=TYPE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name=KIND_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +Name=AUDIT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +Name=REFERENCE_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +Name=LOGICAL_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +Name=PARENT_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name=BATCH_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format settlement_date] +Name=SETTLEMENT_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format value_date] +Name=VALUE_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format expiry_date] +Name=EXPIRY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=MATURITY_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_date] +Name=OPENING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format opening_time] +Name=OPENING_TIME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format closing_date] +Name=CLOSING_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +Name=DEAL_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_price] +Name=DEAL_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +Name=NOMINAL_RATE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +Name=PREMIUM_CURRENCY_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +Name=PREMIUM_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +Name=PREMIUM_DATE +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +Name=PREMIUM_PRICE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +Name=MARGIN1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +Name=MARGIN2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name=UNITS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name=AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +Name=AGREED_AMOUNT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name=TRADING_UNIT +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name=BOOK_VALUE +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name=ACCRUED_INTEREST +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name=PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name=OWNER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name=OWNER_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +Name=TRADER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name=BROKER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name=CP_CLIENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name=CP_CLIENT_UNIT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name=CP_PORTFOLIO_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +Name=CP_TRADER_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name=SECONDARY_INSTRUMENT_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +Name=COLLATERAL_NUMBER +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +Name=SATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +Name=SATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +Name=SATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +Name=SATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +Name=SATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +Name=SATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +Name=SATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +Name=SATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +Name=SATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +Name=SATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +Name=SATTR_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +Name=SATTR_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +Name=SATTR_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +Name=SATTR_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +Name=SATTR_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +Name=SATTR_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +Name=SATTR_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +Name=SATTR_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +Name=SATTR_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +Name=SATTR_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +Name=DATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +Name=DATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +Name=DATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +Name=DATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +Name=DATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +Name=DATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +Name=DATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +Name=DATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +Name=DATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +Name=DATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +Name=IATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +Name=IATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +Name=IATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +Name=IATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +Name=IATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +Name=IATTR_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +Name=IATTR_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +Name=IATTR_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +Name=IATTR_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +Name=IATTR_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +Name=MATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +Name=MATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_2] +Name=MATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +Name=MATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +Name=MATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +Name=FATTR_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +Name=FATTR_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +Name=FATTR_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +Name=FATTR_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +Name=FATTR_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name=PARAM_0 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name=PARAM_1 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name=PARAM_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name=PARAM_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name=PARAM_4 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name=PARAM_5 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name=PARAM_6 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name=PARAM_7 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name=PARAM_8 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name=PARAM_9 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name=PARAM_10 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name=PARAM_11 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name=PARAM_12 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name=PARAM_13 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name=PARAM_14 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name=PARAM_15 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name=PARAM_16 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name=PARAM_17 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name=PARAM_18 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name=PARAM_19 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name=STATE_ID +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name=CONTEXTS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +Name=CREATOR_TAG +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name=STATUS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=FLAGS +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +Name=FLAGS_2 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +Name=FLAGS_3 +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_name] +Name=TYPE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_name] +Name=STATE_NAME +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name=INSTRUMENT_GROUP +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.yml new file mode 100644 index 0000000..e88aea2 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTIONSLOG_CURR.yml @@ -0,0 +1,16 @@ + +parameters: + + from: + value: t + hidden: false + + + to: + value: t + hidden: false + + + date_type: + value: 1 + hidden: true diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION_ADHOC.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION_ADHOC.fkr new file mode 100644 index 0000000..32cfb99 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_TRANSACTION_ADHOC.fkr @@ -0,0 +1,2961 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=transactions +Name=Transaction Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=Transaction Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=number,instrument_id,issuer_id,issuer_unit_id,currency_id,sign_id,type_id,kind_id,audit_number,reference_number,logical_number,parent_number,batch_id,date,settlement_date,value_date,maturity_date,expiry_date,opening_date,opening_time,closing_date,deal_price,premium_currency_id,premium_amount,premium_date,premium_price,margin1,margin2,units,trading_unit,accrued_interest,portfolio_id,owner_id,owner_unit_id,trader_id,secondary_instrument_id,collateral_number,broker_id,cp_client_id,cp_client_unit_id,cp_trader_id,cp_portfolio_id,sattr_0,sattr_1,sattr_2,sattr_3,sattr_4,sattr_5,sattr_6,sattr_7,sattr_8,sattr_9,sattr_10,sattr_11,sattr_12,sattr_13,sattr_14,sattr_15,sattr_16,sattr_17,sattr_18,sattr_19,dattr_0,dattr_1,dattr_2,dattr_3,dattr_4,dattr_5,dattr_6,dattr_7,dattr_8,dattr_9,iattr_0,iattr_1,iattr_2,iattr_3,iattr_4,iattr_5,iattr_6,iattr_7,iattr_8,iattr_9,mattr_0,mattr_1,mattr_2,mattr_3,mattr_4,fattr_0,fattr_1,fattr_2,fattr_3,fattr_4,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19,state_id,contexts,creator_tag,status,flags,flags_2,flags_3,instrument_group,package_main_type_id,package_type_id,package_id,comment,comment_2,comment_3,comment_4,fixing_rate,fixing_period,fixing_offset,date_basis,maturity_code,premium_type,signature_date,fixing_rate_2,fixing_period_2,fixing_offset_2,date_basis_2,fixing_calendar,fixing_calendar_2,frequency,frequency_2,method_value,method_value_2,spread,spread_2,leg_1_id,leg_2_id,currency_2_id,amount_2,nominal_rate,rate_type,nominal_rate_2,rate_type_2,cap,floor,top_portfolio_id,security_identifier_type,security_identifier,deal_rate,book_value,amount,agreed_amount,face_amount,pool_factor,transaction_tag,fx_finance_rate,quote_currency_yield,state_id_sort_order,idx,issue_index,index_ratio,collateral_amount,log_id,log_date,fx_spot_rate,fx_forward_points,deal_pricea,isin_code +Selected=log_id,log_date,opening_date,date,value_date,maturity_date,maturity_code,issuer_id,amount,book_value,currency_id,currency_2_id,instrument_group,instrument_id,security_identifier,portfolio_id,number,cp_client_id,deal_price,deal_rate,type_id,sign_id,fx_spot_rate,fx_forward_points,state_id,param_0,isin_code,param_13,accrued_interest,agreed_amount,amount_2,audit_number,batch_id,cap,closing_date,collateral_number,comment,comment_2,comment_3,comment_4,contexts,cp_portfolio_id,cp_trader_id,cp_client_unit_id,creator_tag,date_basis,date_basis_2,deal_pricea,param_1,expiry_date,fx_finance_rate,mattr_4,face_amount,sattr_1,fixing_calendar,fixing_calendar_2,fixing_offset,fixing_offset_2,fixing_period,fixing_period_2,fixing_rate,fixing_rate_2,flags,flags_2,flags_3,floor,frequency,frequency_2,idx,index_ratio,issue_index,issuer_unit_id,kind_id,leg_1_id,leg_2_id,sattr_4,logical_number,sattr_3,margin1,margin2,param_11,method_value,method_value_2,param_9,nominal_rate,nominal_rate_2,opening_time,iattr_6,param_10,param_4,owner_id,owner_unit_id,package_id,package_main_type_id,package_type_id,param_18,param_3,param_5,param_6,param_7,param_8,param_19,parent_number,sattr_13,pool_factor,premium_amount,premium_currency_id,premium_date,premium_price,premium_type,quote_currency_yield,rate_type,rate_type_2,reference_number,param_2,secondary_instrument_id,security_identifier_type,param_16,settlement_date,signature_date,spread,spread_2,state_id_sort_order,status,broker_id,top_portfolio_id,trader_id,trading_unit,sattr_2,param_12,param_14,param_15,param_17,transaction_tag,units +Grouping= +Sorting= +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=12500670, 4144959, 4144959 +HeaderRowColors=15066597, 4144959 +DataRowColors=16645886, 0, 0 +TotalRowColors=12500670, 4144959, 4144959 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] + + +[Format number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=73 +width_in_characters=14 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=107 +width_in_characters=21 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=97 +width_in_characters=19 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format issuer_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sign_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format kind_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format audit_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format reference_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format logical_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format parent_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format batch_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date] +Name=Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=173 +width_in_characters=34 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format settlement_date] +Name=Settlement Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format value_date] +Name=Value Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format maturity_date] +Name=Maturity Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format expiry_date] +Name=Expiry Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format opening_date] +Name=Opening Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=147 +width_in_characters=29 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format opening_time] +Name=Opening Time +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format closing_date] +Name=Closing Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=113 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format deal_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=92 +width_in_characters=18 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_currency_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_date] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_price] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format margin2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format units] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trading_unit] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format accrued_interest] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=86 +width_in_characters=17 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format owner_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format secondary_instrument_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_number] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=115 +width_in_characters=23 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format broker_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=84 +width_in_characters=16 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_client_unit_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_trader_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cp_portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format sattr_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=10 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format dattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format iattr_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format mattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=108 +width_in_characters=21 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fattr_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_0] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_1] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_5] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_6] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_7] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_8] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_9] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_10] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_11] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_12] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_13] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_14] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_15] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_16] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_17] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_18] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format param_19] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format contexts] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format creator_tag] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format status] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format instrument_group] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=110 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_main_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_type_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format package_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=288 +width_in_characters=57 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_3] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment_4] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_offset] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format maturity_code] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format premium_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format signature_date] +Name=Signature Date +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=left +width=58 +width_in_characters=11 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format fixing_rate_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_period_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_offset_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format date_basis_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_calendar] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fixing_calendar_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format frequency] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format frequency_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format method_value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format method_value_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format spread_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_1_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format leg_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format currency_2_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=2 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=92 +width_in_characters=18 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format nominal_rate_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format rate_type_2] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format cap] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format floor] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format top_portfolio_id] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier_type] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format security_identifier] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=87 +width_in_characters=17 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format deal_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=92 +width_in_characters=18 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format book_value] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=104 +width_in_characters=20 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=130 +width_in_characters=26 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format agreed_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=122 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format face_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=99 +width_in_characters=19 +divider=1 +precision=4 +max_precision=4 +show_zero=1 +NoNumberFormatting=0 + + +[Format pool_factor] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=1 +NoNumberFormatting=0 + + +[Format transaction_tag] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format fx_finance_rate] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format quote_currency_yield] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format state_id_sort_order] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format idx] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=62 +width_in_characters=12 +divider=1 +precision=5 +max_precision=5 +show_zero=1 +NoNumberFormatting=0 + + +[Format issue_index] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=5 +max_precision=5 +show_zero=1 +NoNumberFormatting=0 + + +[Format index_ratio] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=5 +max_precision=5 +show_zero=1 +NoNumberFormatting=0 + + +[Format collateral_amount] +Name= +Expression= +report_total= +date_format= +time_format= +Justify= +width=122 +width_in_characters=24 +divider=1 +precision=4 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format log_id] +Name=Log ID +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format log_date] +Name=Log Date +Expression=0 +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format fx_spot_rate] +Name=FX Spot Rate +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=0 +NoNumberFormatting=0 + + +[Format fx_forward_points] +Name=FX Forward Points +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format isin] +Name=ISIN +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + + +[Format deal_pricea] +Name=Deal Price A +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=10 +max_precision=10 +show_zero=0 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Format isin_code] +Name=ISIN +Expression=0 +report_total= +date_format= +time_format= +Justify=right +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.fkr new file mode 100644 index 0000000..fb7ca2a --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.fkr @@ -0,0 +1,276 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=user-information-report +Name=User Information Report +ExcelWorkbook= +HeaderLeft=[\n]%pagebreak +HeaderCenter=User Information Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria= +TotalsOnly=0 +MergeCellsVertically= +MergeCellsHorizontally= +Fields=id,admin_center_id,name,time_zone_id,menu_name,web_home_group_id,comment,flags,access_role_id,access_role_name,active_from,active_to +Selected=id,name,admin_center_id,time_zone_id,menu_name,web_home_group_id,flags,access_role_id,access_role_name,active_from,active_to +Grouping= +Sorting=id +SortDescending= +GroupSortLevel=0 +GroupSortColumns= +GroupSortDescending= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700, +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700, +GridHeadingColors=14005927, 6441018, 6441018 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 6441018 +DecimalSeparator=. +ThousandsSeparator=, + + +[Parameters] +Fields= +user_id= +admin_center_id= + + +[Format id] +Name=id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format admin_center_id] +Name=admin_center_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format name] +Name=name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format time_zone_id] +Name=time_zone_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format menu_name] +Name=menu_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format web_home_group_id] +Name=web_home_group_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format comment] +Name=comment +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format flags] +Name=flags +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format access_role_id] +Name=access_role_id +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format access_role_name] +Name=access_role_name +Expression= +report_total= +date_format= +time_format= +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_from] +Name=active_from +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Format active_to] +Name=active_to +Expression= +report_total= +date_format=dd/MM/yyyy +time_format=HH:mm:ss +Justify= +width=60 +width_in_characters=12 +divider=1 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Export] +ExportCoverPage=1 +ExportReportName=0 +ExportHeaderRows=0 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=1 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USERINFORMATION.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.fkr b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.fkr new file mode 100644 index 0000000..4af9fb6 --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.fkr @@ -0,0 +1,330 @@ +# encoding: UTF-8 +[fk report] +Version=2.3 +VersionWarning= + + +[Main] +Type=user-information-report +Name=User Information Report +HeaderLeft=[\n]%pagebreak +HeaderCenter=User Information Report +HeaderRight=%datetime[\n]Page %pagenum/%pagecount +FooterLeft= +FooterCenter= +FooterRight= +Criteria=!(access_role_name like "Role for system%") +Fields=id,admin_center_id,flags,name,time_zone_id,menu_name,web_home_group_id,comment,access_role_id,access_role_name,active_from,active_to,date +Selected=date,id,name,menu_name,access_role_id,access_role_name +Grouping= +Sorting=id +GroupSortLevel=0 +GroupSortColumns= +Pagebreak=0 +ZoomPercent=100 +FontGridHeading=Tahoma,-11,700, +Font0=Tahoma,-11,400,i +Font1=Tahoma,-11,400, +Font2=Tahoma,-11,700, +FontGridHeadingDrillDown=Tahoma,-11,700,u +Font1DrillDown=Tahoma,-11,400,u +Font2DrillDown=Tahoma,-11,700,u +GridHeadingColors=14005927, 6441018, 0 +HeaderRowColors=13812152, 6441018 +DataRowColors=16182761, 0, 0 +TotalRowColors=14005927, 6441018, 0 +DecimalSeparator=. +ThousandsSeparator=, +ExcelWorkbook= +MergeCellsVertically= +MergeCellsHorizontally= +SortDescending= +GroupSortDescending= +TotalsOnly=0 + + +[Parameters] + + +[Format id] +Name= +total_id= +report_total= +date_format= +time_format= +width=78 +width_in_characters=15 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format admin_center_id] +Name= +total_id= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format flags] +Name= +total_id= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format version] +Name= +total_id= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format name] +Name= +total_id= +report_total= +date_format= +time_format= +width=199 +width_in_characters=39 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format time_zone_id] +Name= +total_id= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format menu_name] +Name= +total_id= +report_total= +date_format= +time_format= +width=231 +width_in_characters=46 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format web_home_group_id] +Name= +total_id= +report_total= +date_format= +time_format= +width=60 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format comment] +Name= +total_id= +report_total= +date_format= +time_format= +width=108 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format access_role_id] +Name= +total_id= +report_total= +date_format= +time_format= +width=160 +width_in_characters=32 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format access_role_name] +Name= +total_id= +report_total= +date_format= +time_format= +width=147 +width_in_characters=29 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format active_from] +Name= +total_id= +report_total= +date_format= +time_format= +width=104 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Format active_to] +Name= +total_id= +report_total= +date_format= +time_format= +width=104 +width_in_characters=10 +precision=0 +max_precision=0 +show_zero=1 +NoNumberFormatting=0 +total_name= +Expression= +Justify= +divider=1 + + +[Paper] +Type=9 +Width=29700 +Height=21000 +Left=2000 +Right=2000 +Top=2000 +Bottom=2000 + + +[Print] +PrintVertGridLines=1 +PrintHorzGridLines=1 +PrintFrame=1 +PrintRowNumbers=0 +PrintInColor=0 +PrintCenterHorizontally=0 +PrintCenterVertically=0 +PrintTransparentBackground=0 +PrintCoverPage=1 +PrintHeaderRows=1 +PrintPlainRows=1 +PrintFooterRows=1 + + +[Export] +ExportCoverPage=1 +ExportReportName=1 +ExportHeaderRows=1 +ExportPlainRows=1 +ExportFooterRows=1 +ExportEmptyReport=1 + + +[Format date] +Name=Date +Expression=today +report_total= +date_format=%c +time_format= +Justify=right +width=114 +width_in_characters=22 +divider=1 +precision=0 +max_precision=0 +show_zero=0 +NoNumberFormatting=0 \ No newline at end of file diff --git a/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.yml b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.yml new file mode 100644 index 0000000..a6b078b --- /dev/null +++ b/airflow/ods/tms/TMS-layouts/w_ODS_TMS_USER_ACCOUNT.yml @@ -0,0 +1,5 @@ + + +parameters: + + diff --git a/airflow/ods/tms/TMS-tables/TMS_BRANCH.sql b/airflow/ods/tms/TMS-tables/TMS_BRANCH.sql new file mode 100644 index 0000000..1f0528e --- /dev/null +++ b/airflow/ods/tms/TMS-tables/TMS_BRANCH.sql @@ -0,0 +1,8 @@ +create table ct_et_templates.TMS_BRANCH ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(512 char), + name varchar2(512 char), + number_ number(12, 0), + order_number number(12, 0) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMCURRENCYFLOW.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMCURRENCYFLOW.sql new file mode 100644 index 0000000..45bf9e9 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMCURRENCYFLOW.sql @@ -0,0 +1,101 @@ +create table ct_et_templates.ACMCURRENCYFLOW ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + accountingtreatment_name varchar2(255 char), + acmbookkeepingtype_id_user varchar2(255 char), + acmcmmstate_id_user varchar2(255 char), + acmcmmviewtype_id_user varchar2(255 char), + acmcptyrelation_id_user varchar2(255 char), + acmcurrencyentrystate_id_user varchar2(255 char), + acmcurrencyposition_id_user varchar2(255 char), + acmentry_account_holder_id varchar2(255 char), + acmentry_bank_account varchar2(255 char), + acmentry_bank_account_ccy_id varchar2(255 char), + acmentry_bank_id varchar2(255 char), + acmentry_book_value number(19,4), + acmentry_booking_currency_id varchar2(255 char), + acmentry_c_payment_date date, + acmentry_ccy_amount number(19,4), + acmentry_ccy_booking_amount number(19,4), + acmentry_ccy_group_id number(12, 0), + acmentry_ccy_result number(19,4), + acmentry_classification_id varchar2(255 char), + acmentry_cmm_group_id number(12, 0), + acmentry_cmm_payment_method_id varchar2(255 char), + acmentry_cmm_reconciliation_id number(12, 0), + acmentry_counterparty_code varchar2(255 char), + acmentry_counterparty_group_id varchar2(255 char), + acmentry_counterparty_id varchar2(255 char), + acmentry_currency_id varchar2(255 char), + acmentry_date_basis varchar2(255 char), + acmentry_deal_price binary_double, + acmentry_deal_rate binary_double, + acmentry_description varchar2(255 char), + acmentry_event_date date, + acmentry_event_id number(12, 0), + acmentry_figure_id varchar2(255 char), + acmentry_fx_rate binary_double, + acmentry_id number(12, 0), + acmentry_instrument_id varchar2(255 char), + acmentry_issuer_group_id varchar2(255 char), + acmentry_issuer_id varchar2(255 char), + acmentry_leg_group number(12, 0), + acmentry_leg_id varchar2(255 char), + acmentry_load_id number(12, 0), + acmentry_nominal_amount number(19,4), + acmentry_number number(12, 0), + acmentry_one_time_cpty_id varchar2(255 char), + acmentry_package_map_id number(12, 0), + acmentry_package_type_id varchar2(255 char), + acmentry_param_0 varchar2(255 char), + acmentry_param_1 varchar2(255 char), + acmentry_param_10 varchar2(255 char), + acmentry_param_11 varchar2(255 char), + acmentry_param_12 varchar2(255 char), + acmentry_param_13 varchar2(255 char), + acmentry_param_14 varchar2(255 char), + acmentry_param_15 varchar2(255 char), + acmentry_param_16 varchar2(255 char), + acmentry_param_17 varchar2(255 char), + acmentry_param_18 varchar2(255 char), + acmentry_param_19 varchar2(255 char), + acmentry_param_2 varchar2(255 char), + acmentry_param_3 varchar2(255 char), + acmentry_param_4 varchar2(255 char), + acmentry_param_5 varchar2(255 char), + acmentry_param_6 varchar2(255 char), + acmentry_param_7 varchar2(255 char), + acmentry_param_8 varchar2(255 char), + acmentry_param_9 varchar2(255 char), + acmentry_portfolio_id varchar2(255 char), + acmentry_reverse_event_id number(12, 0), + acmentry_source_reference varchar2(255 char), + acmentry_t_currency_2_id varchar2(255 char), + acmentry_t_currency_id varchar2(255 char), + acmentry_t_maturity_date date, + acmentry_t_opening_date date, + acmentry_t_value_date date, + acmentryorigin_id_user varchar2(255 char), + acmentryoriginentity_id_user varchar2(255 char), + acmentryorigingroup_id_user varchar2(255 char), + acmentrytype_id_user varchar2(255 char), + acmeventtype_id_user varchar2(255 char), + acmgroupingrule_id number(12, 0), + acmgroupingrule_id_user varchar2(255 char), + acmledger_id number(12, 0), + acmledger_id_user varchar2(255 char), + acmmappingrule_id number(12, 0), + acmmappingrule_id_user varchar2(255 char), + acmperiod_id_user varchar2(255 char), + acmperiodordernumber_number number(12, 0), + acmplcategory_id_user varchar2(255 char), + acmtransactionsign_name varchar2(255 char), + acmvoucher_booking_date date, + acmvoucher_id number(12, 0), + acmvoucher_posting_date date, + acmvoucher_voucher_number varchar2(255 char), + transactionkind_name varchar2(255 char), + transactiontype_name varchar2(255 char), + umpathnode_id varchar2(255 char), + umpathnode_upath varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql new file mode 100644 index 0000000..57932e8 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.sql @@ -0,0 +1,193 @@ +create table ct_et_templates.w_ODS_TMS_ACMENTRYSTATELEDGERGROUP ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + acmaccount1_id_user varchar2(255 char), + acmplcategory_id_user varchar2(255 char), + acmaccount1_name varchar2(255 char), + acmentry_account_id number(12, 0), + accountingtreatment_name varchar2(255 char), + acmmappingaction_id_user varchar2(255 char), + acmentry_adj_discharge_date date, + acmentry_aggregation_id number(12, 0), + acmgroupingrule1_id_user varchar2(255 char), + acmgroupingrule1_id number(12, 0), + acmsign1_name varchar2(255 char), + acmentry_bank_id varchar2(255 char), + acmentry_bank_account varchar2(255 char), + acmentry_bank_account_ccy_id varchar2(255 char), + acmentry_account_holder_id varchar2(255 char), + acmentry_acc_holder_code varchar2(255 char), + acmentry_acc_holder_group_id varchar2(255 char), + acmentry_batch_id number(12, 0), + acmentry_book_value number(19,4), + acmentry_booking_amount number(19,4), + acmentry_booking_currency_id varchar2(255 char), + acmentry_booking_date date, + acmbookkeepingtype_id_user varchar2(255 char), + acmentry_param_0 varchar2(255 char), + acmentry_cmmse_id number(12, 0), + acmentry_cmm_group_id number(12, 0), + acmentry_cmm_payment_method_id varchar2(255 char), + acmentry_cmm_reconciliation_id number(12, 0), + acmcmmstate_id_user varchar2(255 char), + ctb_category varchar2(255 char), + acmentry_ctb_previous_date date, + acmentry_ctb_sequence_id number(12, 0), + acmctbtyperule_id_user varchar2(255 char), + acmrevcancelstate2_id_user varchar2(255 char), + acmentry_c_payment_date date, + acmentry_figure_id varchar2(255 char), + acmledger_chart_id number(12, 0), + acmentry_classification_id varchar2(255 char), + client_unit_relation_id varchar2(255 char), + acmentry_continuation_number number(12, 0), + acmcostcenter_id_user varchar2(255 char), + acmhedgecostcat_id_user varchar2(255 char), + acmentry_counterparty_id varchar2(255 char), + acmentry_counterparty_code varchar2(255 char), + acmentry_counterparty_group_id varchar2(255 char), + acmcptyrelation_id_user varchar2(255 char), + acmentry_counterparty_unit_id varchar2(255 char), + acmentry_ccy_amount number(19,4), + acmentry_ccy_booking_amount number(19,4), + acmentry_ccy_group_id number(12, 0), + acmentry_ccy_result number(19,4), + ccy_grouping_rule varchar2(255 char), + acmentry_ccy_grouping_rule_id number(12, 0), + ccy_position varchar2(255 char), + ccy_state varchar2(255 char), + acmdrcr_id_user varchar2(255 char), + acmentry_date_basis varchar2(255 char), + acmentry_deal_price binary_double, + acmentry_deal_rate binary_double, + acmentry_description varchar2(255 char), + acmentry_amount number(19,4), + acmentry_currency_id varchar2(255 char), + acmaccount1_erp_account varchar2(255 char), + acmentry_param_1 varchar2(255 char), + acmentry_id number(12, 0), + acmentrytype_id_user varchar2(255 char), + acmvouchertype2_id_user varchar2(255 char), + acmentry_event_date date, + acmentry_event_id number(12, 0), + acmeventtype_id_user varchar2(255 char), + acmentry_export_id number(12, 0), + acmentry_facility_id varchar2(255 char), + acmentry_fac_rep_group varchar2(255 char), + acmentry_figure_amount number(19,4), + acmentry_figure_booking_amount number(19,4), + acmentry_flags varchar2(255 char), + acmentry_flags_2 varchar2(255 char), + acmentry_fx_rate binary_double, + acmsign2_name varchar2(255 char), + acmgroupingrule2_id_user varchar2(255 char), + acmgroupingrule2_id number(12, 0), + acmentry_hedge_book_eff_bal number(19,4), + acmentry_hedge_book_ineff_bal number(19,4), + hedgededesignationtype_name varchar2(255 char), + acmentry_hedge_design_percent number(19,4), + acmentry_hedge_doc_eff_bal number(19,4), + acmentry_hedge_doc_ineff_bal number(19,4), + acmhedgeeffcat_id_user varchar2(255 char), + acmentry_hedge_eff_percent binary_double, + acmhedgequalifcat_id_user varchar2(255 char), + acmhedgelegcat_id_user varchar2(255 char), + acmentry_hedge_risk varchar2(255 char), + acmentry_hedge_strategy_code varchar2(255 char), + acmentry_hedge_id number(12, 0), + acmentry_hedge_test_method varchar2(255 char), + acmentry_hedge_type varchar2(255 char), + acmentry_instrument_id varchar2(255 char), + umpathnode_upath varchar2(255 char), + umpathnode_id varchar2(255 char), + acmentry_issuer_id varchar2(255 char), + acmentry_issuer_group_id varchar2(255 char), + acmentry_issuer_unit_id varchar2(255 char), + acmledger_id_user varchar2(255 char), + acmentry_ledger_id number(12, 0), + acmentry_leg_id varchar2(255 char), + acmentry_leg_group number(12, 0), + acmentry_load_id number(12, 0), + acmentry_logical_reference_id varchar2(255 char), + acmentry_mapping_description varchar2(255 char), + acmmappingrule_id_user varchar2(255 char), + acmmappingrule_id number(12, 0), + acmentry_match_id number(12, 0), + acmentry_modification_time date, + acmentry_n_option_date date, + acmentry_nominal_amount number(19,4), + acmentry_one_time_cpty_id varchar2(255 char), + option_sign varchar2(255 char), + acmentryorigin_id_user varchar2(255 char), + acmentryoriginentity_id_user varchar2(255 char), + acmentryorigingroup_id_user varchar2(255 char), + acmentry_original_amount number(19,4), + acmentry_orig_booking_amount number(19,4), + original_time_band varchar2(255 char), + acmentrystate2_id_user varchar2(255 char), + acmentry_owner_unit_id varchar2(255 char), + acmentry_package_map_id number(12, 0), + acmentry_package_type_id varchar2(255 char), + acmentry_param_10 varchar2(255 char), + acmentry_param_11 varchar2(255 char), + acmentry_param_12 varchar2(255 char), + acmentry_param_13 varchar2(255 char), + acmentry_param_14 varchar2(255 char), + acmentry_param_15 varchar2(255 char), + acmentry_param_17 varchar2(255 char), + acmentry_param_18 varchar2(255 char), + acmentry_param_19 varchar2(255 char), + acmentry_param_5 varchar2(255 char), + acmentry_param_6 varchar2(255 char), + acmentry_param_7 varchar2(255 char), + acmentry_param_8 varchar2(255 char), + acmentry_param_9 varchar2(255 char), + acmentry_parent_id number(12, 0), + acmentry_pase_id number(12, 0), + acmperiod_id_user varchar2(255 char), + acmentry_portfolio_id varchar2(255 char), + acmentry_pre_voucher_id number(12, 0), + acmentry_param_16 varchar2(255 char), + acmproject_id_user varchar2(255 char), + acmentry_reference_date date, + acmentry_number number(12, 0), + repccy varchar2(255 char), + reportccyamount number(19,4), + report_owner_unit_group_id varchar2(255 char), + acmvoucher_req_booking_date date, + acmvoucher_reversal_bundle_id number(12, 0), + acmentry_reverse_event_id number(12, 0), + acmrevcancelstate1_id_user varchar2(255 char), + acmvoucher_rev_voucher_id number(12, 0), + acmaccount2_id_user varchar2(255 char), + acmentry_secondary_account_id number(12, 0), + acmentry_source_reference varchar2(255 char), + acmspecialtreatment_id_user varchar2(255 char), + acmrebookingtype_id_user varchar2(255 char), + acmentrystate1_id_user varchar2(255 char), + acmentry_status number(12, 0), + acmentry_tag varchar2(255 char), + acmentry_t_currency_id varchar2(255 char), + acmentry_t_currency_2_id varchar2(255 char), + transactionkind_name varchar2(255 char), + acmentry_t_maturity_date date, + acmentry_t_opening_date date, + acmentry_param_3 varchar2(255 char), + acmentry_param_4 varchar2(255 char), + acmtransactionsign_name varchar2(255 char), + transactiontype_name varchar2(255 char), + acmentry_t_value_date date, + acmvoucher_bundle_id number(12, 0), + valuationhierarchy_name varchar2(255 char), + acmcmmviewtype_id_user varchar2(255 char), + acmvoucher_booking_date date, + acmvoucher_flags varchar2(255 char), + acmvoucher_modification_time date, + acmvoucher_voucher_number varchar2(255 char), + acmvoucher_posting_date date, + acmrevcancelstate3_id_user varchar2(255 char), + acmvoucherstate_id_user varchar2(255 char), + acmentry_voucher_id number(12, 0), + acmvouchertype_id_user varchar2(255 char), + acmentry_param_2 varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY.sql new file mode 100644 index 0000000..6bcead7 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY.sql @@ -0,0 +1,20 @@ +create table ct_et_templates.w_ODS_TMS_ACTIVITY ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + group_id varchar2(255 char), + id varchar2(255 char), + type_id varchar2(255 char), + domain_id varchar2(255 char), + flags varchar2(255 char), + name varchar2(255 char), + service_name varchar2(255 char), + service_group varchar2(255 char), + start_date date, + time_zone_id varchar2(255 char), + end_date date, + prerequisite_id varchar2(255 char), + owner_id varchar2(255 char), + type_name varchar2(255 char), + start_date_local date, + end_date_local date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITYLOGDUE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITYLOGDUE.sql new file mode 100644 index 0000000..6da7991 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITYLOGDUE.sql @@ -0,0 +1,14 @@ +create table ct_et_templates.w_ODS_TMS_ACTIVITYLOGDUE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + number_ number(12, 0), + batch_id number(12, 0), + prerequisite_id number(12, 0), + due_date date, + start_date date, + finish_date date, + description varchar2(255 char), + domain_id varchar2(255 char), + user_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY_LOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY_LOG.sql new file mode 100644 index 0000000..9605e08 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ACTIVITY_LOG.sql @@ -0,0 +1,14 @@ +create table ct_et_templates.w_ODS_TMS_ACTIVITY_LOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + number_ number(12, 0), + batch_id number(12, 0), + prerequisite_id number(12, 0), + due_date date, + start_date date, + finish_date date, + description varchar2(255 char), + domain_id varchar2(255 char), + user_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_BALANCE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_BALANCE.sql new file mode 100644 index 0000000..a31bcf9 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_BALANCE.sql @@ -0,0 +1,54 @@ +create table ct_et_templates.w_ODS_TMS_BALANCE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + number_ number(12, 0), + id number(12, 0), + date_ date, + amount number(19,4), + accrued_interest_percent binary_double, + client_id varchar2(255 char), + client_unit_id varchar2(255 char), + bank_id varchar2(255 char), + bank_unit_id varchar2(255 char), + account_id varchar2(255 char), + currency_id varchar2(255 char), + cp_client_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + type varchar2(255 char), + order_number number(12, 0), + flags varchar2(255 char), + name varchar2(255 char), + portfolio_id varchar2(255 char), + instrument_id varchar2(255 char), + instrument_group varchar2(255 char), + state_id varchar2(255 char), + payment_id number(12, 0), + payment_date date, + payment_currency_id varchar2(255 char), + payment_amount number(19,4), + fx_spot_rate binary_double, + affect_number number(12, 0), + payment_client_id varchar2(255 char), + payment_client_unit_id varchar2(255 char), + credit_client_id varchar2(255 char), + param_0 varchar2(255 char), + param_1 varchar2(255 char), + param_2 varchar2(255 char), + param_3 varchar2(255 char), + param_4 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_9 varchar2(255 char), + param_10 varchar2(255 char), + param_11 varchar2(255 char), + param_12 varchar2(255 char), + param_13 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_16 varchar2(255 char), + param_17 varchar2(255 char), + param_18 varchar2(255 char), + param_19 varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_BLACKOUT_LOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_BLACKOUT_LOG.sql new file mode 100644 index 0000000..735bdb3 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_BLACKOUT_LOG.sql @@ -0,0 +1,21 @@ +create table ct_et_templates.w_ODS_TMS_BLACKOUT_LOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + log_id number(12, 0), + log_date date, + log_user varchar2(255 char), + log_action varchar2(255 char), + id number(12, 0), + isin varchar2(255 char), + programme varchar2(255 char), + active_from date, + active_until date, + event_date date, + tms_instrument_id varchar2(255 char), + rule_id varchar2(255 char), + description varchar2(255 char), + comment_ varchar2(255 char), + user1 varchar2(255 char), + user2 varchar2(255 char), + name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_CALENDAR.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CALENDAR.sql new file mode 100644 index 0000000..8aa8a98 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CALENDAR.sql @@ -0,0 +1,9 @@ +create table ct_et_templates.w_ODS_TMS_CALENDAR ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + name varchar2(255 char), + holiday_date date, + holiday_reason varchar2(255 char), + default_holidays varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_CASHFLOW.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CASHFLOW.sql new file mode 100644 index 0000000..a491f77 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CASHFLOW.sql @@ -0,0 +1,219 @@ +create table ct_et_templates.w_ODS_TMS_CASHFLOW ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + param_13 varchar2(255 char), + accrued_interest number(19,4), + active_since date, + active_until date, + affect_number number(12, 0), + agreed_amount number(19,4), + amount number(19,4), + audit_number number(12, 0), + base_currency_id varchar2(255 char), + batch_id number(12, 0), + book_value number(19,4), + cap binary_double, + cashflow_flags varchar2(255 char), + cashflow_flags_2 varchar2(255 char), + cashflow_maturity_date date, + cashflow_type_name varchar2(255 char), + cashflow_value_date date, + closing_date date, + collateral_number number(12, 0), + comment_ varchar2(255 char), + comment_1 varchar2(255 char), + comment_2 varchar2(255 char), + comment_3 varchar2(255 char), + comment_4 varchar2(255 char), + contexts varchar2(255 char), + cp_client_id varchar2(255 char), + other_account_1_id varchar2(255 char), + other_account_2_id varchar2(255 char), + other_account_3_id varchar2(255 char), + other_account_4_id varchar2(255 char), + other_bank_1_id varchar2(255 char), + other_bank_2_id varchar2(255 char), + other_bank_2_swift_code varchar2(255 char), + other_bank_3_id varchar2(255 char), + other_bank_3_swift_code varchar2(255 char), + other_bank_4_id varchar2(255 char), + other_bank_4_swift_code varchar2(255 char), + other_bank_1_swift_code varchar2(255 char), + other_bank_1_unit_id varchar2(255 char), + cp_portfolio_id varchar2(255 char), + cp_trader_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + creator_tag varchar2(255 char), + currency_id varchar2(255 char), + currency_class_id varchar2(255 char), + date_ date, + date_basis varchar2(255 char), + dattr_0 date, + dattr_1 date, + dattr_2 date, + dattr_3 date, + dattr_4 date, + dattr_5 date, + dattr_6 date, + dattr_7 date, + dattr_8 date, + dattr_9 date, + deal_price binary_double, + deal_rate binary_double, + param_1 varchar2(255 char), + expiry_date date, + fx_book_rate binary_double, + mattr_4 number(19,4), + fx_rate binary_double, + fx_spot_rate binary_double, + sattr_1 varchar2(255 char), + factor binary_double, + fattr_0 binary_double, + fattr_1 binary_double, + fattr_2 binary_double, + fattr_3 binary_double, + fattr_4 binary_double, + fixing_offset varchar2(255 char), + fixing_period_id varchar2(255 char), + fixing_quote binary_double, + fixing_rate_id varchar2(255 char), + flags number(12, 0), + flags_2 number(12, 0), + flags_3 number(12, 0), + floor binary_double, + id number(12, 0), + iattr_0 number(12, 0), + iattr_1 number(12, 0), + iattr_2 number(12, 0), + iattr_3 number(12, 0), + iattr_4 number(12, 0), + iattr_5 number(12, 0), + iattr_7 number(12, 0), + iattr_8 number(12, 0), + iattr_9 number(12, 0), + instrument_id varchar2(255 char), + instrument_group varchar2(255 char), + instrument_type_id varchar2(255 char), + issuer_id varchar2(255 char), + issuer_unit_id varchar2(255 char), + kind_id varchar2(255 char), + leg_id varchar2(255 char), + leg_group number(12, 0), + sattr_4 varchar2(255 char), + logical_number number(12, 0), + sattr_3 varchar2(255 char), + margin1 binary_double, + margin2 binary_double, + market_id varchar2(255 char), + param_11 varchar2(255 char), + mattr_0 number(19,4), + mattr_1 number(19,4), + mattr_2 number(19,4), + mattr_3 number(19,4), + maturity_date date, + param_9 varchar2(255 char), + nominal_amount number(19,4), + nominal_rate binary_double, + num number(12, 0), + number_ number(12, 0), + opening_date date, + opening_time date, + iattr_6 varchar2(255 char), + param_10 varchar2(255 char), + param_4 varchar2(255 char), + owner_id varchar2(255 char), + local_account_1_id varchar2(255 char), + local_account_2_id varchar2(255 char), + local_account_2_closing_date date, + local_account_3_id varchar2(255 char), + local_account_3_closing_date date, + local_account_4_id varchar2(255 char), + local_account_4_closing_date date, + local_account_1_closing_date date, + local_bank_1_id varchar2(255 char), + local_bank_2_id varchar2(255 char), + local_bank_2_swift_code varchar2(255 char), + local_bank_3_id varchar2(255 char), + local_bank_3_swift_code varchar2(255 char), + local_bank_4_id varchar2(255 char), + local_bank_4_swift_code varchar2(255 char), + local_bank_1_swift_code varchar2(255 char), + local_bank_1_unit_id varchar2(255 char), + owner_unit_id varchar2(255 char), + p_date_basis varchar2(255 char), + package_id number(12, 0), + package_main_type_id varchar2(255 char), + package_type_id varchar2(255 char), + param_18 varchar2(255 char), + param_3 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_19 varchar2(255 char), + parent_number number(12, 0), + payment_amount number(19,4), + payment_currency_id varchar2(255 char), + payment_date date, + sattr_13 varchar2(255 char), + portfolio_id varchar2(255 char), + premium_amount number(19,4), + premium_currency_id varchar2(255 char), + premium_date date, + premium_price binary_double, + premium_type varchar2(255 char), + rate_type varchar2(255 char), + param_0 varchar2(255 char), + reference_number number(12, 0), + param_2 varchar2(255 char), + sattr_0 varchar2(255 char), + sattr_10 varchar2(255 char), + sattr_11 varchar2(255 char), + sattr_12 varchar2(255 char), + sattr_14 varchar2(255 char), + sattr_15 varchar2(255 char), + sattr_16 varchar2(255 char), + sattr_17 varchar2(255 char), + sattr_18 varchar2(255 char), + sattr_19 varchar2(255 char), + sattr_5 varchar2(255 char), + sattr_6 varchar2(255 char), + sattr_7 varchar2(255 char), + sattr_8 varchar2(255 char), + sattr_9 varchar2(255 char), + secondary_instrument_id varchar2(255 char), + param_16 varchar2(255 char), + actual_amount number(19,4), + other_client_id varchar2(255 char), + other_client_unit_id varchar2(255 char), + actual_currency_id varchar2(255 char), + settlement_date date, + local_client_id varchar2(255 char), + local_client_unit_id varchar2(255 char), + sign_id varchar2(255 char), + since_when date, + spread binary_double, + status varchar2(255 char), + subtype_id number(12, 0), + broker_id varchar2(255 char), + top_portfolio_id varchar2(255 char), + trader_id varchar2(255 char), + trading_unit binary_double, + sattr_2 varchar2(255 char), + transaction_currency_id varchar2(255 char), + transaction_currency_2_id varchar2(255 char), + transaction_date date, + transaction_maturity_date date, + transaction_opening_date date, + param_12 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_17 varchar2(255 char), + state_id varchar2(255 char), + transaction_type_id varchar2(255 char), + transaction_value_date date, + type_id number(12, 0), + units binary_double, + until_when date, + value_date date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTACCOUNT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTACCOUNT.sql new file mode 100644 index 0000000..07419ae --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTACCOUNT.sql @@ -0,0 +1,28 @@ +create table ct_et_templates.TMS_CLIENTACCOUNT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + account1 varchar2(255 char), + account2 varchar2(255 char), + account3 varchar2(255 char), + account4 varchar2(255 char), + active_since date, + active_until date, + bank1 varchar2(255 char), + bank2 varchar2(255 char), + bank3 varchar2(255 char), + bank4 varchar2(255 char), + client_id varchar2(255 char), + comment_ varchar2(255 char), + currency_id varchar2(255 char), + delivery_vs_payment binary_double, + full_chain binary_double, + kind number(12, 0), + last_link binary_double, + name varchar2(255 char), + not_rule_id varchar2(255 char), + own_account binary_double, + prefer_account binary_double, + prefer_bank binary_double, + priority number(12, 0), + rule_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTINSTRUCTION.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTINSTRUCTION.sql new file mode 100644 index 0000000..e9871cd --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CLIENTINSTRUCTION.sql @@ -0,0 +1,91 @@ +create table ct_et_templates.w_ODS_TMS_CLIENTINSTRUCTION ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + account_1_comment varchar2(255 char), + account_1_currency varchar2(255 char), + account_1_group varchar2(255 char), + account_1_name varchar2(255 char), + account_1_number varchar2(255 char), + account_1_parent_account varchar2(255 char), + account_1_secondary_number varchar2(255 char), + account_1_target_closing_date date, + account_1_type varchar2(255 char), + account_2_comment varchar2(255 char), + account_2_currency varchar2(255 char), + account_2_group varchar2(255 char), + account_2_name varchar2(255 char), + account_2_number varchar2(255 char), + account_2_parent_account varchar2(255 char), + account_2_secondary_number varchar2(255 char), + account_2_target_closing_date date, + account_2_type varchar2(255 char), + account_3_comment varchar2(255 char), + account_3_currency varchar2(255 char), + account_3_group varchar2(255 char), + account_3_name varchar2(255 char), + account_3_number varchar2(255 char), + account_3_parent_account varchar2(255 char), + account_3_secondary_number varchar2(255 char), + account_3_target_closing_date date, + account_3_type varchar2(255 char), + account_4_comment varchar2(255 char), + account_4_currency varchar2(255 char), + account_4_group varchar2(255 char), + account_4_name varchar2(255 char), + account_4_number varchar2(255 char), + account_4_parent_account varchar2(255 char), + account_4_secondary_number varchar2(255 char), + account_4_target_closing_date date, + account_4_type varchar2(255 char), + active_since date, + active_until date, + bank_1 varchar2(255 char), + bank_2 varchar2(255 char), + bank_3 varchar2(255 char), + bank_4 varchar2(255 char), + client_id varchar2(255 char), + client_name varchar2(255 char), + client_unit_id varchar2(255 char), + comment_ varchar2(255 char), + currency_id varchar2(255 char), + instruction_name varchar2(255 char), + kind varchar2(255 char), + not_rule_cashflow_category varchar2(255 char), + not_rule_cf_subtype_id number(12, 0), + not_rule_cf_type_id number(12, 0), + not_rule_counterparty varchar2(255 char), + not_rule_counterparty_group varchar2(255 char), + not_rule_cpty_main_group varchar2(255 char), + not_rule_currency varchar2(255 char), + not_rule_currency_2 varchar2(255 char), + not_rule_id varchar2(255 char), + not_rule_instrument varchar2(255 char), + not_rule_instrument_group varchar2(255 char), + not_rule_name varchar2(255 char), + not_rule_number number(12, 0), + not_rule_owner varchar2(255 char), + not_rule_portfolio varchar2(255 char), + not_rule_transaction_kind varchar2(255 char), + not_rule_transaction_sign varchar2(255 char), + not_rule_transaction_type varchar2(255 char), + priority number(12, 0), + rule_cashflow_category varchar2(255 char), + rule_cf_subtype_id number(12, 0), + rule_cf_type_id number(12, 0), + rule_cf_type_name varchar2(255 char), + rule_counterparty varchar2(255 char), + rule_counterparty_group varchar2(255 char), + rule_cpty_main_group varchar2(255 char), + rule_currency varchar2(255 char), + rule_currency_2 varchar2(255 char), + rule_id varchar2(255 char), + rule_instrument varchar2(255 char), + rule_instrument_group varchar2(255 char), + rule_name varchar2(255 char), + rule_number number(12, 0), + rule_owner varchar2(255 char), + rule_portfolio varchar2(255 char), + rule_transaction_kind varchar2(255 char), + rule_transaction_sign varchar2(255 char), + rule_transaction_type varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_CPBLOCKEDISSUERS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CPBLOCKEDISSUERS.sql new file mode 100644 index 0000000..e113dbe --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_CPBLOCKEDISSUERS.sql @@ -0,0 +1,6 @@ +create table ct_et_templates.w_ODS_TMS_CPBLOCKEDISSUERS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + issuer_id varchar2(255 char), + name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.sql new file mode 100644 index 0000000..e15f961 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW.sql @@ -0,0 +1,76 @@ +create table ct_et_templates.w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + type_id varchar2(255 char), + domain_id varchar2(255 char), + flags number(12, 0), + name varchar2(255 char), + layout_id varchar2(255 char), + active_since date, + active_until date, + buy_label varchar2(255 char), + sell_label varchar2(255 char), + path_id number(12, 0), + result_type_id varchar2(255 char), + template_id varchar2(255 char), + instrument_group varchar2(255 char), + umi_id varchar2(255 char), + currency_id varchar2(255 char), + issuer_id varchar2(255 char), + settlement_flags number(12, 0), + amount_rounding binary_double, + accrued_interest varchar2(255 char), + minimum_price_denom number(12, 0), + default_price_denom number(12, 0), + sign_id number(12, 0), + coupon_rate binary_double, + issuer_unit_id varchar2(255 char), + key_id number(12, 0), + cashflow_type_id number(12, 0), + subtype_id number(12, 0), + category_id number(12, 0), + value_date date, + date_basis varchar2(255 char), + fixing_price binary_double, + kinds number(12, 0), + cf_flags number(12, 0), + cf_flags_2 number(12, 0), + cf_keep_flags number(12, 0), + reference_id number(12, 0), + referee_id number(12, 0), + amount binary_double, + actual_currency_id varchar2(255 char), + actual_fx_rate binary_double, + actual_amount binary_double, + subcategory_id number(12, 0), + sign number(12, 0), + since_when date, + until_when date, + payment_date date, + realize_date date, + origin_id number(12, 0), + origin_value binary_double, + ex_date date, + reference_date date, + spread binary_double, + cashflow_group number(12, 0), + accrual_date_basis varchar2(255 char), + fixing_rate_id varchar2(255 char), + fixing_period_id varchar2(255 char), + fixing_period_2_id varchar2(255 char), + fixing_param binary_double, + fixing_subscenario_id varchar2(255 char), + fixing_date date, + fixing_from date, + fixing_to date, + fixing_price_2 binary_double, + fixing_quote binary_double, + fixing_type varchar2(255 char), + fixing_divider binary_double, + fixing_cap binary_double, + fixing_floor binary_double, + expression varchar2(255 char), + freeze_date date, + effective_fixing_date date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.sql new file mode 100644 index 0000000..4dbcb99 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4.sql @@ -0,0 +1,11 @@ +create table ct_et_templates.w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + portfolio varchar2(255 char), + market_value_start number(19,4), + cashflows binary_double, + market_value_end number(19,4), + return binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.sql new file mode 100644 index 0000000..a7df794 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5.sql @@ -0,0 +1,12 @@ +create table ct_et_templates.w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + portfolio varchar2(255 char), + currency varchar2(255 char), + market_value_start number(19,4), + cashflows binary_double, + market_value_end number(19,4), + return binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.sql new file mode 100644 index 0000000..94b8fdc --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68.sql @@ -0,0 +1,12 @@ +create table ct_et_templates.w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + portfolio varchar2(255 char), + instrument varchar2(255 char), + market_value_start number(19,4), + cashflows binary_double, + market_value_end number(19,4), + return binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.sql new file mode 100644 index 0000000..b7853d9 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69.sql @@ -0,0 +1,13 @@ +create table ct_et_templates.w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + portfolio varchar2(512 char), + instrument varchar2(512 char), + currency varchar2(512 char), + market_value_start number(19,4), + cashflows binary_double, + market_value_end number(19,4), + return binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.sql new file mode 100644 index 0000000..5b6c20c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0.sql @@ -0,0 +1,26 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id varchar2(255), + operation varchar2(255 char), + figure_date date, + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.sql new file mode 100644 index 0000000..4e86c54 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100.sql @@ -0,0 +1,29 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + operation varchar2(255 char), + by_portfolio varchar2(255 char), + by_maturity_period varchar2(255 char), + by_instrument varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.sql new file mode 100644 index 0000000..d58c5ce --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093.sql @@ -0,0 +1,30 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + operation varchar2(255 char), + figure_date date, + by_instrument varchar2(255 char), + by_transaction_counterparty varchar2(255 char), + by_portfolio varchar2(255 char), + by_currency varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.sql new file mode 100644 index 0000000..9354c13 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128.sql @@ -0,0 +1,27 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + operation varchar2(255 char), + by_instrument varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.sql new file mode 100644 index 0000000..de95e5c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132.sql @@ -0,0 +1,28 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + operation varchar2(255 char), + by_portfolio varchar2(255 char), + by_instrument varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.sql new file mode 100644 index 0000000..c102811 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256.sql @@ -0,0 +1,27 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + operation varchar2(255 char), + by_portfolio_compare varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.sql new file mode 100644 index 0000000..3bd7204 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32.sql @@ -0,0 +1,27 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + operation varchar2(255 char), + figure_date date, + by_maturity_period varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.sql new file mode 100644 index 0000000..b6efa13 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36.sql @@ -0,0 +1,28 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + operation varchar2(255 char), + figure_date date, + by_portfolio varchar2(255 char), + by_maturity_period varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.sql new file mode 100644 index 0000000..cc70b8d --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4.sql @@ -0,0 +1,27 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id varchar2(255), + operation varchar2(255 char), + by_portfolio varchar2(255 char), + figure_date date, + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.sql new file mode 100644 index 0000000..0630945 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5.sql @@ -0,0 +1,28 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + operation varchar2(255 char), + by_currency varchar2(255 char), + by_portfolio varchar2(255 char), + figure_date date, + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.sql new file mode 100644 index 0000000..940cc29 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96.sql @@ -0,0 +1,28 @@ +create table ct_et_templates.w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96 ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + data_set_id number(12,0), + figure_date date, + operation varchar2(255 char), + by_maturity_period varchar2(255 char), + by_instrument varchar2(255 char), + accrued_book_value__normal binary_double, + accrued_book_value_interest binary_double, + accrued_interest__normal binary_double, + accrued_profit__normal binary_double, + nominal_amount binary_double, + market_value__normal binary_double, + commited_mv_total binary_double, + underlying_market_value__normal binary_double, + ir_exposure_1__normal binary_double, + modified_duration__normal binary_double, + effective_duration__contribution_to_total binary_double, + value_at_risk__normal binary_double, + value_at_risk_fx__normal binary_double, + value_at_risk_ir__normal binary_double, + maturity__normal binary_double, + value_at_risk_ir__delta_reference_vector binary_double, + value_at_risk_ir__delta_reference_vector_relative_to_total_mv binary_double, + amount__normal binary_double, + effective_convexity__normal binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEPERMISSIONS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEPERMISSIONS.sql new file mode 100644 index 0000000..2af04ab --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEPERMISSIONS.sql @@ -0,0 +1,21 @@ +create table ct_et_templates.w_ODS_TMS_EFFECTIVEPERMISSIONS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(512 char), + name varchar2(512 char), + admin_center_id varchar2(512 char), + time_zone_id varchar2(512 char), + menu_name varchar2(512 char), + web_home_group_id varchar2(512 char), + comment_ varchar2(512 char), + flags number(12, 0), + access_profile_id varchar2(512 char), + access_profile_name varchar2(512 char), + access_profile_type_id varchar2(512 char), + access_profile_type_name varchar2(512 char), + entity_id varchar2(512 char), + permission_id varchar2(512 char), + resource_rule varchar2(512 char), + active_from date, + active_to date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEROLEPROFILE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEROLEPROFILE.sql new file mode 100644 index 0000000..3346f99 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_EFFECTIVEROLEPROFILE.sql @@ -0,0 +1,20 @@ +create table ct_et_templates.w_ODS_TMS_EFFECTIVEROLEPROFILE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + name varchar2(255 char), + admin_center_id varchar2(255 char), + time_zone_id varchar2(255 char), + menu_name varchar2(255 char), + web_home_group_id varchar2(255 char), + flags number(12, 0), + access_role_id varchar2(255 char), + access_role_name varchar2(255 char), + direct_p number(12, 0), + active_from date, + active_to date, + access_profile_id varchar2(255 char), + access_profile_name varchar2(255 char), + access_profile_type_id varchar2(255 char), + access_profile_type_name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_FINMESSAGELOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_FINMESSAGELOG.sql new file mode 100644 index 0000000..f92d13f --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_FINMESSAGELOG.sql @@ -0,0 +1,77 @@ +create table ct_et_templates.w_ODS_TMS_FINMESSAGELOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + call_movement_id number(12, 0), + confirmation_reference varchar2(4000 char), + confirmation_time varchar2(4000 char), + cp_client_id varchar2(4000 char), + cp_client_unit_id varchar2(4000 char), + currency_id varchar2(4000 char), + date_ date, + delivery varchar2(4000 char), + failed varchar2(4000 char), + fixing_id number(12, 0), + flags number(12, 0), + id number(12, 0), + input_time date, + instrument_id varchar2(4000 char), + interface_info varchar2(4000 char), + leg_id varchar2(4000 char), + log_action varchar2(4000 char), + log_caller varchar2(4000 char), + log_date date, + log_serial number(12, 0), + log_type varchar2(4000 char), + log_user varchar2(4000 char), + message_input_reference varchar2(4000 char), + number_ number(12, 0), + operation varchar2(4000 char), + output_time date, + owner_id varchar2(4000 char), + owner_unit_id varchar2(4000 char), + param_0 varchar2(4000 char), + param_1 varchar2(4000 char), + param_10 varchar2(4000 char), + param_11 varchar2(4000 char), + param_12 varchar2(4000 char), + param_13 varchar2(4000 char), + param_14 varchar2(4000 char), + param_15 varchar2(4000 char), + param_16 varchar2(4000 char), + param_17 varchar2(4000 char), + param_18 varchar2(4000 char), + param_19 varchar2(4000 char), + param_2 varchar2(4000 char), + param_3 varchar2(4000 char), + param_4 varchar2(4000 char), + param_5 varchar2(4000 char), + param_6 varchar2(4000 char), + param_7 varchar2(4000 char), + param_8 varchar2(4000 char), + param_9 varchar2(4000 char), + portfolio_id varchar2(4000 char), + receiver_swift_code varchar2(4000 char), + reference varchar2(4000 char), + related_reference varchar2(4000 char), + release_batch_id number(12, 0), + sender_swift_code varchar2(4000 char), + sending varchar2(4000 char), + sent varchar2(4000 char), + sent_date_time date, + service_type_id varchar2(4000 char), + session_id varchar2(4000 char), + session_sequence varchar2(4000 char), + settlement_id number(12, 0), + source_system_id varchar2(4000 char), + state_id varchar2(4000 char), + status_description varchar2(4000 char), + stp varchar2(4000 char), + system_id number(12, 0), + target_system varchar2(4000 char), + type varchar2(4000 char), + uetr varchar2(4000 char), + urgent varchar2(4000 char), + uti varchar2(4000 char), + version_id varchar2(4000 char), + warning varchar2(4000 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_HISTORY_LOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_HISTORY_LOG.sql new file mode 100644 index 0000000..90f339c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_HISTORY_LOG.sql @@ -0,0 +1,16 @@ +create table ct_et_templates.w_ODS_TMS_HISTORY_LOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + object_id varchar2(4000 char), + action varchar2(4000 char), + user_id varchar2(4000 char), + date_ date, + key_1_string varchar2(4000 char), + field_name varchar2(4000 char), + key_2_string varchar2(4000 char), + key_3_string varchar2(4000 char), + key_4_string varchar2(4000 char), + from_value varchar2(4000 char), + to_value varchar2(4000 char), + domain_id varchar2(4000 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.sql new file mode 100644 index 0000000..567ade8 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENTBONDCASHFLOW.sql @@ -0,0 +1,82 @@ +create table ct_et_templates.w_ODS_TMS_INSTRUMENTBONDCASHFLOW ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + accrual_date_basis varchar2(255 char), + active_since date, + active_until date, + actual_amount binary_double, + actual_currency_id varchar2(255 char), + actual_fx_rate binary_double, + amount binary_double, + amount_rounding binary_double, + cashflow_group number(12, 0), + category_id varchar2(255 char), + cf_flags varchar2(255 char), + cf_flags_2 varchar2(255 char), + cf_keep_flags varchar2(255 char), + currency_id varchar2(255 char), + date_basis varchar2(255 char), + effective_fixing_date date, + ex_date date, + expression varchar2(255 char), + fixing_cap binary_double, + fixing_date date, + fixing_divider binary_double, + fixing_floor binary_double, + fixing_from date, + fixing_param binary_double, + fixing_period_2_id varchar2(255 char), + fixing_period_id varchar2(255 char), + fixing_price binary_double, + fixing_price_2 binary_double, + fixing_processing_date date, + fixing_quote binary_double, + fixing_rate_id varchar2(255 char), + fixing_subscenario_id varchar2(255 char), + fixing_to date, + fixing_type varchar2(255 char), + flags number(12, 0), + freeze_date date, + instrument_accrued_interest varchar2(255 char), + instrument_active_since date, + instrument_active_until date, + instrument_amount_rounding binary_double, + instrument_buy_label varchar2(255 char), + instrument_coupon_rate binary_double, + instrument_currency_id varchar2(255 char), + instrument_default_price_denom number(12, 0), + instrument_domain_id varchar2(255 char), + instrument_flags number(12, 0), + instrument_group varchar2(255 char), + instrument_id varchar2(255 char), + instrument_issuer_id varchar2(255 char), + instrument_issuer_unit_id varchar2(255 char), + instrument_layout_id varchar2(255 char), + instrument_minimum_price_denom number(12, 0), + instrument_name varchar2(255 char), + instrument_result_type_id varchar2(255 char), + instrument_sell_label varchar2(255 char), + instrument_settlement_flags number(12, 0), + instrument_sign_id number(12, 0), + instrument_template_id varchar2(255 char), + instrument_type_id varchar2(255 char), + key_id number(12, 0), + kinds varchar2(255 char), + origin_id number(12, 0), + origin_value binary_double, + payment_date date, + realize_date date, + referee_id number(12, 0), + reference_date date, + reference_id number(12, 0), + sign varchar2(255 char), + since_when date, + spread binary_double, + subcategory_id varchar2(255 char), + subtype_id number(12, 0), + type_id number(12, 0), + type_name varchar2(255 char), + umi_id varchar2(255 char), + until_when date, + value_date date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.sql new file mode 100644 index 0000000..6c08b25 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE.sql @@ -0,0 +1,12 @@ +create table ct_et_templates.w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + instrument_active_until date, + instrument_id varchar2(255 char), + frequency number(12, 0), + fixing_rate_id varchar2(255 char), + spread binary_double, + floor binary_double, + cap binary_double, + factor binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_REPORT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_REPORT.sql new file mode 100644 index 0000000..07cda16 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_INSTRUMENT_REPORT.sql @@ -0,0 +1,60 @@ +create table ct_et_templates.w_ODS_TMS_INSTRUMENT_REPORT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + type_id varchar2(255 char), + domain_id varchar2(255 char), + flags varchar2(255 char), + name varchar2(255 char), + layout_id varchar2(255 char), + active_since date, + active_until date, + buy_label varchar2(255 char), + sell_label varchar2(255 char), + path_id number(12, 0), + result_type_id varchar2(255 char), + template_id varchar2(255 char), + instrument_group varchar2(255 char), + security_identifier_type varchar2(255 char), + security_identifier varchar2(255 char), + branch0_id varchar2(255 char), + branch1_id varchar2(255 char), + branch2_id varchar2(255 char), + branch3_id varchar2(255 char), + branch4_id varchar2(255 char), + branch5_id varchar2(255 char), + branch6_id varchar2(255 char), + branch7_id varchar2(255 char), + branch8_id varchar2(255 char), + branch9_id varchar2(255 char), + branch10_id varchar2(255 char), + branch11_id varchar2(255 char), + branch12_id varchar2(255 char), + branch13_id varchar2(255 char), + branch14_id varchar2(255 char), + branch15_id varchar2(255 char), + branch16_id varchar2(255 char), + branch17_id varchar2(255 char), + branch18_id varchar2(255 char), + branch19_id varchar2(255 char), + branch20_id varchar2(255 char), + branch21_id varchar2(255 char), + branch22_id varchar2(255 char), + branch23_id varchar2(255 char), + branch24_id varchar2(255 char), + branch25_id varchar2(255 char), + branch26_id varchar2(255 char), + branch27_id varchar2(255 char), + branch28_id varchar2(255 char), + branch29_id varchar2(255 char), + branch30_id varchar2(255 char), + branch31_id varchar2(255 char), + branch32_id varchar2(255 char), + branch33_id varchar2(255 char), + branch34_id varchar2(255 char), + branch35_id varchar2(255 char), + branch36_id varchar2(255 char), + branch37_id varchar2(255 char), + branch38_id varchar2(255 char), + branch39_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_MOPDBPRICES.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_MOPDBPRICES.sql new file mode 100644 index 0000000..e4cf39f --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_MOPDBPRICES.sql @@ -0,0 +1,31 @@ +create table ct_et_templates.w_ODS_TMS_MOPDBPRICES ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + target_date date, + type_id varchar2(255 char), + name_id varchar2(255 char), + id number(12, 0), + scenario_id varchar2(255 char), + subscenario_id varchar2(255 char), + source_id varchar2(255 char), + item varchar2(255 char), + mi_group varchar2(255 char), + currency_id varchar2(255 char), + period_from date, + period_to date, + period_id varchar2(255 char), + period_1_id varchar2(255 char), + period_2_id varchar2(255 char), + price_type varchar2(255 char), + date_basis varchar2(255 char), + date_ date, + date_time date, + price_1 binary_double, + price_2 binary_double, + bid binary_double, + ask binary_double, + maturity_date date, + security_identifier varchar2(255 char), + security_identifier_type varchar2(255 char), + instrument_group varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_PARAMETER.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PARAMETER.sql new file mode 100644 index 0000000..7fb0902 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PARAMETER.sql @@ -0,0 +1,9 @@ +create table ct_et_templates.w_ODS_TMS_PARAMETER ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + type_id varchar2(255 char), + number_ number(12, 0), + order_number number(12, 0), + param varchar2(255 char), + name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.sql new file mode 100644 index 0000000..031c05d --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_POS_PERF_ACTIVITY_STATUS.sql @@ -0,0 +1,7 @@ +create table ct_et_templates.w_ODS_TMS_POS_PERF_ACTIVITY_STATUS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + due_date date, + finish_date date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_PRICE_RATE_REPORT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PRICE_RATE_REPORT.sql new file mode 100644 index 0000000..52def3c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PRICE_RATE_REPORT.sql @@ -0,0 +1,17 @@ +create table ct_et_templates.w_ODS_TMS_PRICE_RATE_REPORT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + name_id varchar2(255 char), + period_id varchar2(255 char), + bid binary_double, + ask binary_double, + mid binary_double, + price_type_csd varchar2(255 char), + currency_id varchar2(255 char), + date_ date, + date_time date, + isin varchar2(255 char), + reuters_item varchar2(255 char), + instr_group varchar2(255 char), + scenario_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_PROPERTY.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PROPERTY.sql new file mode 100644 index 0000000..e471092 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_PROPERTY.sql @@ -0,0 +1,14 @@ +create table ct_et_templates.w_ODS_TMS_PROPERTY ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + object_id varchar2(255 char), + key_id varchar2(255 char), + key_2_id varchar2(255 char), + key_3_id varchar2(255 char), + order_number number(12, 0), + type_id varchar2(255 char), + value_id varchar2(255 char), + name varchar2(255 char), + flags varchar2(255 char), + validate_hook varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENT.sql new file mode 100644 index 0000000..e61d872 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENT.sql @@ -0,0 +1,16 @@ +create table ct_et_templates.w_ODS_TMS_RAR_CLIENT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + client varchar2(255 char), + domain varchar2(255 char), + country varchar2(255 char), + roles varchar2(255 char), + name varchar2(255 char), + parent varchar2(255 char), + postal_address varchar2(255 char), + street_address varchar2(255 char), + city varchar2(255 char), + risk_country varchar2(255 char), + swift_code varchar2(255 char), + flags varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENTGROUPMAP.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENTGROUPMAP.sql new file mode 100644 index 0000000..d6140d6 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_CLIENTGROUPMAP.sql @@ -0,0 +1,9 @@ +create table ct_et_templates.w_ODS_TMS_RAR_CLIENTGROUPMAP ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + client_id varchar2(255 char), + main_group_id varchar2(255 char), + main_group_name varchar2(255 char), + group_id varchar2(255 char), + group_name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_ECBINSTRUMENTS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_ECBINSTRUMENTS.sql new file mode 100644 index 0000000..81017c1 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_ECBINSTRUMENTS.sql @@ -0,0 +1,10 @@ +create table ct_et_templates.w_ODS_TMS_RAR_ECBINSTRUMENTS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + instrument_id varchar2(255 char), + figure_date date, + ctd_instrument_id varchar2(255 char), + modified_duration binary_double, + ytm binary_double, + convexity binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FRM_HOLIDAYS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FRM_HOLIDAYS.sql new file mode 100644 index 0000000..041ebcd --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FRM_HOLIDAYS.sql @@ -0,0 +1,9 @@ +create table ct_et_templates.w_ODS_TMS_RAR_FRM_HOLIDAYS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + name varchar2(255 char), + holiday_date date, + holiday_reason varchar2(255 char), + default_holidays varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FUTURE_BONDS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FUTURE_BONDS.sql new file mode 100644 index 0000000..c26b76c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_FUTURE_BONDS.sql @@ -0,0 +1,8 @@ +create table ct_et_templates.w_ODS_TMS_RAR_FUTURE_BONDS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + instrument_id varchar2(255 char), + conversion_factor binary_double, + flags number(12, 0) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_INSTRUMENT_REPORT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_INSTRUMENT_REPORT.sql new file mode 100644 index 0000000..5d02ab1 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_INSTRUMENT_REPORT.sql @@ -0,0 +1,58 @@ +create table ct_et_templates.w_ODS_TMS_RAR_INSTRUMENT_REPORT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + type_id varchar2(255 char), + name varchar2(255 char), + instrument_group varchar2(255 char), + instrument_group_name varchar2(255 char), + issue_date date, + issuer varchar2(255 char), + maturity_date date, + currency_id varchar2(255 char), + coupon_rate binary_double, + date_basis varchar2(255 char), + coupons_per_year binary_double, + active_since date, + active_until date, + branch0_id varchar2(255 char), + branch1_id varchar2(255 char), + branch2_id varchar2(255 char), + branch3_id varchar2(255 char), + branch4_id varchar2(255 char), + branch5_id varchar2(255 char), + branch6_id varchar2(255 char), + branch7_id varchar2(255 char), + branch8_id varchar2(255 char), + branch9_id varchar2(255 char), + branch10_id varchar2(255 char), + branch11_id varchar2(255 char), + branch12_id varchar2(255 char), + branch13_id varchar2(255 char), + branch14_id varchar2(255 char), + branch15_id varchar2(255 char), + branch16_id varchar2(255 char), + branch17_id varchar2(255 char), + branch18_id varchar2(255 char), + branch19_id varchar2(255 char), + branch20_id varchar2(255 char), + branch21_id varchar2(255 char), + branch22_id varchar2(255 char), + branch23_id varchar2(255 char), + branch24_id varchar2(255 char), + branch25_id varchar2(255 char), + branch26_id varchar2(255 char), + branch27_id varchar2(255 char), + branch28_id varchar2(255 char), + branch29_id varchar2(255 char), + branch30_id varchar2(255 char), + branch31_id varchar2(255 char), + branch32_id varchar2(255 char), + branch33_id varchar2(255 char), + branch34_id varchar2(255 char), + branch35_id varchar2(255 char), + branch36_id varchar2(255 char), + branch37_id varchar2(255 char), + branch38_id varchar2(255 char), + branch39_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMIT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMIT.sql new file mode 100644 index 0000000..92fac16 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMIT.sql @@ -0,0 +1,14 @@ +create table ct_et_templates.w_ODS_TMS_RAR_LIMIT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + domain_id varchar2(255 char), + category_id varchar2(255 char), + usage_log_type binary_double, + name varchar2(255 char), + global_client_query_id varchar2(255 char), + limit_user_group_id varchar2(255 char), + comment_ varchar2(255 char), + flags number(12, 0), + template_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITACCESS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITACCESS.sql new file mode 100644 index 0000000..e8685c0 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITACCESS.sql @@ -0,0 +1,7 @@ +create table ct_et_templates.w_ODS_TMS_RAR_LIMITACCESS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + limit_id varchar2(255 char), + user_id varchar2(255 char), + ecbusrbdts date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITLOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITLOG.sql new file mode 100644 index 0000000..41030ba --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_LIMITLOG.sql @@ -0,0 +1,111 @@ +create table ct_et_templates.w_ODS_TMS_RAR_LIMITLOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + date_ date, + time_utc date, + sublimit_id varchar2(255 char), + amount number(19,4), + old_amount number(19,4), + violation_type number(12, 0), + violation_p number(12, 0), + minimum_amount number(19,4), + maximum_amount number(19,4), + item_id number(12, 0), + number_ number(12, 0), + portfolio_id varchar2(255 char), + package_type_id varchar2(255 char), + package_id number(12, 0), + market_id varchar2(255 char), + instrument_id varchar2(255 char), + currency_id varchar2(255 char), + currency_class_id varchar2(255 char), + cp_client_id varchar2(255 char), + credit_client_id varchar2(255 char), + issuer_id varchar2(255 char), + underlying_issuer_id varchar2(255 char), + leg_id varchar2(255 char), + payment_date date, + bank_id varchar2(255 char), + account_id varchar2(255 char), + branch0_id varchar2(255 char), + branch1_id varchar2(255 char), + branch2_id varchar2(255 char), + branch3_id varchar2(255 char), + branch4_id varchar2(255 char), + branch5_id varchar2(255 char), + branch6_id varchar2(255 char), + branch7_id varchar2(255 char), + branch8_id varchar2(255 char), + branch9_id varchar2(255 char), + branch10_id varchar2(255 char), + branch11_id varchar2(255 char), + branch12_id varchar2(255 char), + branch13_id varchar2(255 char), + branch14_id varchar2(255 char), + branch15_id varchar2(255 char), + branch16_id varchar2(255 char), + branch17_id varchar2(255 char), + branch18_id varchar2(255 char), + branch19_id varchar2(255 char), + causing_number number(12, 0), + server_id varchar2(255 char), + delivery_gap_id varchar2(255 char), + maturity_gap_id varchar2(255 char), + risk_gap_id varchar2(255 char), + instrument_path varchar2(255 char), + affect_date date, + affect_gap_id varchar2(255 char), + branch20_id varchar2(255 char), + branch21_id varchar2(255 char), + branch22_id varchar2(255 char), + branch23_id varchar2(255 char), + branch24_id varchar2(255 char), + branch25_id varchar2(255 char), + branch26_id varchar2(255 char), + branch27_id varchar2(255 char), + branch28_id varchar2(255 char), + branch29_id varchar2(255 char), + branch30_id varchar2(255 char), + branch31_id varchar2(255 char), + branch32_id varchar2(255 char), + branch33_id varchar2(255 char), + branch34_id varchar2(255 char), + branch35_id varchar2(255 char), + branch36_id varchar2(255 char), + branch37_id varchar2(255 char), + branch38_id varchar2(255 char), + branch39_id varchar2(255 char), + client_group_id varchar2(255 char), + client_id varchar2(255 char), + client_main_group_id varchar2(255 char), + client_rating_code_id varchar2(255 char), + client_rating_id varchar2(255 char), + delivery_date date, + guarantee_level number(12, 0), + guarantee_param_0 varchar2(255 char), + guarantee_param_1 varchar2(255 char), + guarantee_param_2 varchar2(255 char), + guarantee_param_3 varchar2(255 char), + guarantee_param_4 varchar2(255 char), + guarantee_param_5 varchar2(255 char), + guarantee_param_6 varchar2(255 char), + guarantee_param_7 varchar2(255 char), + guarantee_param_8 varchar2(255 char), + guarantee_param_9 varchar2(255 char), + guarantee_type_id varchar2(255 char), + limit_currency_id varchar2(255 char), + log_id number(12, 0), + owner_id varchar2(255 char), + package_main_type_id varchar2(255 char), + parent_id varchar2(255 char), + risk_country_id varchar2(255 char), + umi_rating_code_id varchar2(255 char), + umi_rating_id varchar2(255 char), + underlying_instrument_id varchar2(255 char), + underlying_maturity_gap_id varchar2(255 char), + adjusted_risk_gap_id varchar2(255 char), + leg_group number(12, 0), + leg_path_id varchar2(255 char), + position_type varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_MARKETINFO.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_MARKETINFO.sql new file mode 100644 index 0000000..37070b1 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_MARKETINFO.sql @@ -0,0 +1,27 @@ +create table ct_et_templates.w_ODS_TMS_RAR_MARKETINFO ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + item varchar2(255 char), + name_id varchar2(255 char), + scenario_id varchar2(255 char), + subscenario_id varchar2(255 char), + type_id varchar2(255 char), + period_id varchar2(255 char), + period_1_id varchar2(255 char), + period_2_id varchar2(255 char), + flags varchar2(255 char), + source varchar2(255 char), + producer varchar2(255 char), + group_tag varchar2(255 char), + bid_tag varchar2(255 char), + ask_tag varchar2(255 char), + date_tag varchar2(255 char), + change_period number(12, 0), + change_percent binary_double, + time_zone_id varchar2(255 char), + valid_from_time number(12, 0), + valid_to_time number(12, 0), + calendar_id varchar2(255 char), + divider binary_double, + price_type varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIO.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIO.sql new file mode 100644 index 0000000..f87d76e --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIO.sql @@ -0,0 +1,17 @@ +create table ct_et_templates.w_ODS_TMS_RAR_PORTFOLIO ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + client_id varchar2(255 char), + domain_id varchar2(255 char), + opening_date date, + valuate_flags number(12, 0), + flags varchar2(255 char), + name varchar2(255 char), + template_id varchar2(255 char), + currency_id varchar2(255 char), + closing_date date, + user_id varchar2(255 char), + risk_rate binary_double, + client_unit_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOACCESS.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOACCESS.sql new file mode 100644 index 0000000..3a8b9bc --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOACCESS.sql @@ -0,0 +1,7 @@ +create table ct_et_templates.TMS_RAR_PORTFOLIOACCESS ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + portfolio_id varchar2(255 char), + user_id varchar2(255 char), + ecbusrbdts date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOTREE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOTREE.sql new file mode 100644 index 0000000..ffd0bdd --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RAR_PORTFOLIOTREE.sql @@ -0,0 +1,7 @@ +create table ct_et_templates.w_ODS_TMS_RAR_PORTFOLIOTREE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + parent_id varchar2(255 char), + child_id varchar2(255 char), + factor binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RECONCILIATION.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RECONCILIATION.sql new file mode 100644 index 0000000..1286d0c --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RECONCILIATION.sql @@ -0,0 +1,30 @@ +create table ct_et_templates.w_ODS_TMS_RECONCILIATION ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + account_id varchar2(512 char), + amount number(19,4), + balance number(19,4), + bank_id varchar2(512 char), + bank_unit_id varchar2(512 char), + case_comment varchar2(512 char), + case_id number(12, 0), + case_reason_code varchar2(512 char), + client_id varchar2(512 char), + client_unit_id varchar2(512 char), + cp_client_id varchar2(512 char), + cp_client_unit_id varchar2(512 char), + currency_class_id varchar2(512 char), + currency_id varchar2(512 char), + instrument_id varchar2(512 char), + interest number(19,4), + item_type varchar2(512 char), + number_ number(12, 0), + payment_date date, + payment_statement_id number(12, 0), + portfolio_id varchar2(512 char), + rate binary_double, + recon_group number(12, 0), + recon_group_comment varchar2(512 char), + recon_group_log varchar2(512 char), + recon_status varchar2(512 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_ROLEPORTFOLIOPROFILE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ROLEPORTFOLIOPROFILE.sql new file mode 100644 index 0000000..4a7b8e8 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_ROLEPORTFOLIOPROFILE.sql @@ -0,0 +1,10 @@ +create table ct_et_templates.w_ODS_TMS_ROLEPORTFOLIOPROFILE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + role varchar2(512 char), + inherited_role varchar2(512 char), + access_profile_id varchar2(512 char), + access_profile_type_id varchar2(512 char), + permission_id varchar2(512 char), + entity_id varchar2(512 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_RULES.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RULES.sql new file mode 100644 index 0000000..bd12252 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_RULES.sql @@ -0,0 +1,166 @@ +create table ct_et_templates.w_ODS_TMS_RULES ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + sdm_object_id number(12, 0), + sdm_key_id number(12, 0), + sdm_create_date date, + sdm_access_level number(12, 0), + sdm_active number(12, 0), + id varchar2(255 char), + number_ number(12, 0), + priority number(12, 0), + portfolio_id varchar2(255 char), + instrument_path_id varchar2(255 char), + status varchar2(255 char), + not_status varchar2(255 char), + cf_leg_path_id varchar2(255 char), + cf_flags varchar2(255 char), + cf_not_flags varchar2(255 char), + flags number(12, 0), + name varchar2(255 char), + client_id varchar2(255 char), + client_main_group_id varchar2(255 char), + client_group_id varchar2(255 char), + user_id varchar2(255 char), + currency_id varchar2(255 char), + currency_2_id varchar2(255 char), + instrument_id varchar2(255 char), + secondary_instrument_id varchar2(255 char), + package_main_type_id varchar2(255 char), + package_type_id varchar2(255 char), + issuer_id varchar2(255 char), + issuer_main_group_id varchar2(255 char), + issuer_group_id varchar2(255 char), + cp_client_id varchar2(255 char), + cp_client_main_group_id varchar2(255 char), + cp_client_group_id varchar2(255 char), + cp_portfolio_id varchar2(255 char), + broker_id varchar2(255 char), + branch0_id varchar2(255 char), + branch1_id varchar2(255 char), + branch2_id varchar2(255 char), + branch3_id varchar2(255 char), + branch4_id varchar2(255 char), + param_0 varchar2(255 char), + param_1 varchar2(255 char), + param_2 varchar2(255 char), + param_3 varchar2(255 char), + param_4 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_9 varchar2(255 char), + param_10 varchar2(255 char), + param_11 varchar2(255 char), + param_12 varchar2(255 char), + param_13 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_16 varchar2(255 char), + param_17 varchar2(255 char), + param_18 varchar2(255 char), + param_19 varchar2(255 char), + contexts number(12, 0), + not_contexts number(12, 0), + transaction_flags number(12, 0), + transaction_not_flags number(12, 0), + sign_id varchar2(255 char), + type_id varchar2(255 char), + kind_id varchar2(255 char), + state_id varchar2(255 char), + cf_status number(12, 0), + cf_not_status number(12, 0), + cf_currency_id varchar2(255 char), + cf_payment_currency_id varchar2(255 char), + cf_sign varchar2(255 char), + cf_type_id number(12, 0), + cf_subtype_id number(12, 0), + cf_category_id varchar2(255 char), + cf_subcategory_id varchar2(255 char), + cf_leg_group number(12, 0), + cf_leg_id varchar2(255 char), + cf_branch0_id varchar2(255 char), + cf_branch1_id varchar2(255 char), + cf_branch2_id varchar2(255 char), + cf_branch3_id varchar2(255 char), + cf_branch4_id varchar2(255 char), + cf_flags_2 varchar2(255 char), + cf_not_flags_2 varchar2(255 char), + cf_kinds varchar2(255 char), + cf_not_kinds varchar2(255 char), + branch10_id varchar2(255 char), + branch11_id varchar2(255 char), + branch12_id varchar2(255 char), + branch13_id varchar2(255 char), + branch14_id varchar2(255 char), + branch15_id varchar2(255 char), + branch16_id varchar2(255 char), + branch17_id varchar2(255 char), + branch18_id varchar2(255 char), + branch19_id varchar2(255 char), + branch20_id varchar2(255 char), + branch21_id varchar2(255 char), + branch22_id varchar2(255 char), + branch23_id varchar2(255 char), + branch24_id varchar2(255 char), + branch25_id varchar2(255 char), + branch26_id varchar2(255 char), + branch27_id varchar2(255 char), + branch28_id varchar2(255 char), + branch29_id varchar2(255 char), + branch30_id varchar2(255 char), + branch31_id varchar2(255 char), + branch32_id varchar2(255 char), + branch33_id varchar2(255 char), + branch34_id varchar2(255 char), + branch35_id varchar2(255 char), + branch36_id varchar2(255 char), + branch37_id varchar2(255 char), + branch38_id varchar2(255 char), + branch39_id varchar2(255 char), + branch5_id varchar2(255 char), + branch6_id varchar2(255 char), + branch7_id varchar2(255 char), + branch8_id varchar2(255 char), + branch9_id varchar2(255 char), + cf_branch10_id varchar2(255 char), + cf_branch11_id varchar2(255 char), + cf_branch12_id varchar2(255 char), + cf_branch13_id varchar2(255 char), + cf_branch14_id varchar2(255 char), + cf_branch15_id varchar2(255 char), + cf_branch16_id varchar2(255 char), + cf_branch17_id varchar2(255 char), + cf_branch18_id varchar2(255 char), + cf_branch19_id varchar2(255 char), + cf_branch20_id varchar2(255 char), + cf_branch21_id varchar2(255 char), + cf_branch22_id varchar2(255 char), + cf_branch23_id varchar2(255 char), + cf_branch24_id varchar2(255 char), + cf_branch25_id varchar2(255 char), + cf_branch26_id varchar2(255 char), + cf_branch27_id varchar2(255 char), + cf_branch28_id varchar2(255 char), + cf_branch29_id varchar2(255 char), + cf_branch30_id varchar2(255 char), + cf_branch31_id varchar2(255 char), + cf_branch32_id varchar2(255 char), + cf_branch33_id varchar2(255 char), + cf_branch34_id varchar2(255 char), + cf_branch35_id varchar2(255 char), + cf_branch36_id varchar2(255 char), + cf_branch37_id varchar2(255 char), + cf_branch38_id varchar2(255 char), + cf_branch39_id varchar2(255 char), + cf_branch5_id varchar2(255 char), + cf_branch6_id varchar2(255 char), + cf_branch7_id varchar2(255 char), + cf_branch8_id varchar2(255 char), + cf_branch9_id varchar2(255 char), + client_unit_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + issuer_unit_id varchar2(255 char), + cf_type_name varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_SDM_ENTITY_STATE.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SDM_ENTITY_STATE.sql new file mode 100644 index 0000000..d0f722a --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SDM_ENTITY_STATE.sql @@ -0,0 +1,21 @@ +create table ct_et_templates.w_ODS_TMS_SDM_ENTITY_STATE ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id number(12, 0), + key_id number(12, 0), + state_id number(12, 0), + state_name varchar2(255 char), + state_edit_allowed number(12, 0), + no_state_transition number(12, 0), + category varchar2(255 char), + status varchar2(255 char), + entity_name varchar2(255 char), + entered_by varchar2(255 char), + updated_by varchar2(255 char), + initiated_by varchar2(255 char), + entity_type varchar2(255 char), + state_log_updated_by varchar2(255 char), + state_log_from_state varchar2(255 char), + state_log_to_state varchar2(255 char), + state_log_update_time date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_SECURITYPOSITION.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SECURITYPOSITION.sql new file mode 100644 index 0000000..0480c89 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SECURITYPOSITION.sql @@ -0,0 +1,49 @@ +create table ct_et_templates.w_ODS_TMS_SECURITYPOSITION ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + date_ date, + portfolio_id varchar2(255 char), + instrument_id varchar2(255 char), + contexts number(12, 0), + currency_id varchar2(255 char), + package_main_type_id varchar2(255 char), + package_type_id varchar2(255 char), + package_id number(12, 0), + inflow_nominal_amount number(19,4), + outflow_nominal_amount number(19,4), + nominal_amount number(19,4), + inflow_units number(19,4), + outflow_units number(19,4), + units number(19,4), + inflow_book_value number(19,4), + outflow_book_value number(19,4), + book_value number(19,4), + redemption_amount number(19,4), + accrued_profit number(19,4), + indexation_profit number(19,4), + discount_premium number(19,4), + accrued_book_value number(19,4), + realized_mtom_profit number(19,4), + realized_accrued_profit number(19,4), + realized_indexation_profit number(19,4), + bvc_mtom_profit number(19,4), + mtom_profit number(19,4), + market_value number(19,4), + accrued_interest number(19,4), + accrued_interest_bought number(19,4), + accrued_interest_sold number(19,4), + yield binary_double, + price binary_double, + market_quote binary_double, + index_value binary_double, + issuer_id varchar2(255 char), + issue_date date, + maturity_date date, + isin_code varchar2(255 char), + accrual_method varchar2(255 char), + accrual_interest_type varchar2(255 char), + accrual_date_basis varchar2(255 char), + input_date date, + flags number(12, 0), + face_amount number(19,4) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTCASHFLOW.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTCASHFLOW.sql new file mode 100644 index 0000000..cc3f14d --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTCASHFLOW.sql @@ -0,0 +1,90 @@ +create table ct_et_templates.w_ODS_TMS_SETTLEMENTCASHFLOW ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + accepted_by varchar2(255 char), + advice_method varchar2(255 char), + amount number(19,4), + call_movement_trader_id varchar2(255 char), + cashflow_amount number(19,4), + cashflow_currency_id varchar2(255 char), + cashflow_id number(12, 0), + cashflow_param_0 varchar2(255 char), + cashflow_param_1 varchar2(255 char), + cashflow_param_2 varchar2(255 char), + cashflow_param_3 varchar2(255 char), + cashflow_param_4 varchar2(255 char), + cashflow_type_name varchar2(255 char), + class_id varchar2(255 char), + cp_client_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + currency_id varchar2(255 char), + delivery_instrument varchar2(255 char), + flags number(12, 0), + id number(12, 0), + local_account_1_id varchar2(255 char), + local_account_2_id varchar2(255 char), + local_account_3_id varchar2(255 char), + local_account_4_id varchar2(255 char), + local_bank_1_id varchar2(255 char), + local_bank_1_swift_code varchar2(255 char), + local_bank_1_unit_id varchar2(255 char), + local_bank_2_id varchar2(255 char), + local_bank_2_swift_code varchar2(255 char), + local_bank_3_id varchar2(255 char), + local_bank_3_swift_code varchar2(255 char), + local_bank_4_id varchar2(255 char), + local_bank_4_swift_code varchar2(255 char), + local_client_id varchar2(255 char), + local_client_swift_code varchar2(255 char), + local_client_unit_id varchar2(255 char), + netting_rule_id varchar2(255 char), + number_ number(12, 0), + other_account_1_id varchar2(255 char), + other_account_2_id varchar2(255 char), + other_account_3_id varchar2(255 char), + other_account_4_id varchar2(255 char), + other_bank_1_id varchar2(255 char), + other_bank_1_swift_code varchar2(255 char), + other_bank_1_unit_id varchar2(255 char), + other_bank_2_id varchar2(255 char), + other_bank_2_swift_code varchar2(255 char), + other_bank_3_id varchar2(255 char), + other_bank_3_swift_code varchar2(255 char), + other_bank_4_id varchar2(255 char), + other_bank_4_swift_code varchar2(255 char), + other_client_id varchar2(255 char), + other_client_swift_code varchar2(255 char), + other_client_unit_id varchar2(255 char), + owner_id varchar2(255 char), + owner_unit_id varchar2(255 char), + param_0 varchar2(255 char), + param_1 varchar2(255 char), + param_10 varchar2(255 char), + param_11 varchar2(255 char), + param_12 varchar2(255 char), + param_13 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_16 varchar2(255 char), + param_17 varchar2(255 char), + param_18 varchar2(255 char), + param_19 varchar2(255 char), + param_2 varchar2(255 char), + param_3 varchar2(255 char), + param_4 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_9 varchar2(255 char), + payment_date date, + payment_id number(12, 0), + portfolio_id varchar2(255 char), + settlement_client_id varchar2(255 char), + settlement_client_swift_code varchar2(255 char), + settlement_type varchar2(255 char), + split_source_id number(12, 0), + state_id varchar2(255 char), + state_id_sort_order number(12, 0), + transfer_method varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTLOG.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTLOG.sql new file mode 100644 index 0000000..588b430 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_SETTLEMENTLOG.sql @@ -0,0 +1,126 @@ +create table ct_et_templates.w_ODS_TMS_SETTLEMENTLOG ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + accepted_by varchar2(255 char), + action varchar2(255 char), + action_date date, + advice_type varchar2(255 char), + amount number(19,4), + cashflow_subtype_id number(12, 0), + cashflow_type_id number(12, 0), + code varchar2(255 char), + comment_ varchar2(255 char), + comment_1 varchar2(255 char), + comment_2 varchar2(255 char), + comment_3 varchar2(255 char), + comment_4 varchar2(255 char), + cp_client_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + currency_id varchar2(255 char), + date_time date, + delivery_units binary_double, + description varchar2(255 char), + error varchar2(255 char), + face_amount number(19,4), + flags number(12, 0), + group_id number(12, 0), + id number(12, 0), + instrument_id varchar2(255 char), + instrument_path_id number(12, 0), + last_user_id varchar2(255 char), + local_account_1_id varchar2(255 char), + local_account_2_id varchar2(255 char), + local_account_3_id varchar2(255 char), + local_account_4_id varchar2(255 char), + local_bank_1_id varchar2(255 char), + local_bank_2_id varchar2(255 char), + local_bank_1_swift_code varchar2(255 char), + local_bank_1_unit_id varchar2(255 char), + local_bank_2_swift_code varchar2(255 char), + local_bank_3_id varchar2(255 char), + local_bank_3_swift_code varchar2(255 char), + local_bank_4_id varchar2(255 char), + local_bank_4_swift_code varchar2(255 char), + local_client_id varchar2(255 char), + local_client_swift_code varchar2(255 char), + local_client_unit_id varchar2(255 char), + log_action varchar2(255 char), + log_caller varchar2(255 char), + log_date date, + log_event varchar2(255 char), + log_type varchar2(255 char), + log_user varchar2(255 char), + message_id varchar2(255 char), + message_type varchar2(255 char), + netting_criteria_id varchar2(255 char), + netting_cutoff_time date, + netting_flags number(12, 0), + netting_group_id varchar2(255 char), + netting_rule_id varchar2(255 char), + number_ number(12, 0), + on_behalf_account_id varchar2(255 char), + on_behalf_id varchar2(255 char), + on_behalf_swift_code varchar2(255 char), + orig_amount number(19,4), + orig_currency_id varchar2(255 char), + other_account_1_id varchar2(255 char), + other_account_2_id varchar2(255 char), + other_account_3_id varchar2(255 char), + other_account_4_id varchar2(255 char), + other_bank_1_id varchar2(255 char), + other_bank_1_swift_code varchar2(255 char), + other_bank_1_unit_id varchar2(255 char), + other_bank_2_id varchar2(255 char), + other_bank_2_swift_code varchar2(255 char), + other_bank_3_id varchar2(255 char), + other_bank_3_swift_code varchar2(255 char), + other_bank_4_id varchar2(255 char), + other_bank_4_swift_code varchar2(255 char), + other_client_id varchar2(255 char), + other_client_swift_code varchar2(255 char), + other_client_unit_id varchar2(255 char), + owner_id varchar2(255 char), + owner_unit_id varchar2(255 char), + param_0 varchar2(255 char), + param_1 varchar2(255 char), + param_10 varchar2(255 char), + param_11 varchar2(255 char), + param_12 varchar2(255 char), + param_13 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_16 varchar2(255 char), + param_17 varchar2(255 char), + param_18 varchar2(255 char), + param_19 varchar2(255 char), + param_2 varchar2(255 char), + param_3 varchar2(255 char), + param_4 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_9 varchar2(255 char), + payment_date date, + portfolio_id varchar2(255 char), + receiver varchar2(255 char), + related_funct_type varchar2(255 char), + related_message_type varchar2(255 char), + related_trmswift_id varchar2(255 char), + release_cutoff_time date, + sender varchar2(255 char), + settlement_client_id varchar2(255 char), + settlement_client_swift_code varchar2(255 char), + settlement_type_id varchar2(255 char), + sign number(12, 0), + source_type_id varchar2(255 char), + split_source_id number(12, 0), + state_id varchar2(255 char), + status_id varchar2(255 char), + trading_unit binary_double, + transfer_type varchar2(255 char), + trmswift_message varchar2(255 char), + trmswift_timestamp date, + uetr varchar2(255 char), + unique_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.sql new file mode 100644 index 0000000..2a126ad --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK.sql @@ -0,0 +1,17 @@ +create table ct_et_templates.w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + name_id varchar2(255 char), + period_id varchar2(255 char), + bid binary_double, + ask binary_double, + mid binary_double, + price_type_csd varchar2(255 char), + currency_id varchar2(255 char), + date_ date, + date_time date, + isin varchar2(255 char), + reuters_item varchar2(255 char), + instr_group varchar2(255 char), + scenario_id varchar2(255 char) +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_TRANSACTION.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_TRANSACTION.sql new file mode 100644 index 0000000..c102099 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_TRANSACTION.sql @@ -0,0 +1,139 @@ +create table ct_et_templates.w_ODS_TMS_TRANSACTION ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + last_log_id binary_double, + last_log_date date, + opening_date date, + date_ date, + value_date date, + maturity_date date, + maturity_code varchar2(255 char), + issuer_id varchar2(255 char), + amount binary_double, + book_value binary_double, + currency_id varchar2(255 char), + currency_2_id varchar2(255 char), + instrument_group varchar2(255 char), + instrument_id varchar2(255 char), + security_identifier varchar2(255 char), + portfolio_id varchar2(255 char), + number_ number(12, 0), + cp_client_id varchar2(255 char), + deal_price2 varchar2(255 char), + deal_rate binary_double, + type_id varchar2(255 char), + sign_id varchar2(255 char), + fx_rate varchar2(255 char), + fx_fwd_points varchar2(255 char), + state_id varchar2(255 char), + param_0 varchar2(255 char), + isin_code varchar2(255 char), + param_13 varchar2(255 char), + accrued_interest number(19,4), + agreed_amount binary_double, + amount_2 number(19,4), + audit_number number(12, 0), + batch_id number(12, 0), + cap binary_double, + closing_date date, + collateral_number number(12, 0), + comment_ varchar2(255 char), + comment_2 varchar2(255 char), + comment_3 varchar2(255 char), + comment_4 varchar2(255 char), + contexts varchar2(255 char), + cp_portfolio_id varchar2(255 char), + cp_trader_id varchar2(255 char), + cp_client_unit_id varchar2(255 char), + creator_tag varchar2(255 char), + date_basis varchar2(255 char), + date_basis_2 varchar2(255 char), + deal_price varchar2(255 char), + param_1 varchar2(255 char), + expiry_date date, + fx_finance_rate binary_double, + mattr_4 number(19,4), + face_amount number(19,4), + sattr_1 varchar2(255 char), + fixing_calendar varchar2(255 char), + fixing_calendar_2 varchar2(255 char), + fixing_offset number(12, 0), + fixing_offset_2 number(12, 0), + fixing_period varchar2(255 char), + fixing_period_2 varchar2(255 char), + fixing_rate varchar2(255 char), + fixing_rate_2 varchar2(255 char), + flags number(12, 0), + flags_2 number(12, 0), + flags_3 number(12, 0), + floor binary_double, + frequency number(12, 0), + frequency_2 number(12, 0), + idx binary_double, + index_ratio binary_double, + issue_index binary_double, + issuer_unit_id varchar2(255 char), + kind_id varchar2(255 char), + leg_1_id varchar2(255 char), + leg_2_id varchar2(255 char), + sattr_4 varchar2(255 char), + logical_number number(12, 0), + sattr_3 varchar2(255 char), + margin1 binary_double, + margin2 binary_double, + param_11 varchar2(255 char), + method_value varchar2(255 char), + method_value_2 varchar2(255 char), + param_9 varchar2(255 char), + nominal_rate binary_double, + nominal_rate_2 binary_double, + opening_time date, + iattr_6 varchar2(255 char), + param_10 varchar2(255 char), + param_4 varchar2(255 char), + owner_id varchar2(255 char), + owner_unit_id varchar2(255 char), + package_id number(12, 0), + package_main_type_id varchar2(255 char), + package_type_id varchar2(255 char), + param_18 varchar2(255 char), + param_3 varchar2(255 char), + param_5 varchar2(255 char), + param_6 varchar2(255 char), + param_7 varchar2(255 char), + param_8 varchar2(255 char), + param_19 varchar2(255 char), + parent_number number(12, 0), + sattr_13 varchar2(255 char), + pool_factor varchar2(255 char), + premium_amount number(19,4), + premium_currency_id varchar2(255 char), + premium_date date, + premium_price binary_double, + premium_type varchar2(255 char), + quote_currency_yield binary_double, + rate_type varchar2(255 char), + rate_type_2 varchar2(255 char), + reference_number number(12, 0), + param_2 varchar2(255 char), + secondary_instrument_id varchar2(255 char), + security_identifier_type varchar2(255 char), + param_16 varchar2(255 char), + settlement_date date, + signature_date date, + spread binary_double, + spread_2 binary_double, + state_id_sort_order number(12, 0), + status varchar2(255 char), + broker_id varchar2(255 char), + top_portfolio_id varchar2(255 char), + trader_id varchar2(255 char), + trading_unit binary_double, + sattr_2 varchar2(255 char), + param_12 varchar2(255 char), + param_14 varchar2(255 char), + param_15 varchar2(255 char), + param_17 varchar2(255 char), + transaction_tag varchar2(255 char), + units binary_double +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_USERINFORMATION.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_USERINFORMATION.sql new file mode 100644 index 0000000..157a177 --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_USERINFORMATION.sql @@ -0,0 +1,15 @@ +create table ct_et_templates.w_ODS_TMS_USERINFORMATION ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + id varchar2(255 char), + name varchar2(255 char), + admin_center_id varchar2(255 char), + time_zone_id varchar2(255 char), + menu_name varchar2(255 char), + web_home_group_id varchar2(255 char), + flags varchar2(255 char), + access_role_id varchar2(255 char), + access_role_name varchar2(255 char), + active_from date, + active_to date +) diff --git a/airflow/ods/tms/TMS-tables/w_ODS_TMS_USER_ACCOUNT.sql b/airflow/ods/tms/TMS-tables/w_ODS_TMS_USER_ACCOUNT.sql new file mode 100644 index 0000000..3f0510d --- /dev/null +++ b/airflow/ods/tms/TMS-tables/w_ODS_TMS_USER_ACCOUNT.sql @@ -0,0 +1,10 @@ +create table ct_et_templates.w_ODS_TMS_USER_ACCOUNT ( + a_key number(38, 0), + a_workflow_history_key number(38, 0), + date_ date, + id varchar2(255 char), + name varchar2(255 char), + menu_name varchar2(255 char), + access_role_id varchar2(255 char), + access_role_name varchar2(255 char) +) diff --git a/airflow/ods/tms/config/TMS.yml b/airflow/ods/tms/config/TMS.yml new file mode 100644 index 0000000..ce76fbe --- /dev/null +++ b/airflow/ods/tms/config/TMS.yml @@ -0,0 +1,14 @@ +name: 'mars_poc' +version: '1.0.0' +environment: dev + +TMS-URL: "https://tmsxd104.ecbt1.tadnet.net:9443/report/" +TMS-user: schilli +TMS-password: chili03 + +destination: "mrds_inbox_dev:INBOX/TMS/" +dest-bucket: "mrds_inbox_dev" +dest-prefix: "INBOX/TMS/" +data-prefix: "ODS/TMS/" + + diff --git a/airflow/ods/tms/generate_tms_ods_dags.py b/airflow/ods/tms/generate_tms_ods_dags.py new file mode 100644 index 0000000..7dc72db --- /dev/null +++ b/airflow/ods/tms/generate_tms_ods_dags.py @@ -0,0 +1,539 @@ +from airflow import DAG +from airflow.decorators import dag +from airflow.operators.python import PythonOperator +from datetime import datetime, timedelta + +import logging +from pytz import timezone + +import mrds.utils.manage_files as mf +import mrds.utils.manage_runs as mr +import re +import subprocess +import sys +import traceback +import copy +import itertools +import pprint + +import yaml +import sys, os +from pathlib import Path + + +gScriptDir = Path(globals().get("__file__", "./_")).absolute().parent +gDataDir = str(gScriptDir) + '/TMS-layouts/' +gConfigDir = str(gScriptDir) + '/config' +gConnDir = '/opt/airflow/python/connectors/tms/' +gTableDir = str(gScriptDir) + '/TMS-tables/' + + +default_args = { + "owner": "ecb", + "depends_on_past": False, + "email_on_failure": False, + "email_on_retry": False, + "retries": 0, + "execution_timeout": timedelta(minutes=60), + "retry_delay": timedelta(minutes=5), +} + + +def create_dag(dag_name): + + ODS_TABLE =re.sub('^w_ODS_','',dag_name) + DATABASE_NAME = "ODS" + DAG_NAME = dag_name + MAPPING_NAME = dag_name + WF_NAME = dag_name + + print("create_dag(" + dag_name + ")") + + + with open(gDataDir + DAG_NAME + '.yml', 'r') as file: + report_desc = yaml.safe_load(file) + + + params_sequential = report_desc.get('parameters_sequential', False) + + params = {} + params_visible = {} + params_hidden = {} + + params_dict = report_desc.get('parameters', {}) + params_dict = params_dict or {} + + for p in params_dict.keys(): + params[p] = params_dict[p].get('value', None) + if not params_dict[p].get('hidden', False): + params_visible[p] = params_dict[p].get('value', None) + else: + params_hidden[p] = params_dict[p].get('value', None) + + + + with open(gConfigDir + '/TMS.yml', 'r') as f: + tms_config = yaml.safe_load(f) + + + tms_url = tms_config['TMS-URL'] + prefix = tms_config['dest-prefix'] + ODS_TABLE + '/' + ODS_TABLE + '/' + data_prefix = tms_config['data-prefix'] + ODS_TABLE + '/' + dest = tms_config['dest-bucket'] + ':' + prefix + + tms_user = tms_config['TMS-user'] + tms_pwd = tms_config['TMS-password'] + + + + + # Define function for the retrieval of the current run_id + def retrieve_run_id(**kwargs): + # Retrieve the run_id from the Airflow context + run_id = kwargs['run_id'] + # Store the run_id in XCom for future reference + ti = kwargs['ti'] + ti.xcom_push(key='run_id', value=run_id) + return run_id + + def check_dag_status(**kwargs): + for task_instance in kwargs['dag_run'].get_task_instances(): + if task_instance.state == 'failed' and task_instance.task_id != kwargs['task_instance'].task_id: + raise Exception("Task {} failed. Failing this DAG run".format(task_instance.task_id)) + + # Define function for the check of the status of the previous tasks + def determine_workflow_status(**kwargs): + # Check the status of previous tasks + task_statuses = kwargs['ti'].xcom_pull(task_ids=['retrieve_run_id', 'control_external_run_start', 'mapping_mopdb']) + # If any task failed, set workflow_status to 'N', otherwise 'Y' + workflow_status = 'N' if any(status != 'success' for status in task_statuses) else 'Y' + return workflow_status + + + + def enum_param_combinations_recursive(params, keys): + + + k = None + result = [] + + while(keys): + k = keys.pop(0) + v = params[k] + if v or v == "": + break + + if not k: + return [] + + if not(isinstance(v, list)): + v = [v] + + derived_columns = [] + for c in list(params_dict[k]): + if re.match(r'column\(.*\)', c): + vtmp = params_dict[k][c] + if not(isinstance(vtmp, list)): + vtmp = [vtmp] + derived_columns.append((c, vtmp)) + + print("derived columns = " + str(derived_columns)) + + if not keys: + for i, value in enumerate(v): + result.append([(k, value)]) + for t in derived_columns: + result[len(result)-1].append((t[0], t[1][i])) + + return result + + + + combinations = enum_param_combinations_recursive(params, keys) + + for l in combinations: + + for i, vtmp in enumerate(v): + tmp = copy.deepcopy(l) + tmp.append((k, vtmp)) + for t in derived_columns: + tmp.append((t[0], t[1][i])) + result.append(tmp) + + return result + + + def enum_param_combinations_sequential(params): + + iterator = itertools.count(0, 1) + keys = list(params) + + result = [] + + for i in iterator: + + param_list = [] + while(keys): + k = keys.pop(0) + v = params[k] + if instance(v, list): + param_list.append((k, v[i])) + else: + param_list.append((k, v)) + + result.append(param_list) + + return result + + + + def enum_param_combinations(params, sequential = False): + + if sequential: + return enum_param_combinations_sequential(params) + else: + return enum_param_combinations_recursive(params, list(params)) + + + + def select(table, expression, condition = "1 = 1"): + + if table.upper() not in (ODS_TABLE.upper(), 'DUAL', 'ct_mrds.a_workflow_history'.upper()): + raise Exception("Not allowed to select from %s" % table) + + res = mr.select_ods_tab(table, expression, condition) + + return res[0] + + + + # Input is a list of tuples containing (parameter name, value) + # Returns evaluated parameters in case functional expressions are used + + def eval_params(params): + + params_new = [] + for t in params: + if re.match(r"\s*select\(.*\)", str(t[1])): + print("globals = " + str(globals())) + print("locals = " + str(locals())) + print("expression = " + t[1]) + params_new.append((t[0], eval(t[1], globals() | {'select': select}, locals()))) + else: + params_new.append(t) + + return params_new + + + def eval_param(v): + + if re.match(r"\s*select\(.*\)", str(v)): + v = eval(v, {'select': select}, {}) + elif re.match(r"\s*eval\(.*\)*$", str(v)): + expr = re.match(r"\s*eval\(.*\)*$", str(v)).group(0) + v = eval(expr) + return v + + + + def execute_report_task(**context): + + params = context['params'] + + all_params = params | params_hidden + + + # Calculation of all combinations of parameters + # By default, this is done recursively, exploring all vertices + param_combinations = enum_param_combinations(params | params_hidden, sequential = False) + + # The evaluation of parameters requires a few iterations + + # First iteration is for evaluating functional expressions + tmp = [] + for l in param_combinations: + d = dict(l) + for p in list(d): + d[p] = eval_param(d[p]) + tmp.append(list(d.items())) + + param_combinations = tmp + + + # Second iteration is for replacing virtual symbolic parameters + tmp = [] + for l in param_combinations: + d = dict(l) + + for p in list(d): + if p in params_dict and params_dict[p].get('replace_parameter', None): + d[params_dict[p]['replace_parameter']] = d[params_dict[p]['replace_parameter']].replace(p, d[p]) + tmp.append(list(d.items())) + + param_combinations = tmp + + + # Next iteration selects actual parameters to be used + tmp = [] + for l in param_combinations: + tmp_list = [] + for t in l: + if t[0] not in params_dict or not params_dict[t[0]].get('virtual', False): + tmp_list.append((t[0], t[1])) + tmp.append(tmp_list) + + param_combinations = tmp + + + run_id = context['dag_run'].run_id + + # Finally, we execute the report for each evaluated combination + if param_combinations: + i = 1 + for c in param_combinations: + execute_report(c, run_id) + i = i + 1 + else: + execute_report([], run_id) + + + + + def execute_report(param_list, airflow_run_id): + + print("execute_report " + str(param_list)) + + logger = logging.getLogger("airflow.task") + logger.setLevel(logging.DEBUG) + + ts = '{:%Y%m%d_%H%M%S}'.format(datetime.now(timezone('Europe/Berlin'))) + wf_key = mr.init_workflow('ODS', WF_NAME, airflow_run_id) + + data_set = wf_key + + file_name = ODS_TABLE + '.' + str(data_set) + '.' + ts + '.csv' + + + command = ['python', gConnDir + 'TMSDBT.py', 'retrieve', '--name', ODS_TABLE, + '--url', tms_url, + '-U', tms_user, '--password', tms_pwd, + '--layoutfile ' + gDataDir + dag_name + '.fkr', + '-f', 'scsv', '--dataset', str(wf_key), + '-d', dest + file_name] + + for t in param_list: + if t[1] or t[1] == "": + m = re.match(r'column\((.*)\)$', t[0]) + if m: + command.extend(['-c', '"' + m.group(1) + '"' + '="%s"' % str(t[1]).rstrip()]) + elif t[1] or t[1] == "": + command.extend(['-p', t[0] + '="%s"' % str(t[1]).rstrip()]) + logger.debug(f"workflow_property: {wf_key}, {'ODS'}, {t[0]}, {str(t[1]).rstrip()}") + mr.set_workflow_property(wf_key, 'ODS', t[0], str(t[1]).rstrip()) + + # Useful printout of command to execute manually + # p = Path('/tmp/command-' + str(wf_key)) + # p.write_bytes(' '.join(command).encode()) + + command = [' '.join(command)] + + logger.debug(str(command)) + try: + logger.debug(f'running TMSDB with arguments: {command}') + res = subprocess.run(command, capture_output=True, check=False, shell=True) + logger.debug('subprocess stdout =' + res.stdout.decode()) + logger.debug('subprocess stderr =' + res.stderr.decode()) + + + if res.returncode == None: + raise Exception('Data Retrieval failure') + elif res.returncode == 1: + logger.debug('No data returned') + raise + elif res.returncode != 0: + raise Exception('Other failure') + elif res.returncode == 0: + # Data was returned + # Check if configuration record exists already') + res = mf.execute_query("select * from CT_MRDS.A_SOURCE_FILE_CONFIG where a_source_key = 'TMS' and table_id = '%s'" % ODS_TABLE) + logger.debug(f'result of queryng source_file_config {res}') + if not res: + logger.debug(f'creating table {ODS_TABLE}') + mf.add_source_file_config('TMS', 'INPUT', DAG_NAME.strip(), ODS_TABLE, '.*\.csv', ODS_TABLE, 'CT_ET_TEMPLATES.' + ODS_TABLE) + command = ['python', gConnDir + 'TMSDBT.py', 'create-oracle-table', '--name', ODS_TABLE, + '--url', tms_url, + '-U', tms_user, '--password', tms_pwd, + '--layoutfile ' + gDataDir + dag_name + '.fkr', + '-d', gTableDir + ODS_TABLE + '.sql'] + + for t in param_list: + m = re.match(r'column\((.*)\)$', t[0]) + if m: + command.extend(['-c', '"' + m.group(1) + '"' + '="%s"' % str(t[1]).rstrip()]) + elif t[1] or t[1] == "": + command.extend(['-p', t[0] + '="%s"' % t[1]]) + + # print(str(command)) + command = [' '.join(command)] + logger.debug(f'running TMSDB with arguments: {command}') + + res = subprocess.run(command, capture_output=True, check=True, shell=True) + logger.debug('subprocess stdout =' + res.stdout.decode()) + logger.debug('subprocess stderr =' + res.stderr.decode()) + + + sql = Path(gTableDir + ODS_TABLE + '.sql').read_text() + # print('SQL - '+ sql) + res = mf.execute(sql) + res = mf.add_column_date_format('CT_ET_TEMPLATES.' + ODS_TABLE, 'DEFAULT', 'DD/MM/YYYY HH24:MI:SS') + res = mf.create_external_table(ODS_TABLE, 'CT_ET_TEMPLATES.' + ODS_TABLE, data_prefix) + + + mf.process_source_file(prefix, file_name) + + mr.finalise_workflow(wf_key, 'Y') + + except BaseException as ex: + ex_type, ex_value, ex_traceback = sys.exc_info() + + trace_back = traceback.extract_tb(ex_traceback) + + stack_trace = list() + + for trace in trace_back: + stack_trace.append("File : %s , Line : %d, Func.Name : %s, Message : %s" % (trace[0], trace[1], trace[2], trace[3])) + + logger.error("Exception type : %s " % ex_type.__name__) + logger.error("Exception message : %s" %ex_value) + logger.error("Stack trace : %s" %stack_trace) + + mr.finalise_workflow(wf_key, 'N') + raise + + + + + @dag( + DAG_NAME, + default_args = default_args, + description = DAG_NAME, + schedule_interval = None, + params = params_visible, + start_date = datetime(2025, 1, 1), + catchup = False, + tags = [DAG_NAME], + ) + def run_dag(): + + PythonOperator( + task_id = "retrieve_report", + execution_timeout = timedelta(minutes = 30), + python_callable = execute_report_task + ) + + new_dag = run_dag() + globals()[DAG_NAME] = new_dag + + + +create_dag('w_ODS_TMS_RAR_PORTFOLIOACCESS') +create_dag('w_ODS_TMS_CLIENTACCOUNT') +create_dag('w_ODS_TMS_BRANCH') +create_dag('w_ODS_TMS_RAR_ECBINSTRUMENTS') +create_dag('w_ODS_TMS_CALENDAR') +create_dag('w_ODS_TMS_CLIENTINSTRUCTION') +create_dag('w_ODS_TMS_HISTORY_LOG') +create_dag('w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW') +create_dag('w_ODS_TMS_PARAMETER') +create_dag('w_ODS_TMS_RAR_FRM_HOLIDAYS') +create_dag('w_ODS_TMS_RAR_LIMITACCESS') +create_dag('w_ODS_TMS_RAR_PORTFOLIOTREE') +create_dag('w_ODS_TMS_RAR_RARCOLLATERALINVENTORY') +create_dag('w_ODS_TMS_RULES') +create_dag('w_ODS_TMS_SDM_ENTITY_STATE') +create_dag('w_ODS_TMS_USER_ACCOUNT') + + + +""" +create_dag('w_ODS_TMS_ACMCURRENCYFLOW') +create_dag('w_ODS_TMS_ACMENTRYSTATELEDGERGROUP') +create_dag('w_ODS_TMS_ACTIVITY') +create_dag('w_ODS_TMS_ACTIVITY_LOG') +create_dag('w_ODS_TMS_ACTIVITYLOGDUE') +create_dag('w_ODS_TMS_BALANCE') +create_dag('w_ODS_TMS_BLACKOUT_LOG') +create_dag('w_ODS_TMS_BRANCH') +create_dag('w_ODS_TMS_CALENDAR') +create_dag('w_ODS_TMS_CASHFLOW') +create_dag('w_ODS_TMS_CLIENTACCOUNT') +create_dag('w_ODS_TMS_CLIENTINSTRUCTION') +create_dag('w_ODS_TMS_CPBLOCKEDISSUERS') +create_dag('w_ODS_TMS_CUSTODYBALANCE') +create_dag('w_ODS_TMS_ECBINSTRUMENTBONDCASHFLOW') +create_dag('w_ODS_TMS_EFFECTIVEROLEPROFILE') +create_dag('w_ODS_TMS_FINMESSAGELOG') +create_dag('w_ODS_TMS_HISTORY_LOG') +create_dag('w_ODS_TMS_INSTRUMENT_BOND_SCHEDULE') +create_dag('w_ODS_TMS_INSTRUMENT_REPORT') +create_dag('w_ODS_TMS_INSTRUMENTBONDCASHFLOW') +create_dag('w_ODS_TMS_MOPDBPRICES') +create_dag('w_ODS_TMS_PARAMETER') +create_dag('w_ODS_TMS_POS_PERF_ACTIVITY_STATUS') +create_dag('w_ODS_TMS_PRICE_RATE_REPORT') +create_dag('w_ODS_TMS_PROPERTY') +create_dag('w_ODS_TMS_RAR_CLIENT') +create_dag('w_ODS_TMS_RAR_CLIENTGROUPMAP') +create_dag('w_ODS_TMS_RAR_ECBINSTRUMENTS') +create_dag('w_ODS_TMS_RAR_FRM_HOLIDAYS') +create_dag('w_ODS_TMS_RAR_FUTURE_BONDS') +create_dag('w_ODS_TMS_RAR_INSTRUMENT_REPORT') +create_dag('w_ODS_TMS_RAR_LIMIT') +create_dag('w_ODS_TMS_RAR_LIMITACCESS') +create_dag('w_ODS_TMS_RAR_LIMITLOG') +create_dag('w_ODS_TMS_RAR_MARKETINFO') +create_dag('w_ODS_TMS_RAR_PORTFOLIO') +create_dag('w_ODS_TMS_RAR_PORTFOLIOACCESS') +create_dag('w_ODS_TMS_RAR_PORTFOLIOTREE') +create_dag('w_ODS_TMS_RAR_PRICES') +create_dag('w_ODS_TMS_RAR_RARCOLLATERALINVENTORY') +create_dag('w_ODS_TMS_RAR_RARTRANSACTIONPROPERTY') +create_dag('w_ODS_TMS_RAR_RARTRANSLIMITCOND') +create_dag('w_ODS_TMS_RAR_SUBLIMIT') +create_dag('w_ODS_TMS_RAR_TRANSACTIONSLOG') +create_dag('w_ODS_TMS_RAR_UMICREDITCLIENT') +create_dag('w_ODS_TMS_RAR_UMISECURITYCODE') +create_dag('w_ODS_TMS_RAR_UMISECURITYINFO') +create_dag('w_ODS_TMS_RECONCILIATION') +create_dag('w_ODS_TMS_ROLEPORTFOLIOPROFILE') +create_dag('w_ODS_TMS_RULES') +create_dag('w_ODS_TMS_SDM_ENTITY_STATE') +create_dag('w_ODS_TMS_SECURITYPOSITION') +create_dag('w_ODS_TMS_SETTLEMENTCASHFLOW') +create_dag('w_ODS_TMS_SETTLEMENTLOG') +create_dag('w_ODS_TMS_THIRD_PRICE_RATE_REPORT_CHECK') +create_dag('w_ODS_TMS_TRANSACTION') +create_dag('w_ODS_TMS_USER_ACCOUNT') +create_dag('w_ODS_TMS_USERINFORMATION') + +create_dag('w_ODS_TMS_EFFECTIVEPERMISSIONS') + +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_0') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_4') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_5') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_32') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_36') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_96') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_100') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_128') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_132') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_256') +create_dag('w_ODS_TMS_ECBPOSITIONDATA_GROUPING_1093') + + +create_dag('w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_4') +create_dag('w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_5') +create_dag('w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_68') +create_dag('w_ODS_TMS_ECBPERFORMANCEDATA_GROUPING_69') + +""" \ No newline at end of file diff --git a/airflow/ods/top/aggregated_allotment/config/m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE.yaml b/airflow/ods/top/aggregated_allotment/config/m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE.yaml new file mode 100644 index 0000000..a7f8df9 --- /dev/null +++ b/airflow/ods/top/aggregated_allotment/config/m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE.yaml @@ -0,0 +1,184 @@ +# Global configurations +tmpdir: /tmp +inbox_prefix: INBOX/TOP/AggregatedAllotmentNCB +archive_prefix: ARCHIVE/TOP/AggregatedAllotmentNCB +workflow_name: w_ODS_TOP_AGGREGATED_ALLOTMENT +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/top/top_common/TOPV2.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_TOP_AGGREGATED_ALLOTMENT_PARSE + ods_prefix: INBOX/TOP/AggregatedAllotmentNCB/TOP_AGGREGATED_ALLOTMENT + output_table: TOP_AGGREGATED_ALLOTMENT + namespaces: + topV2: 'http://top.ecb.int/v2' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:RefNo' + column_header: 'REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:TimeStampSender' + column_header: 'TIMESTAMP_SENDER' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ID' + column_header: 'ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:Checksum' + column_header: 'CHECKSUM' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:Revision' + column_header: 'REVISION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:TransactionType' + column_header: 'TRANSACTION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:OperationCurrency' + column_header: 'OPERATION_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:OperationName' + column_header: 'OPERATION_NAME' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:OperationCategory' + column_header: 'OPERATION_CATEGORY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:OperationType' + column_header: 'OPERATION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:Procedure' + column_header: 'PROCEDURE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:AuctionType' + column_header: 'AUCTION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:AllotmentMethod' + column_header: 'ALLOTMENT_METHOD' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:Quotation' + column_header: 'QUOTATION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:AllotmentDate' + column_header: 'ALLOTMENT_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:ValueDate' + column_header: 'VALUE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:MaturityDate' + column_header: 'MATURITY_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:DurationOfOperation' + column_header: 'DURATION_OF_OPERATION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:ReferenceEntity' + column_header: 'REFERENCE_ENTITY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:SpotRate' + column_header: 'SPOT_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:RepaidRefNo' + column_header: 'REPAID_REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:BusinessValueType' + column_header: 'BUSINESS_VALUE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:BidCurrency' + column_header: 'BID_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:OtherCurrency' + column_header: 'OTHER_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:TotalNumberBidders' + column_header: 'TOTAL_NUMBER_BIDDERS' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:TotalBidAmount' + column_header: 'TOTAL_BID_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:TotalAllottedAmount' + column_header: 'TOTAL_ALLOTTED_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:NcbSenderReference' + column_header: 'NCB_SENDER_REFERENCE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:NcbNumberBidders' + column_header: 'NCB_NUMBER_BIDDERS' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:NcbBidAmount' + column_header: 'NCB_BID_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:NcbAllottedAmount' + column_header: 'NCB_ALLOTTED_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bidders/topV2:Bidder/topV2:BidderId' + column_header: 'BIDDER_ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bidders/topV2:Bidder/topV2:BidderName' + column_header: 'BIDDER_NAME' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bidders/topV2:Bidder/topV2:BidderBidAmount' + column_header: 'BIDDER_BID_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bidders/topV2:Bidder/topV2:BidderAllottedAmount' + column_header: 'BIDDER_ALLOTTED_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bidders/topV2:Bidder/topV2:BidderAllottedBusinessValue' + column_header: 'BIDDER_ALLOTTED_BUSINESS_VALUE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bids/topV2:Bid/topV2:BidAmount' + column_header: 'BID_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bids/topV2:Bid/topV2:BidBusinessValue' + column_header: 'BID_BUSINESS_VALUE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bids/topV2:Bid/topV2:BidAllottedAmount' + column_header: 'BID_ALLOTTED_AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bids/topV2:Bid/topV2:BidAllottedBusinessValue' + column_header: 'BID_ALLOTTED_BUSINESS_VALUE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotment/topV2:NCBs/topV2:NCB/topV2:Bids/topV2:Bid/topV2:BidAllottedPercentage' + column_header: 'BID_ALLOTTED_PERCENTAGE' + is_key: 'N' \ No newline at end of file diff --git a/airflow/ods/top/allotment/config/.gitkeep b/airflow/ods/top/allotment/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/top/allotment/config/m_ODS_TOP_ALLOTMENT_PARSE.yaml b/airflow/ods/top/allotment/config/m_ODS_TOP_ALLOTMENT_PARSE.yaml new file mode 100644 index 0000000..496db10 --- /dev/null +++ b/airflow/ods/top/allotment/config/m_ODS_TOP_ALLOTMENT_PARSE.yaml @@ -0,0 +1,139 @@ +tmpdir: /tmp +inbox_prefix: INBOX/TOP/AllotmentPublicationC2D +archive_prefix:: ARCHIVE/TOP/AllotmentPublicationC2D +workflow_name: w_ODS_TOP_ALLOTMENT +validation_schema_path: 'opt/airflow/src/airflow/dags/ods/top/top_common/TOPV2.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_TOP_ALLOTMENT_PARSE + ods_prefix: INBOX/TOP/AllotmentPublicationC2D/TOP_ALLOTMENT + output_table: TOP_ALLOTMENT + namespaces: + topV2: http://top.ecb.int/v2 + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:RefNo' + column_header: 'REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:SenderReference' + column_header: 'SENDER_REF' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:UserReference' + column_header: 'USER_REF' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:TimeStamp' + column_header: 'TIMESTAMP' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:FourEyesVersion' + column_header: 'VERSION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:BidAmount' + column_header: 'AMOUNT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:NumberBidders' + column_header: 'NUMBER_BIDDERS' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:CutoffLevel' + column_header: 'CUTOFF_LEVEL' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:AllottedAmount' + column_header: 'AMNT_ALLOTED' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:MinAllotment' + column_header: 'MIN_ALLOTMENT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:PercentageAllotted' + column_header: 'PERC_ALLOTED' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:BusinessValueType' + column_header: 'MARGINAL_RATE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:MarginalBusinessValue' + column_header: 'MARGINAL_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:MinBusinessValue' + column_header: 'MIN_RATE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:MinBusinessValue' + column_header: 'MIN_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:BusinessValueType' + column_header: 'MAX_RATE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:MaxBusinessValue' + column_header: 'MAX_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:BusinessValueType' + column_header: 'AVG_RATE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:BusinessValueType' + column_header: 'AVG_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:SpotRate' + column_header: 'SPOT_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:FreeText' + column_header: 'FREE_TEXT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:Checksum' + column_header: 'CHECKSUM' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:Isin' + column_header: 'ISIN' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:ReferenceEntity' + column_header: 'REF_ENTITY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:ReferenceEntityValue' + column_header: 'REF_ENTITY_VALUE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:SpotRateWithMargin' + column_header: 'SPOT_RATE_WITH_MARGIN' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:TimeStampSender' + column_header: 'TIMESTAMP_SENDER' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ID' + column_header: 'ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:RepaidRefNo' + column_header: 'REPAID_REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AllotmentMessage/topV2:Revision' + column_header: 'REVISION' + is_key: 'N' \ No newline at end of file diff --git a/airflow/ods/top/allotment_modifications/config/.gitkeep b/airflow/ods/top/allotment_modifications/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/top/allotment_modifications/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_PARSE.yaml b/airflow/ods/top/allotment_modifications/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_PARSE.yaml new file mode 100644 index 0000000..e0937b6 --- /dev/null +++ b/airflow/ods/top/allotment_modifications/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_PARSE.yaml @@ -0,0 +1,78 @@ +tmpdir: /tmp +inbox_prefix: INBOX/TOP/TenderAllotmentModifications +archive_prefix: ARCHIVE/TOP/TenderAllotmentModifications +workflow_name: w_ODS_TOP_ALLOTMENT_MODIFICATIONS +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/top/top_common/TOPV2.xsd' +file_type: xml + +# List of tasks +tasks: +- task_name: m_ODS_TOP_ALLOTMENT_MODIFICATION_HEADER_PARSE + ods_prefix: INBOX/TOP/TenderAllotmentModifications/TOP_ALLOTMENT_MODIFICATIONS_HEADER + output_table: TOP_ALLOTMENT_MODIFICATIONS_HEADER + namespaces: + topV2: 'http://top.ecb.int/v2' + output_columns: + - type: 'xpath_element_id' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:RefNo' + column_header: 'REF_NO' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:OperationCurrency' + column_header: 'OPERATION_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TimeStamp' + column_header: 'TIMESTAMP_' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:Revision' + column_header: 'REVISION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:TimeStampSender' + column_header: 'TIMESTAMP_SENDER' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ID' + column_header: 'ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:Checksum' + column_header: 'CHECKSUM' + is_key: 'N' +- task_name: m_ODS_TOP_ALLOTMENT_MODIFICATION_ITEM_PARSE + ods_prefix: INBOX/TOP/TenderAllotmentModifications/TOP_ALLOTMENT_MODIFICATIONS_ITEM + output_table: TOP_ALLOTMENT_MODIFICATIONS_ITEM + namespaces: + topV2: 'http://top.ecb.int/v2' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath_element_id' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification' + column_header: 'A_ALL_MOD_FK' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:SenderReferenc' + column_header: 'SENDER_REFERENCE' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:BidderId' + column_header: 'BIDDER_ID' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:FlowType' + column_header: 'FLOW_TYPE' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:Amount' + column_header: 'AMOUNT_ITEM' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:ValueDate' + column_header: 'VALUE_DATE' + - type: 'xpath' + value: '/topV2:Message/topV2:ReportAggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:FreeText' + column_header: 'FREE_TEXT' \ No newline at end of file diff --git a/airflow/ods/top/allotment_modifications_aggregated/.gitkeep b/airflow/ods/top/allotment_modifications_aggregated/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/top/allotment_modifications_aggregated/config/.gitkeep b/airflow/ods/top/allotment_modifications_aggregated/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/top/allotment_modifications_aggregated/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_AGGREGATED_PARSE.yaml b/airflow/ods/top/allotment_modifications_aggregated/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_AGGREGATED_PARSE.yaml new file mode 100644 index 0000000..8a49240 --- /dev/null +++ b/airflow/ods/top/allotment_modifications_aggregated/config/m_ODS_TOP_ALLOTMENT_MODIFICATION_AGGREGATED_PARSE.yaml @@ -0,0 +1,66 @@ +tmpdir: /tmp +inbox_prefix: INBOX/TOP/AggregatedAllotmentModification +archive_prefix: ARCHIVE/TOP/AggregatedAllotmentModification +workflow_name: w_ODS_TOP_ALLOTMENT_MODIFICATIONS_AGGREGATED +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/top/top_common/TOPV2.xsd' +file_type: xml + +# List of tasks +tasks: +- task_name: m_ODS_TOP_ALLOTMENT_MODIFICATION_AGGREGATED_PARSE + ods_prefix: INBOX/TOP/AggregatedAllotmentModification/TOP_ALLOTMENT_MODIFICATION_AGGREGATED + output_table: TOP_ALLOTMENT_MODIFICATION_AGGREGATED + namespaces: + topV2: 'http://top.ecb.int/v2' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/topV2:Message/topV2:Checksum' + column_header: 'CHECKSUM' + - type: 'xpath' + value: '/topV2:Message/topV2:ID' + column_header: 'ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:OperationCurrency' + column_header: 'OPERATION_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:RefNo' + column_header: 'REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:Revision' + column_header: 'REVISION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:TimeStampSender' + column_header: 'TIMESTAMP_SENDER' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:Amount' + column_header: 'AMOUNT_ITEM' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:BidderId' + column_header: 'BIDDER_ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:FreeText' + column_header: 'FREE_TEXT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:SenderReference' + column_header: 'SENDER_REFERENCE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:ValueDate' + column_header: 'VALUE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AggregatedAllotmentModification/topV2:TransactionList/topV2:TransactionElement/topV2:NewBidderId' + column_header: 'NEW_BIDDER' + is_key: 'N' diff --git a/airflow/ods/top/announcement/config/m_ODS_TOP_ACCOUNCEMENT_PARSE.yaml b/airflow/ods/top/announcement/config/m_ODS_TOP_ACCOUNCEMENT_PARSE.yaml new file mode 100644 index 0000000..76c6e42 --- /dev/null +++ b/airflow/ods/top/announcement/config/m_ODS_TOP_ACCOUNCEMENT_PARSE.yaml @@ -0,0 +1,195 @@ +tmpdir: /tmp +inbox_prefix: INBOX/TOP/AnnouncementPublicationC2D +archive_prefix: ARCHIVE/TOP/AnnouncementPublicationC2D +workflow_name: w_ODS_TOP_ANNOUNCEMENT +validation_schema_path: '/opt/airflow/src/airflow/dags/ods/top/top_common/TOPV2.xsd' +file_type: xml + +# List of tasks +tasks: + - task_name: m_ODS_TOP_ACCOUNCEMENT_PARSE + ods_prefix: INBOX/TOP/AnnouncementPublicationC2D/TOP_ANNOUNCEMENT + output_table: TOP_ANNOUNCEMENT + namespaces: + topV2: http://top.ecb.int/v2 + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:RefNo' + column_header: 'REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:SenderReference' + column_header: 'SENDER_REF' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:UserReference' + column_header: 'USER_REF' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:TimeStamp' + column_header: 'TIMESTAMP' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:FourEyesVersion' + column_header: 'VERSION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OperationType' + column_header: 'OPERATION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:Procedure' + column_header: 'PROCEDURE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:TransactionType' + column_header: 'TRANSACTION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OperationName' + column_header: 'OPERATION_NAME' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:AuctionType' + column_header: 'AUCTION_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:AllotmentMethod' + column_header: 'ALL_METHOD' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:Quotation' + column_header: 'QUOTATION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:BusinessValueType' + column_header: 'BUSINESS_VALUE_TYPE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:BusinessValue' + column_header: 'BUSINESS_VALUE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:AllotmentDate' + column_header: 'ALLOTMENT_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:ValueDate' + column_header: 'VALUE_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MaturityDate' + column_header: 'MATURITY_DATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:DurationOfOperation' + column_header: 'DURATION_OF_OPERATION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:SpotRate' + column_header: 'SPOT_RATE' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OtherCurrency' + column_header: 'OTHER_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OperationCurrencyLeg' + column_header: 'OPERATION_CURRENCY_LEG' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MinBidAmount' + column_header: 'MIN_BID_AMT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MaxBidAmount' + column_header: 'MAX_BID_AMT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:IntendedVolume' + column_header: 'INTENDED_VOLUME' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MinAllotment' + column_header: 'MIN_ALLOTMENT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MinAllotmentRatio' + column_header: 'MIN_ALLOTMENT_RATIO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:DeadlineCP' + column_header: 'DEADLINE_CP' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:DeadlineNCB' + column_header: 'DEADLINE_NCB' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:PublicAnnouncement' + column_header: 'PUBLIC_ANNOUNCEMENT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:FreeText' + column_header: 'FREE_TEXT' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:Checksum' + column_header: 'CHECKSUM' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OperationCurrency' + column_header: 'OPER_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:Isin' + column_header: 'ISIN' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:ReferenceEntity' + column_header: 'REF_ENTITY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:SpotRateWithMargin' + column_header: 'SPOT_RATE_WITH_MARGIN' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:MaxBidsPerCounterparty' + column_header: 'MAX_BIDS_COUNTER_PARTY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:ExactDenomination' + column_header: 'EXACT_DENOMINATION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:BidCurrency' + column_header: 'BID_CURRENCY' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:TimeStampSender' + column_header: 'TIMESTAMP_SENDER' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:ID' + column_header: 'ID' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:RepaidRefNo' + column_header: 'REPAID_REF_NO' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:Revision' + column_header: 'REVISION' + is_key: 'N' + - type: 'xpath' + value: '/topV2:Message/topV2:AnnouncementMessage/topV2:OperationCategory' + column_header: 'OPERATION_CATEGORY' + is_key: 'N' + + + + \ No newline at end of file diff --git a/airflow/ods/top/top_common/.gitkeep b/airflow/ods/top/top_common/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/ods/top/top_common/TOPV2.xsd b/airflow/ods/top/top_common/TOPV2.xsd new file mode 100644 index 0000000..7b41e49 --- /dev/null +++ b/airflow/ods/top/top_common/TOPV2.xsd @@ -0,0 +1,967 @@ + + + + + Top Schema 2.4.0 (2012-07-19) + AnnouncementMessage: RepaidRefNo added. + AllotmentMessage: RepaidRefNo added. + BidMessage: RepaidTenderNote added. + WebMessage: Quotation, RepaidRefNo added. + FullBidMessage: BusinessValue added. + FullBidArrayCompiled: RepaidRefNo, BusinessValue added. + BidsAndAllotmentPerNCBMessage: RepaidRefNo added. + ReportFullBidArrayCompiled: BusinessValue added. FixedRateHeader removed. + ReportTenderHistory: RepaidRefNo added. + ReportTenderResultPlusBids: RepaidRefNo added. + ReportBidsAllotmentNCB: RepaidRefNo added. + ReportLargeBidders: RepaidRefNo added. + ReportFullBidArrayCompiled: RepaidRefNo added. + Top Schema 2.3.0 (2011-04-21) + Changes for Foreign Currency Tenders and Debt Certificates (Announcement: Quotation (was DCQuotation), OperationCurrencyLeg (was EuroFixVar), ExactDenomination and BidCurrency; Allotment: SpotRateWithMargin; BidsAndAllotmentPerNCBMessage: OperationCurrency, Spot Rate, Spot Rate with Margin; FullBidArrayCompiled: OperationCurrency; ReportTenderHistory: Quotation, OperationCurrency, OtherCurrency, SpotRate, SpotRateWithMargin; ReportTenderResultPlusBids: OperationCurrency; ReportBidsAllotmentNCB: OperationCurrency; ReportLargeBidders: OperationCurrency; ReportFullBidArrayCompiled: OperationCurrency. Optional CurrencyInfo element added to AnnouncementMessage, BidMessage, AllotmentMessage, FullBidMessage, AllotmentMatrix, EuroAreaCompiledBids, EuroAreaLargeBidders, EuroAreaConsolidatedLargeBidders, BidsAndAllotmentPerNCBMessage, FullBidArrayCompiled) + FullBidArrayCompiled: ID added. IsAscending, FullBidderList optional. + ReplyMessage: reply message for webservice requests (replaces AcknowledgementMessage) + AcceptMessage: ID, DataChecksum, Revision, TimeStampSender added. + RecallMessage: ID, TimeStampSender added. + AnnouncementMessage: ID, TimeStampSender added. + AllotmentMessage: ID, TimeStampSender added. + BidMessage: ID added. + FullBidMessage: ID added. + BidsAllotmentNCB renamed to BidsAndAllotmentPerNCBMessage: ID added. + AllotmentMatrix: PrevRefNo added. + WebMessage: ID added. + Top Schema 2.1.0 (2009-07-01) + Changes for Outright Transaction and Foreign Currency Tenders (Operation Currency, ISIN, Reference Entity, Reference Entity Value, Spot Rate with Margin and Max Bids per Counterparty). + Top Schema 2.0.1 (2008-03-04) + Business Value element list added to FullBidArrayCompiled and ReportFullBidArrayCompiled types. + AllotmentModificationMessage: new message added. + AggregatedAllotmentModification: new message added. + ReportAggregatedAllotmentModification: new message added. + Top Schema 3.3.0: Changes in reply messages + Top Schema 3.3.1: Changes in AllotmentModificationMessage: Added minOccurs + Top Schema 3.4.0: Changes in AggregatedAllotmentModificationMessage: Added Revision + Top Schema 3.4.0: Changes in AnnouncementMessage and AllotmentMessage: Added Revision + Top Schema 3.4.1: Added V2 (2018-04-18) + Top Schema 4.0.0 (2019-06-18) + Top Schema 4.0.2 (2019-08-20): Added minOccurs in ExportAndExdiMessageType, added FixedBusinessValue in BidsAndAllotmentKeyValues + Top Schema 4.0.3 (2019-09-10): Changes in WebMessage + Top Schema 4.0.6 (2019-12-10): Changes of ReferenceEntity and ReferenceEntityValue + Top Schema 4.2.0 (2022-04-05): + Removed elements: EuroAreaConsolidatedLargeBidders, ConsolidatedLargeBiddersList, WebMessage, SegregatedOperationName in AggregatedAllotment + Added element OperationCategory in AnnouncementMessage, AllotmentMatrix, BidsAndAllotmentPerNCBMessage, AggregatedAllotment + Changed length restriction of ISIN from 30 to 12 + Changed type of ReferenceEntityValue in AllotmentMessage from ReferenceEntity to BusinessValue + Changed type of Amount in AggregatedAllotmentModificationList and ReportAggregatedAllotmentModificationList from Amount to TransactionAmount + + Top Schema 4.2.15 (2022-07-22): Raised length of LargeBidderName from 50 to 255 + Top Schema 4.3.5 (2023-08-30): + Changes in AllotmentMatrix: + Renamed PercentageAllotted to PercentageOfIntendedAllotment, AllottedAmount to IntendedAllotmentVolume, + Added PercentageOfActualAllotment, ActualAllotmentVolume + + + + + + Root Element + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Message Element + + + + + + + + + + + + Message Element + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Message Element + + + + + + + + + + + + Message Element + + + + + + + + + + + + + + + + + + + + + + + Message Element + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Message Element for created XML report Aggregated Allotment Modification (on TOP report creation/save) + + + + + + + + + + + + + Message Element for sending Aggregated Allotment Modification Report to EXDI (on TOP report creation/save) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airflow/tmp/.gitkeep b/airflow/tmp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow/tmp/oci_principal_authentication_debug.py b/airflow/tmp/oci_principal_authentication_debug.py new file mode 100644 index 0000000..3f010a3 --- /dev/null +++ b/airflow/tmp/oci_principal_authentication_debug.py @@ -0,0 +1,73 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from datetime import datetime +import os +import oci +import base64 +from oci.auth.signers import InstancePrincipalsSecurityTokenSigner, get_resource_principals_signer + +def debug_oci_authentication_method(**context): + resource_principal_version = os.environ.get('OCI_RESOURCE_PRINCIPAL_VERSION') + + if resource_principal_version: + print("RESOURCE PRINCIPAL - Container") + signer = get_resource_principals_signer() + else: + print("INSTANCE PRINCIPAL - VM/Compute") + signer = InstancePrincipalsSecurityTokenSigner() + + env = os.getenv("MRDS_ENV") + + if env == "dev": + secret_ocid = "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya3tsglrzfgiyfisxchref774l5y4nrler2vn54lr3li7q" + secret_name = "ap-devo_lab-mrds" + region = "eu-frankfurt-1" + + elif env == "tst": + secret_ocid = "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyayqqotyowhpoml3v5szkwhmtu4rq6bplpkvdruzupz3ma" + secret_name = "ap-devo_tst-mrds" + region = "eu-frankfurt-1" + + else: + raise ValueError(f"Unsupported environment: {env}. Expected 'dev' or 'tst'") + + print(f"Environment: {env}") + print(f"Secret Name: {secret_name}") + print(f"Secret OCID: {secret_ocid}") + print(f"Region: {region}") + + config = {"region": region} + secrets_client = oci.secrets.SecretsClient(config=config, signer=signer) + + try: + bundle = secrets_client.get_secret_bundle(secret_id=secret_ocid) + password = base64.b64decode(bundle.data.secret_bundle_content.content).decode('utf-8') + + print(f"Secret '{secret_name}' retrieved successfully: {len(password)} characters") + + return { + 'password': password, + 'secret_name': secret_name, + 'secret_ocid': secret_ocid, + 'environment': env, + 'region': region + } + + except Exception as e: + print(f"Error retrieving secret '{secret_name}' with OCID '{secret_ocid}': {str(e)}") + raise + +dag = DAG( + 'oci_principal_authentication_debug', + start_date=datetime(2024, 1, 1), + schedule_interval=None, + catchup=False, + description='Debug OCI authentication and retrieve secrets using Secret OCID' +) + +debug_task = PythonOperator( + task_id='detect_principal_type_and_get_secret', + python_callable=debug_oci_authentication_method, + dag=dag +) + diff --git a/airflow/tmp/rotate_airflow_logs_to_oci.py b/airflow/tmp/rotate_airflow_logs_to_oci.py new file mode 100644 index 0000000..8254e7c --- /dev/null +++ b/airflow/tmp/rotate_airflow_logs_to_oci.py @@ -0,0 +1,195 @@ +# dag /rotate_airflow_logs_to_oci.py + +import os +import sys +import json +import time +from datetime import datetime, timedelta +from pathlib import Path + +import pendulum +from airflow import DAG +from airflow.operators.python import PythonOperator + + +# Config setup1 + +MRDS_ENV = (os.getenv("MRDS_ENV") or "tst").strip().lower() +if MRDS_ENV not in {"dev", "tst", "acc", "prd"}: + MRDS_ENV = "tst" + +# Namespace (tenancy Object Storage namespace) +NAMESPACE = (os.getenv("BUCKET_NAMESPACE") or "frcnomajoc7v").strip() + +# Buckets per env (USED) +DEFAULT_BUCKETS = { + "dev": "mrds_airflow_logs_dev", + "tst": "mrds_airflow_logs_tst", + "acc": "mrds_airflow_logs_acc", + "prd": "mrds_airflow_logs_prd", +} + +# Airflow log path +LOG_DIR = Path("/opt/airflow/logs") +STDOUT_LOG = LOG_DIR / "stdout.log" +DAG_PM_DIR = LOG_DIR / "dag_processor_manager" +DAG_PM_LOG = DAG_PM_DIR / "dag_processor_manager.log" + +# Object key prefix in the bucket (folder) +OBJECT_PREFIX = os.getenv("LOGS_OBJECT_PREFIX", "AIRFLOW_STD_OUT_LOG_ROTATE").strip("/") + + +# Helpers + +def pp(tag: str, obj) -> None: + try: + print(f"[{tag}] {json.dumps(obj, indent=2, sort_keys=True)}", flush=True) + except Exception: + print(f"[{tag}] {obj!r}", flush=True) + +def _oci_client(): + """ + Create an OCI Object Storage client... + Order: Resource Principals -> Instance Principals. + """ + import oci + region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1" + + # Resource Principals + try: + rp_signer = oci.auth.signers.get_resource_principals_signer() + cfg = {"region": region} + pp("oci.auth", {"mode": "resource_principals", "region": region}) + return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer) + except Exception as e: + pp("oci.auth.rp_unavailable", str(e)) + + # Instance Principals + try: + ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + cfg = {"region": region} + pp("oci.auth", {"mode": "instance_principals", "region": region}) + return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer) + except Exception as e: + pp("oci.auth.ip_unavailable", str(e)) + + raise RuntimeError("No OCI credentials found (Resource Principals or Instance Principals).") + +def _resolve_bucket() -> str: + """Always use the bucket mapped from DEFAULT_BUCKETS based on MRDS_ENV.""" + return DEFAULT_BUCKETS[MRDS_ENV] + +def _resolve_namespace(client) -> str: + """Use provided NAMESPACE if set; otherwise discover it.""" + if NAMESPACE: + return NAMESPACE + ns = client.get_namespace().data + pp("oci.namespace.discovered", ns) + return ns + +def _rotate(src_path: Path, rotated_path: Path) -> bool: + """ + a) Copy current log to a dated file and truncate original. + b) Returns True if a rotated file was created (and non-empty), else False. + """ + if not src_path.exists(): + pp("rotate.skip", {"reason": "missing", "path": str(src_path)}) + return False + + try: + content = src_path.read_text(encoding="utf-8", errors="ignore") + except Exception: + content = src_path.read_bytes().decode("utf-8", errors="ignore") + + if not content.strip(): + pp("rotate.skip", {"reason": "empty", "path": str(src_path)}) + return False + + rotated_path.write_text(content, encoding="utf-8", errors="ignore") + # truncate original + src_path.write_text("", encoding="utf-8") + pp("rotate.done", {"src": str(src_path), "rotated": str(rotated_path), "bytes": len(content.encode("utf-8"))}) + return True + +def _upload(client, namespace: str, bucket: str, object_name: str, file_path: Path) -> None: + import oci + for attempt in range(1, 4): + try: + with file_path.open("rb") as fh: + resp = client.put_object(namespace, bucket, object_name, fh) + pp("oci.put_object.ok", {"bucket": bucket, "name": object_name, "status": getattr(resp, "status", None)}) + return + except Exception as e: + pp("oci.put_object.error", {"attempt": attempt, "bucket": bucket, "name": object_name, "error": str(e)}) + if attempt < 3: + time.sleep(1.5 * attempt) + else: + raise + +def _process_one(client, namespace: str, bucket: str, date_str: str, src: Path, base_name: str): + rotated = src.with_name(f"{src.name}.{date_str}") + if not _rotate(src, rotated): + return + # Build object key (no leading slash) + obj = f"{OBJECT_PREFIX}/{date_str}/{base_name}" + _upload(client, namespace, bucket, obj, rotated) + try: + rotated.unlink(missing_ok=True) + pp("cleanup.rotated_removed", str(rotated)) + except Exception as e: + pp("cleanup.rotated_remove_failed", {"path": str(rotated), "error": str(e)}) + +def rotate_and_upload_main(): + try: + client = _oci_client() + namespace = _resolve_namespace(client) + bucket = _resolve_bucket() + + # compute "yesterday" at runtime (UTC) so it's always current + date_str = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%d") + + pp("uploader.config", { + "env": MRDS_ENV, + "namespace": namespace, + "bucket": bucket, + "object_prefix": OBJECT_PREFIX, + "date": date_str, + "log_dir": str(LOG_DIR), + }) + + # Small delay to ensure logs are flushed before rotation + time.sleep(3) + + _process_one(client, namespace, bucket, date_str, STDOUT_LOG, "stdout.log") + _process_one(client, namespace, bucket, date_str, DAG_PM_LOG, "dag_processor_manager.log") + + pp("uploader.done", {"env": MRDS_ENV}) + except Exception as e: + pp("uploader.failed", str(e)) + raise + + +# DAG definition + +default_args = { + "owner": "data-eng", + "retries": 1, + "retry_delay": pendulum.duration(minutes=5), +} + +with DAG( + dag_id="rotate_airflow_logs_to_oci", + description="Rotate Airflow logs and upload to OCI Object Storage daily.", + start_date=datetime(2025, 10, 21), + schedule_interval="0 0 * * *", # runs daily right after 00:00 UTC + catchup=False, + default_args=default_args, + tags=["logs", "oci", "maintenance"], +) as dag: + + rotate_and_upload = PythonOperator( + task_id="rotate_and_upload_logs", + python_callable=rotate_and_upload_main, + ) + + rotate_and_upload diff --git a/dbt/.gitignore b/dbt/.gitignore new file mode 100644 index 0000000..49f147c --- /dev/null +++ b/dbt/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/dbt/README.md b/dbt/README.md new file mode 100644 index 0000000..7874ac8 --- /dev/null +++ b/dbt/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/dbt/__profiles.yml b/dbt/__profiles.yml new file mode 100644 index 0000000..26a4cea --- /dev/null +++ b/dbt/__profiles.yml @@ -0,0 +1,47 @@ +mrds: + target: dev + outputs: + dev: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + tst: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + acc: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + prd: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 \ No newline at end of file diff --git a/dbt/_profiles.yml b/dbt/_profiles.yml new file mode 100644 index 0000000..26a4cea --- /dev/null +++ b/dbt/_profiles.yml @@ -0,0 +1,47 @@ +mrds: + target: dev + outputs: + dev: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + tst: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + acc: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 + prd: + type: $DBT_PROFILE_TARGET_TYPE + host: $DBT_PROFILE_TARGET_HOST + user: $DBT_PROFILE_TARGET_USER + password: $DBT_PROFILE_TARGET_PASSWORD + port: $DBT_PROFILE_TARGET_PORT + database: $DBT_DATABASE_NAME + service: $DBT_PROFILE_TARGET_SERVICE_NAME + protocol: tcps + schema: public + threads: 4 \ No newline at end of file diff --git a/dbt/analyses/.gitkeep b/dbt/analyses/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml new file mode 100644 index 0000000..945de64 --- /dev/null +++ b/dbt/dbt_project.yml @@ -0,0 +1,68 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'mrds' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'mrds' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +vars: + dbt_timezone: 'Europe/Berlin' + + orchestration_run_id: null + input_service_name: null + workflow_name: null + tag_to_run: 'default_tag_lineage' + +flags: + send_anonymous_usage_stats: False + +#on-run-start: +# - "{{ control_run_start() }}" + +on-run-end: + - "{{ control_run_end(results) }}" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + +on_schema_change: "sync_all_columns" + mrds: + # Config indicated by + and applies to all files under models/example/ + example: + +enabled: false + mopdb: + source_qualifiers: + +pre-hook: "{{ control_model_start() }}" + +post-hook: "{{ control_model_end('SQ') }}" + +materialized: table + targets: + +pre-hook: "{{ control_model_start() }}" + +post-hook: "{{ control_model_end('TRG') }}" + rar: + source_qualifiers: + +materialized: table + +snapshots: + +pre-hook: "{{ control_model_start() }}" + +post-hook: "{{ control_model_end() }}" diff --git a/dbt/dxc/remove_dbt_env_variables.ps1 b/dbt/dxc/remove_dbt_env_variables.ps1 new file mode 100644 index 0000000..3d57db1 --- /dev/null +++ b/dbt/dxc/remove_dbt_env_variables.ps1 @@ -0,0 +1,39 @@ +# Script to remove DBT environment variables permanently from User scope +# Run this script to clean up all DBT-related environment variables + +Write-Host "Removing DBT environment variables for all targets..." -ForegroundColor Red +Write-Host "It can runs slowly, please wait with patience ..." -ForegroundColor Red +Write-Host "" + + +# POC Environment +Write-Host "Removing POC environment variables..." -ForegroundColor Yellow +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_TYPE_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_HOST_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_USER_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PASSWORD_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PORT_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_DATABASE_NAME_POC', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_SERVICE_NAME_POC', $null, 'User') + +# DEV Environment +Write-Host "Removing DEV environment variables..." -ForegroundColor Yellow +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_TYPE_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_HOST_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_USER_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PASSWORD_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PORT_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_DATABASE_NAME_DEV', $null, 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_SERVICE_NAME_DEV', $null, 'User') + + +Write-Host "" +Write-Host "All environment variables have been removed successfully!" -ForegroundColor Green +Write-Host "" +Write-Host "IMPORTANT: You need to restart PowerShell for changes to take effect." -ForegroundColor Yellow +Write-Host "" +Write-Host "To verify, open a NEW PowerShell window and run:" -ForegroundColor Yellow +Write-Host " Get-ChildItem Env: | Where-Object Name -like 'DBT_*'" -ForegroundColor White +Write-Host "" +Write-Host "Expected result: No variables should be found." -ForegroundColor White + \ No newline at end of file diff --git a/dbt/dxc/set_dbt_env_variables.ps1 b/dbt/dxc/set_dbt_env_variables.ps1 new file mode 100644 index 0000000..73fea77 --- /dev/null +++ b/dbt/dxc/set_dbt_env_variables.ps1 @@ -0,0 +1,38 @@ +# Script to set DBT environment variables permanently for User +# Run this script once to configure all environment variables for all targets + +Write-Host "Setting DBT environment variables for all targets..." -ForegroundColor Green +Write-Host "" +Write-Host "It can runs slowly, please wait with patience ..." -ForegroundColor Red +Write-Host "" + +# POC Environment +Write-Host "Setting POC environment variables..." -ForegroundColor Cyan +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_TYPE_POC', 'oracle', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_HOST_POC', 'PE-lzprd-mrds-tst-dmarsdb1.adb.eu-frankfurt-1.oraclecloud.com', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_USER_POC', 'MRDS_LOADER', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PASSWORD_POC', 'Mars_db_12345678', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PORT_POC', '1521', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_DATABASE_NAME_POC', 'G9E081DDAD95B2D_DMARSDB1', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_SERVICE_NAME_POC', 'g9e081ddad95b2d_dmarsdb1_low.adb.oraclecloud.com', 'User') + +# DEV Environment +Write-Host "Setting DEV environment variables..." -ForegroundColor Cyan +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_TYPE_DEV', 'oracle', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_HOST_DEV', 'PE-lzprd-mrds-tst-dmopdb.adb.eu-frankfurt-1.oraclecloud.com', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_USER_DEV', 'MRDS_LOADER', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PASSWORD_DEV', 'Tesat_ysklLawga_uskjas3', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_PORT_DEV', '1521', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_DATABASE_NAME_DEV', 'G9E081DDAD95B2D_DMOPDB', 'User') +[System.Environment]::SetEnvironmentVariable('DBT_PROFILE_TARGET_SERVICE_NAME_DEV', 'g9e081ddad95b2d_dmopdb_medium.adb.oraclecloud.com', 'User') + + +Write-Host "" +Write-Host "All environment variables have been set successfully!" -ForegroundColor Green +Write-Host "" +Write-Host "IMPORTANT: You need to restart PowerShell for changes to take effect." -ForegroundColor Yellow +Write-Host "" +Write-Host "To verify, open a NEW PowerShell window and run:" -ForegroundColor Yellow +Write-Host " Get-ChildItem Env: | Where-Object Name -like 'DBT_*'" -ForegroundColor White + + \ No newline at end of file diff --git a/dbt/macros/.gitkeep b/dbt/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt/macros/check_data_exists.sql b/dbt/macros/check_data_exists.sql new file mode 100644 index 0000000..a61ede6 --- /dev/null +++ b/dbt/macros/check_data_exists.sql @@ -0,0 +1,19 @@ +{% macro check_data_exists(table_name) %} +{% if execute %} + + {% set run_sql %} + select count(*) + from ( + select 1 res + from {{ table_name }} + where rownum = 1 + ) + {% endset %} + + {% set res = run_query(run_sql) %} + {% set row_ct = res.columns[0].values()[0] %} + + {{ return(row_ct) }} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/common/check_model_exists.sql b/dbt/macros/common/check_model_exists.sql new file mode 100644 index 0000000..d7a2503 --- /dev/null +++ b/dbt/macros/common/check_model_exists.sql @@ -0,0 +1,16 @@ +{% macro check_model_exists(model_name) %} +{% if execute %} + {% if manifest %} + {% set model_exists = false %} + {% for node in manifest.nodes.values() %} + {% if node.resource_type == 'model' and node.name == model_name %} + {% set model_exists = true %} + {% break %} + {% endif %} + {% endfor %} + {{ return(model_exists) }} + {% else %} + {{ exceptions.raise_compiler_error("'manifest' object is not available in this context.") }} + {% endif %} +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/common/get_child_models.sql b/dbt/macros/common/get_child_models.sql new file mode 100644 index 0000000..0148690 --- /dev/null +++ b/dbt/macros/common/get_child_models.sql @@ -0,0 +1,15 @@ +{% macro get_child_models(model_name) %} + {% if execute %} + {% set graph = graph.nodes %} + {% set children = [] %} + {% for node, details in graph.items() %} + {% for item in details.depends_on.nodes %} + {% if model_name == item.split('.')[2] %} + {% do children.append(node.split('.')[2]) %} + {% break %} + {% endif %} + {% endfor %} + {% endfor %} + {{ return(children) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/common/get_parent_models.sql b/dbt/macros/common/get_parent_models.sql new file mode 100644 index 0000000..e14a235 --- /dev/null +++ b/dbt/macros/common/get_parent_models.sql @@ -0,0 +1,12 @@ +{% macro get_parent_models(model_name) %} + {% if execute %} + {% set model = ( graph.nodes.values() | selectattr('name', 'equalto', model_name) | list ).pop() %} + {% set depends_on = model.depends_on['nodes'] %} + {% set new_depends_on = [] %} + {% for item in depends_on %} + {% do new_depends_on.append(item.split('.')[2]) %} + {% endfor %} + {{ return(new_depends_on) }} + + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/common/get_table_qualified_name.sql b/dbt/macros/common/get_table_qualified_name.sql new file mode 100644 index 0000000..7a188ca --- /dev/null +++ b/dbt/macros/common/get_table_qualified_name.sql @@ -0,0 +1,12 @@ +{% macro get_table_qualified_name(model_name) %} + {% if execute %} + {% set models = graph.nodes.values() %} + {% set model = (models | selectattr('name', 'equalto', model_name) | list).pop() %} + {% if model is not none %} + {% set table_qualified_name = model.schema ~ '.' ~ model.alias if model.alias is not none and model.alias != '' else model.schema ~ '.' ~ model.name %} + {{ return(table_qualified_name) }} + {% else %} + {{ exceptions.raise("Model not found: " ~ model_name) }} + {% endif %} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_external_run_end.sql b/dbt/macros/control_tables/control_external_run_end.sql new file mode 100644 index 0000000..4d6b132 --- /dev/null +++ b/dbt/macros/control_tables/control_external_run_end.sql @@ -0,0 +1,22 @@ +{% macro control_external_run_end() %} + + {{ elog("START macro -> control_external_run_end", "INFO", 0)}} + + {% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %} + {{ elog("Get workflow status", "INFO", 1)}} + {% set status = get_workflow_status() %} + {{ check_null("status", status) }} + + {{ elog("Run macro update_A_WORKFLOW_HISTORY", "INFO", 1)}} + {{ update_A_WORKFLOW_HISTORY(convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), status) }} + + {# {{ elog("Run macro insert_A_MOPDB_LOAD_HISTORY", "INFO", 1)}} + {{ insert_A_MOPDB_LOAD_HISTORY() }}#} + {% do run_query('COMMIT') %} + {% else %} + {{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}} + {% endif %} + + {{ elog("END macro -> control_external_run_end", "INFO", 0)}} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_external_run_start.sql b/dbt/macros/control_tables/control_external_run_start.sql new file mode 100644 index 0000000..e603289 --- /dev/null +++ b/dbt/macros/control_tables/control_external_run_start.sql @@ -0,0 +1,13 @@ +{% macro control_external_run_start() %} + {{ elog("START macro -> control_external_run_start", "INFO", 0)}} + + {% if flags.WHICH in ('run', 'seed', 'snapshot', 'build', 'run-operation') %} + {{ elog("Run macro insert_A_WORKFLOW_HISTORY", "INFO", 1)}} + {{ insert_A_WORKFLOW_HISTORY() }} + {% do run_query('COMMIT') %} + {% else %} + {{ elog("Flag WHICH " ~ flags.WHICH ~ " is out of the list, do nothing.", "WARN", 1)}} + {% endif %} + + {{ elog("END macro -> control_external_run_start", "INFO", 0)}} +{% endmacro %} diff --git a/dbt/macros/control_tables/control_model_end.sql b/dbt/macros/control_tables/control_model_end.sql new file mode 100644 index 0000000..43c50e9 --- /dev/null +++ b/dbt/macros/control_tables/control_model_end.sql @@ -0,0 +1,66 @@ +{% macro control_model_end(model_type) %} + {% set model_name = model.name %} + {{ elog("START macro -> control_model_end, model " ~ model_name, "INFO", 0)}} + + {{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}} + {% set a_task_history_key = get_task_history_key(model_name) %} + {{ check_null("a_task_history_key", a_task_history_key) }} + + {% if model_type == 'SQ' %} + {% set table_name = this.schema ~ '.' ~ this.name %} + {{ elog("Check if data exists in SQ, model " ~ model_name, "INFO", 1)}} + {% set sq_data_exists = check_data_exists(table_name) %} + {% if 'm_DWH' in model_name and 'm_DWH_PRELOAD' not in model_name %} + {% set model_sources =get_parent_models(model_name) %} + {% if model_sources | length > 0 %} + {% for model_source in model_sources %} + {% set source_name = get_table_qualified_name(model_source) %} + {{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}} + {{ control_source_count(a_task_history_key, source_name,'', '') }} + {% endfor %} + {% endif %} + {% else %} + {% set model_sources =model.sources %} + {% for model_source in model_sources %} + {% if model_source[0] != 'control_tables' %} + {% set source_name = source(model_source[0], model_source[1]) %} + {{ elog("Run control_source_count for source " ~ source_name ~ ", model " ~ model_name, "INFO", 1)}} + {{ control_source_count(a_task_history_key, source_name,'', '') }} + {% endif %} + {% endfor %} + {% endif %} + + {% elif model_type == 'TRG' %} + {# {{ elog('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', "INFO", 1) }} #} + {% set parent_models = get_parent_models(model_name) %} + {% if parent_models | length > 0 %} + {% for model in parent_models %} + {% if 'm_DWH' in model or 'm_MOPDB' in model %} + {% set table_name = get_table_qualified_name(model) %} + {# {{ elog('--------------->> Table: ', "INFO", 1) }} #} + {# {{ elog(table_name, "INFO", 1) }} #} + {{ elog("Check if data exists in SQ table: " ~ table_name ~ " for model: " ~ model_name, "INFO", 1)}} + {% set trg_data_exists = check_data_exists(table_name) %} + {{ elog("Run control_source_count for source " ~ table_name ~ ", model " ~ model_name, "INFO", 1)}} + {{ control_source_count(a_task_history_key, table_name,'A_WORKFLOW_HISTORY_KEY', trg_data_exists) }} + {% endif %} + {% endfor %} + {% else %} + {{ elog("There are no parent SQ for model: " ~ model_name, "INFO", 1)}} + {% endif %} + {# {{ log(parent_models, info=True) }} #} + + {% endif %} + + + + {{ elog("Run macro update_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}} + {{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(m_current_timestamp('dbt'),'YYYY-MM-DD HH24:MI:SS',''), 'Y') }} + + {% if sq_data_exists == 0 %} + {{ exceptions.warn("Model " ~ model_name ~ " has no rows.") }} + {% endif %} + + {% do run_query('COMMIT') %} + {{ elog("END macro -> control_model_end, model " ~ model_name, "INFO", 0)}} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_model_start.sql b/dbt/macros/control_tables/control_model_start.sql new file mode 100644 index 0000000..734453a --- /dev/null +++ b/dbt/macros/control_tables/control_model_start.sql @@ -0,0 +1,16 @@ +{% macro control_model_start() %} + {% set model_name = model.name %} + + {{ elog("START macro -> control_model_start, model " ~ model_name, "INFO", 0)}} + + {{ elog("Get process_key for model " ~ model_name, "INFO", 1)}} + {% set a_workflow_history_key = get_workflow_history_key() %} + {{ check_null("a_workflow_history_key", a_workflow_history_key) }} + {{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}} + + {{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}} + {{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }} + {% do run_query('COMMIT') %} + + {{ elog("END macro -> control_model_start, model " ~ model_name, "INFO", 0)}} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_run_end.sql b/dbt/macros/control_tables/control_run_end.sql new file mode 100644 index 0000000..8564ab0 --- /dev/null +++ b/dbt/macros/control_tables/control_run_end.sql @@ -0,0 +1,49 @@ +{% macro control_run_end(results) %} + {% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %} + {{ elog("START macro -> control_run_end", "INFO", 0)}} + {% for result in results %} + {% if result.status != 'skipped' %} + {% set model_name = result.node.name %} + {% set rows_affected = result.adapter_response.get('rows_affected', 0) %} + {% if result.status == 'success' %} {% if rows_affected == 0 %}{% set status = 'W' %}{% else %}{% set status = 'Y' %}{% endif %} {% else %} {% set status = 'N' %} {% endif %} + {% set completed_at = result.timing[-1].completed_at.strftime("%Y-%m-%d %H:%M:%S") %} + {% set table_name = result.node.relation_name %} + + {%set full_model_key = result.node.unique_id%} + {%set model = graph.nodes.get(full_model_key)%} + + {%if model is not none and (result.node.path.startswith('mopdb') or result.node.path.startswith('rar')) %} + {%set model_tags = model.config.get('tags',[])%} + {%set input_service_name_var = var('input_service_name') %} + {%if 'A_TASK_HISTORY_TARGET_insert' not in model_tags and input_service_name_var == 'RAR'%} + {# Skip the following lines and go straight to the end #} + {{elog("RAR model without a A_TASK_HISTORY_TARGET_insert tag. Skip inserting into A_TASK_HISTORY_TARGET for" ~ model_name, "INFO", 1)}} + {%elif model.config.materialized == 'table_if_rows_exist' %} + {{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}} + {% set a_task_history_key = get_task_history_key(model_name) %} + {%if a_task_history_key != null and rows_affected > 0%} + {{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}} + {{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }} + {% do run_query('COMMIT') %} + {% endif %} + {%else%} + {{ elog("Get subprocess_key for model " ~ model_name, "INFO", 1)}} + {% set a_task_history_key = get_task_history_key(model_name) %} + {{ check_null("a_task_history_key", a_task_history_key) }} + + {{ elog("Run macro insert_A_TASK_HISTORY_TARGET for model " ~ model_name, "INFO", 1)}} + {{ insert_A_TASK_HISTORY_TARGET(a_task_history_key, table_name, status, rows_affected) }} + {% do run_query('COMMIT') %} + {% endif %} + {%endif%} +{# + {% if status == 'N' %} + {{ update_A_TASK_HISTORY(a_task_history_key, convert_timezone(completed_at,'YYYY-MM-DD HH24:MI:SS','UTC'), 'N') }} + {% endif %} +#} + {% endif %} + + {% endfor %} + {{ elog("END macro -> control_run_end", "INFO", 0)}} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_run_start.sql b/dbt/macros/control_tables/control_run_start.sql new file mode 100644 index 0000000..785c0bc --- /dev/null +++ b/dbt/macros/control_tables/control_run_start.sql @@ -0,0 +1,12 @@ +{% macro control_run_start() %} + {% if flags.WHICH in ('run', 'seed', 'snapshot', 'build') %} + {{ elog("START macro -> control_run_start", "INFO", 0)}} + + {{ elog("Checking required variables...", "INFO", 1)}} + {{ check_variable("orchestration_run_id") }} + {{ check_variable("input_service_name") }} + {{ check_variable("workflow_name") }} + + {{ elog("END macro -> control_run_start", "INFO", 0)}} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/control_source_count.sql b/dbt/macros/control_tables/control_source_count.sql new file mode 100644 index 0000000..628b35d --- /dev/null +++ b/dbt/macros/control_tables/control_source_count.sql @@ -0,0 +1,7 @@ +{% macro control_source_count(a_task_history_key, table_name, column_name, sq_data_exists) %} + {% set model_name = model.name %} + + {{ elog("Run macro insert_A_TASK_HISTORY_SOURCE for model " ~ model_name, "INFO", 2)}} + {{ insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) }} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/get_task_history_key.sql b/dbt/macros/control_tables/get_task_history_key.sql new file mode 100644 index 0000000..83c3a2e --- /dev/null +++ b/dbt/macros/control_tables/get_task_history_key.sql @@ -0,0 +1,20 @@ +{% macro get_task_history_key(model_name) %} +{% if execute %} + + {% set fetch_fk %} + SELECT A_TASK_HISTORY_KEY + FROM {{ source('control_tables', 'A_TASK_HISTORY') }} + WHERE TASK_RUN_ID = '{{ invocation_id }}' + AND TASK_NAME = '{{ model_name }}' + AND SERVICE_NAME = '{{ var("input_service_name") }}' + {% endset %} + + {% set a_task_history_key_result = run_query(fetch_fk) %} + {% set a_task_history_key = a_task_history_key_result.columns[0].values()[0] %} + + {{ return(a_task_history_key) }} + +{% endif %} +{% endmacro %} + + diff --git a/dbt/macros/control_tables/get_workflow_history_key.sql b/dbt/macros/control_tables/get_workflow_history_key.sql new file mode 100644 index 0000000..a2ee0e6 --- /dev/null +++ b/dbt/macros/control_tables/get_workflow_history_key.sql @@ -0,0 +1,17 @@ +{% macro get_workflow_history_key() %} +{% if execute %} + + {% set fetch_fk %} + SELECT A_WORKFLOW_HISTORY_KEY + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} + WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id")}}' + and SERVICE_NAME = '{{ var("input_service_name")}}' + {% endset %} + + {% set a_workflow_history_key_result = run_query(fetch_fk) %} + {% set a_workflow_history_key = a_workflow_history_key_result.columns[0].values()[0] %} + + {{ return(a_workflow_history_key) }} + +{% endif %} +{% endmacro %} diff --git a/dbt/macros/control_tables/get_workflow_status.sql b/dbt/macros/control_tables/get_workflow_status.sql new file mode 100644 index 0000000..7a51b22 --- /dev/null +++ b/dbt/macros/control_tables/get_workflow_status.sql @@ -0,0 +1,20 @@ +{% macro get_workflow_status() %} + {% if execute %} + + {% set run_sql %} + select case when sum(case when tht.load_successful in ('W','Y') then 0 else 1 end) = 0 then 'Y' else 'N' end status + from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh + join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name + join {{ source('control_tables','A_TASK_HISTORY_TARGET') }} tht on th.a_task_history_key = tht.a_task_history_key and th.service_name = tht.service_name + where wh.service_name = '{{ var("input_service_name") }}' + and wh.workflow_name = '{{ var("workflow_name") }}' + and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}' + {% endset %} + + {% set res = run_query(run_sql) %} + {% set status = res.columns[0].values()[0] %} + + {{ return(status) }} + + {% endif %} +{% endmacro %} diff --git a/dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql b/dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql new file mode 100644 index 0000000..1344cf3 --- /dev/null +++ b/dbt/macros/control_tables/insert_A_MOPDB_LOAD_HISTORY.sql @@ -0,0 +1,25 @@ +{% macro insert_A_MOPDB_LOAD_HISTORY() %} +{% if execute %} + + {% set insert_A_MOPDB_LOAD_HISTORY %} + INSERT INTO {{ source('control_tables', 'A_MOPDB_LOAD_HISTORY') }} (A_WORKFLOW_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, TASK_NAME, TASK_START, TASK_END, WORKFLOW_STATUS) + select distinct wh.a_workflow_history_key + ,ths.a_workflow_history_source_key + ,th.a_task_history_key + ,th.task_name + ,th.task_start + ,th.task_end + ,wh.workflow_status + from {{ source('control_tables','A_WORKFLOW_HISTORY') }} wh + join {{ source('control_tables','A_TASK_HISTORY') }} th on wh.a_workflow_history_key = th.a_workflow_history_key and wh.service_name = th.service_name + join {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} ths on th.a_task_history_key = ths.a_task_history_key and th.service_name = ths.service_name + where wh.service_name = '{{ var("input_service_name") }}' + and wh.workflow_name = '{{ var("workflow_name") }}' + and wh.orchestration_run_id = '{{ var("orchestration_run_id") }}' + and ths.a_workflow_history_source_key is not null + {% endset %} + + {% do run_query(insert_A_MOPDB_LOAD_HISTORY) %} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/insert_A_TASK_HISTORY.sql b/dbt/macros/control_tables/insert_A_TASK_HISTORY.sql new file mode 100644 index 0000000..e33f357 --- /dev/null +++ b/dbt/macros/control_tables/insert_A_TASK_HISTORY.sql @@ -0,0 +1,12 @@ +{% macro insert_A_TASK_HISTORY(a_workflow_history_key, model_name) %} +{% if execute %} + + {% set insert_A_TASK_HISTORY %} + INSERT INTO {{ source('control_tables', 'A_TASK_HISTORY') }} (A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_KEY, TASK_RUN_ID, TASK_NAME, TASK_START, TASK_SUCCESSFUL, SERVICE_NAME) + VALUES ({{ source('control_sequences','A_TASK_HISTORY_SEQ') }}.NEXTVAL, {{ a_workflow_history_key }}, '{{ invocation_id}}' ,'{{ model_name }}', cast(current_timestamp as timestamp(0)), 'N', '{{ var("input_service_name") }}') + {% endset %} + + {% do run_query(insert_A_TASK_HISTORY) %} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql b/dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql new file mode 100644 index 0000000..43a7493 --- /dev/null +++ b/dbt/macros/control_tables/insert_A_TASK_HISTORY_SOURCE.sql @@ -0,0 +1,33 @@ +{% macro insert_A_TASK_HISTORY_SOURCE(table_name, a_task_history_key, column_name, sq_data_exists) %} + +{% if execute %} + + {% set insert_A_TASK_HISTORY_SOURCE %} + {% if column_name | length %} + {% if sq_data_exists == 0 %} + INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME) + SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', 0, '{{ var("input_service_name") }}' FROM dual + {% else %} + INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, A_WORKFLOW_HISTORY_SOURCE_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME) + SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, col_name, '{{ table_name }}', ct, '{{ var("input_service_name") }}' + FROM ( + SELECT {{ column_name }} col_name, count(*) ct + FROM {{ table_name }} + GROUP BY {{ column_name }} + ) + {% endif %} + {% else %} + INSERT INTO {{ source('control_tables','A_TASK_HISTORY_SOURCE') }} (A_TASK_HISTORY_SOURCE_KEY, A_TASK_HISTORY_KEY, SOURCE_NAME, ROW_COUNT, SERVICE_NAME) + SELECT {{ source('control_sequences','A_TASK_HISTORY_SOURCE_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ table_name }}', ct, '{{ var("input_service_name") }}' + FROM ( + SELECT count(*) ct + FROM {{ table_name }} + ) + {% endif %} + {% endset %} + + {% do run_query(insert_A_TASK_HISTORY_SOURCE) %} + +{% endif %} + +{% endmacro %} diff --git a/dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql b/dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql new file mode 100644 index 0000000..40a8f60 --- /dev/null +++ b/dbt/macros/control_tables/insert_A_TASK_HISTORY_TARGET.sql @@ -0,0 +1,13 @@ +{% macro insert_A_TASK_HISTORY_TARGET(a_task_history_key, model_name, status, rows_affected) %} + +{% if execute %} + + {% set insert_A_TASK_HISTORY_TARGET %} + INSERT INTO {{ source('control_tables','A_TASK_HISTORY_TARGET') }} (A_TASK_HISTORY_TARGET_KEY, A_TASK_HISTORY_KEY, TARGET_NAME, ROW_COUNT_APPLIED, LOAD_SUCCESSFUL, SERVICE_NAME) + VALUES ({{ source('control_sequences','A_TASK_HISTORY_TARGET_SEQ') }}.NEXTVAL, {{ a_task_history_key }}, '{{ model_name }}', {{ rows_affected }}, '{{ status }}', '{{ var("input_service_name") }}') + {% endset %} + + {% do run_query(insert_A_TASK_HISTORY_TARGET) %} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql b/dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql new file mode 100644 index 0000000..6577019 --- /dev/null +++ b/dbt/macros/control_tables/insert_A_WORKFLOW_HISTORY.sql @@ -0,0 +1,13 @@ +{% macro insert_A_WORKFLOW_HISTORY() %} + +{% if execute %} + + {% set insert_A_WORKFLOW_HISTORY %} + INSERT INTO {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} (SERVICE_NAME, A_WORKFLOW_HISTORY_KEY, ORCHESTRATION_RUN_ID, WORKFLOW_NAME, WORKFLOW_START, WORKFLOW_SUCCESSFUL, DBT_INVOCATION_ID) + VALUES ('{{ var("input_service_name") }}', {{ source('control_sequences','A_WORKFLOW_HISTORY_SEQ') }}.NEXTVAL, '{{ var("orchestration_run_id") }}', '{{ var("workflow_name") }}', cast(current_timestamp as timestamp(0)), 'R', '{{ invocation_id }}') + {% endset %} + + {% do run_query(insert_A_WORKFLOW_HISTORY) %} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/update_A_TASK_HISTORY.sql b/dbt/macros/control_tables/update_A_TASK_HISTORY.sql new file mode 100644 index 0000000..81964c0 --- /dev/null +++ b/dbt/macros/control_tables/update_A_TASK_HISTORY.sql @@ -0,0 +1,16 @@ +{% macro update_A_TASK_HISTORY(a_task_history_key, process_end, status) %} + +{% if execute %} + + {% set update_table %} + UPDATE {{ source('control_tables', 'A_TASK_HISTORY') }} + SET TASK_END = {{ process_end }}, + TASK_SUCCESSFUL = '{{ status }}' + WHERE A_TASK_HISTORY_KEY = {{ a_task_history_key }} + {% endset %} + + {% do run_query(update_table) %} + +{% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql b/dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql new file mode 100644 index 0000000..ff4be70 --- /dev/null +++ b/dbt/macros/control_tables/update_A_WORKFLOW_HISTORY.sql @@ -0,0 +1,14 @@ +{% macro update_A_WORKFLOW_HISTORY(process_end, status) %} +{% if execute %} + + {% set update_table %} + UPDATE {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} + SET WORKFLOW_END = {{ process_end }}, + WORKFLOW_SUCCESSFUL = '{{ status }}' + WHERE ORCHESTRATION_RUN_ID = '{{ var("orchestration_run_id") }}' + {% endset %} + + {% do run_query(update_table) %} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/convert_timezone.sql b/dbt/macros/convert_timezone.sql new file mode 100644 index 0000000..d2c2df3 --- /dev/null +++ b/dbt/macros/convert_timezone.sql @@ -0,0 +1,12 @@ +{% macro convert_timezone(value, format, timezone) %} + {% if timezone | length %} + {% set expression %} + cast (FROM_TZ(to_timestamp('{{ value }}', '{{ format }}'), '{{ timezone }}') at time zone '{{ var("dbt_timezone") }}' as timestamp) + {% endset %} + {% else %} + {% set expression %} + to_timestamp('{{ value }}', '{{ format }}') + {% endset %} + {% endif %} + {{ return(expression) }} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/data_transform/clean_boolean_fxcd.sql b/dbt/macros/data_transform/clean_boolean_fxcd.sql new file mode 100644 index 0000000..010555d --- /dev/null +++ b/dbt/macros/data_transform/clean_boolean_fxcd.sql @@ -0,0 +1,8 @@ +{% macro clean_boolean_fxcd(column_name) %} + {# Replaces Informatica UDF: udf_clean_BOOLEAN_FIELD_STR_FXCD #} + CASE + WHEN {{ column_name }} IS NULL THEN NULL + WHEN UPPER({{ column_name }}) IN ('N', 'NO', '0', ' ') THEN 'N' + ELSE 'Y' + END +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/data_transform/create_table_from_source.sql b/dbt/macros/data_transform/create_table_from_source.sql new file mode 100644 index 0000000..1052289 --- /dev/null +++ b/dbt/macros/data_transform/create_table_from_source.sql @@ -0,0 +1,21 @@ +{% macro create_table_from_source (schema, table, wf_name, task_name, except) %} +{# +Creates a table with a structure identical to the source table. +The content of the created table are all rows from the source table that belong + to the hightest WORKFLOW_HISTORY_KEY in the source table +#} +{% set a_workflow_history_key = get_workflow_history_key() %} +--{{ check_null("a_workflow_history_key", a_workflow_history_key) }} +{{ elog("a_workflow_history_key: " ~ a_workflow_history_key, "INFO", 2)}} + {{ elog("Run macro insert_A_TASK_HISTORY for model " ~ model_name, "INFO", 1)}} + {{ insert_A_TASK_HISTORY(a_workflow_history_key, model_name) }} + +select + {{ a_workflow_history_key }} AS A_WORKFLOW_HISTORY_KEY, + source_table.* +from {{ source(schema, table) }} source_table + where source_table.A_WORKFLOW_HISTORY_KEY IN + {{ + filter_workflow_history_key(wf_name, task_name) + }} +{% endmacro %} diff --git a/dbt/macros/data_transform/create_table_target.sql b/dbt/macros/data_transform/create_table_target.sql new file mode 100644 index 0000000..293ee5a --- /dev/null +++ b/dbt/macros/data_transform/create_table_target.sql @@ -0,0 +1,9 @@ +{% macro create_table_target (model) %} +{# +Creates a target table with the same structure and content as the source table. +Usually used to move data from SQ tables to the target table. +#} +select + * +from {{ ref(model) }} source_table +{% endmacro %} diff --git a/dbt/macros/data_transform/filter_workflow_history_key.sql b/dbt/macros/data_transform/filter_workflow_history_key.sql new file mode 100644 index 0000000..9f767d7 --- /dev/null +++ b/dbt/macros/data_transform/filter_workflow_history_key.sql @@ -0,0 +1,52 @@ +{% macro filter_workflow_history_key(wf_name, tasks_names,custom_string='') %} +{# +The query in this macro determines if data in a target table is up to date or if there is additional data in ODS +Paramters: + wf_name : The name of the ODS workflow that populates the ODS source table + task_name: The name of the task that executes the target model or an expression that returns it +Logic: + 1) The query selects all A_WORKFLOW_HISTORY_KEYs that exist in ODS for the workflow passed as parameter wf_name from the workflow history + 2) The subquery on LOAD_HISTORY joins the A_WORKFLOW_HISTORY_KEYs from 1) agains the LOAD_HISTORY for the target table and returns a 1 for all + A_WORKFLOW_HISTORY_KEYs that are found in LOAD_HISTORY + 3) The NOT IN operator excludes the A_WORKFLOW_HISTORY_KEYs matched in 2) from the result of query 1, returning only the + A_WORKFLOW_HISTORY_KEYs that were not yet loaded into the target table + +Example: + For model m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_LOAD_SQ this macro is called with + wf_name = "w_ODS_TMS_TRANSACTION" + task_name = "substr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', 1, instr('m_MOPDB_TMS_T_TRANSACTION_OU_TMS_TRANSACTION_SQ', '_', -1) -1)" + This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed) +#} +{% if var("input_service_name") == 'MOPDB' %} + {% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %} +{% elif var("input_service_name") == 'RAR' %} + {% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %} +{% endif %} + +{# Set the service name filter used in the WORKFLOW_HISTORY subquery. + Here we change the filter based on whether the input service name is MOPDB or RAR #} +{% if var("input_service_name") == 'MOPDB' %} + {% set service_name_filter = 'MOPDB' %} +{% else %} + {% set service_name_filter = 'RAR' %} +{% endif %} + +(SELECT DISTINCT A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY +{{ custom_string }} + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}') + AND A_WORKFLOW_HISTORY.WORKFLOW_END < ( + SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}' + AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} ) + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y' + AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS' + {% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %} + AND NOT EXISTS + (SELECT 1 + FROM {{ load_history_tab }} LOAD_HISTORY + WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY + AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} ) + ) + ) + {% endmacro %} \ No newline at end of file diff --git a/dbt/macros/data_transform/filter_workflow_history_max_key.sql b/dbt/macros/data_transform/filter_workflow_history_max_key.sql new file mode 100644 index 0000000..745e9d7 --- /dev/null +++ b/dbt/macros/data_transform/filter_workflow_history_max_key.sql @@ -0,0 +1,52 @@ +{% macro filter_workflow_history_max_key(wf_name, task_name) %} +{# +The query in this macro determines if data in a target table is up to date or if there is newer data in ODS +Paramters: + wf_name : The name of the ODS workflow that populates the ODS source table + task_name: The name of the task that executes the target model or an expression that returns it +Logic: + 1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY for the ODS workflow passed in as wf_name, + assuming that this is the latest data available to be loaded into the target + 2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded into the target table. + If it has already been loaded, the query returns a NULL value, indicating that there is no data to be loaded. + If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is returned to the calling model (and likely used there + to select the data from the ODS table) + +Example: + For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is called with + wf_name = "w_ODS_CSDB_RATINGS_DEVO" + task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', 1, INSTR( + 'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)" + This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed) +#} +{% if var("input_service_name") == 'MOPDB' %} + {% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %} +{% elif var("input_service_name") == 'RAR' %} + {% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %} +{% endif %} + + +{# Set the service name filter used in the WORKFLOW_HISTORY subquery. + Here we change the filter based on whether the input service name is MOPDB or RAR #} +{% if var("input_service_name") == 'MOPDB' %} + {% set service_name_filter = 'MOPDB' %} +{% else %} + {% set service_name_filter = 'RAR' %} +{% endif %} + +(SELECT CASE WHEN (SELECT COUNT(*) + FROM {{ load_history_tab }} LOAD_HISTORY + WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY IN (WH.A_WORKFLOW_HISTORY_KEY) AND LOAD_HISTORY.TASK_NAME = {{ task_name }}) > 0 + THEN NULL + ELSE WH.A_WORKFLOW_HISTORY_KEY END AS A_WORKFLOW_HISTORY_KEY +FROM (SELECT MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) AS A_WORKFLOW_HISTORY_KEY + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME like ('{{ wf_name }}') + AND A_WORKFLOW_HISTORY.WORKFLOW_END < ( + SELECT WORKFLOW_START FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}' + AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} ) + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y' + AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS' + {% if var("input_service_name") == 'RAR' %} {% endif %}) WH) + {% endmacro %} diff --git a/dbt/macros/data_transform/get_ODS_wf_start_max_key.sql b/dbt/macros/data_transform/get_ODS_wf_start_max_key.sql new file mode 100644 index 0000000..0562b3b --- /dev/null +++ b/dbt/macros/data_transform/get_ODS_wf_start_max_key.sql @@ -0,0 +1,65 @@ +{% macro get_ODS_wf_start_max_key(wf_name, tasks_names) %} +{# + The query in this macro determines if data in a target table is up to date or if there is newer data in ODS + + Parameters: + wf_name : The name of the ODS workflow that populates the ODS source table + task_name : The name of the task that executes the target model or an expression that returns it + + Logic: + 1) The subquery (second select from A_WORKFLOW_HISTORY) selects the highest A_WORKFLOW_HISTORY_KEY + for the ODS workflow passed in as wf_name, assuming that this is the latest data available to be loaded + into the target. + 2) The query then queries the LOAD_HISTORY view to check whether that workflow ID has already been loaded + into the target table. If it has already been loaded, the query returns a NULL value, indicating that there + is no data to be loaded. If it has not yet been loaded, the A_WORKFLOW_HISTORY_KEY from the subquery is + returned to the calling model (and likely used there to select the data from the ODS table) + + Example: + For model m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ this macro is + called with wf_name = "w_ODS_CSDB_RATINGS_DEVO" + task_name = "SUBSTR('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', + 1, INSTR( 'm_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ', '_', - 1) - 1)" + This task_name means that the name of the current model is changed to the name of the target model (_SQ being trimmed) +#} + +{# Set the lookup table based on the input service name #} +{% if var("input_service_name") == 'MOPDB' %} + {% set load_history_tab = source('control_tables','A_MOPDB_LOAD_HISTORY') %} +{% elif var("input_service_name") == 'RAR' %} + {% set load_history_tab = source('control_tables','A_DWH_LOAD_HISTORY') %} +{% endif %} + +{# Set the service name filter used in the WORKFLOW_HISTORY subquery. + Here we change the filter based on whether the input service name is MOPDB or RAR #} +{% if var("input_service_name") == 'MOPDB' %} + {% set service_name_filter = 'MOPDB' %} +{% else %} + {% set service_name_filter = 'RAR' %} +{% endif %} + +(SELECT + MAX(A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY) OVER (PARTITION BY TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) ) as MAX_ODS_WORKFLOW_KEY_DAY, + A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY, + TRUNC(A_WORKFLOW_HISTORY.WORKFLOW_START) as ODS_WORKFLOW_START +FROM + {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY +WHERE + A_WORKFLOW_HISTORY.WORKFLOW_NAME = ('{{ wf_name }}') + AND A_WORKFLOW_HISTORY.WORKFLOW_END < ( + SELECT WORKFLOW_START + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.SERVICE_NAME = '{{ service_name_filter }}' + AND A_WORKFLOW_HISTORY_KEY = {{ get_workflow_history_key() }} + ) + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y' + AND A_WORKFLOW_HISTORY.SERVICE_NAME = 'ODS' + {% if var("input_service_name") == 'RAR' %}AND A_WORKFLOW_HISTORY.DQ_FLAG = 'T'{% endif %} + AND NOT EXISTS + (SELECT 1 + FROM {{ load_history_tab }} LOAD_HISTORY + WHERE LOAD_HISTORY.A_WORKFLOW_HISTORY_SOURCE_KEY = A_WORKFLOW_HISTORY.A_WORKFLOW_HISTORY_KEY + AND LOAD_HISTORY.TASK_NAME in ( {{ tasks_names }} ) + ) +) + {% endmacro %} \ No newline at end of file diff --git a/dbt/macros/data_transform/sequence_next_value.sql b/dbt/macros/data_transform/sequence_next_value.sql new file mode 100644 index 0000000..2f7c1c2 --- /dev/null +++ b/dbt/macros/data_transform/sequence_next_value.sql @@ -0,0 +1,33 @@ +{% macro sequence_next_value(sequence_name, check_disable=false) %} +{# macro checks if a function for this sequence exists, +if it does, the function is used +if not, the standard command is used +check can be disabled by passing true in the 2nd paramater +#} +{% if execute %} + {% if not check_disable %} + {% set get_func %} + select count(*) + from all_objects + where object_type = 'FUNCTION' + and status = 'VALID' + and owner || '.' || object_name = upper('{{ sequence_name }}_fnc') + {% endset %} + + {% set result = run_query(get_func) %} + {% set fnc_count = result.columns[0].values()[0] %} + {% else %} + {% set fnc_count = 0 %} + {% endif %} + + {% if fnc_count > 0 %} + {% set nextvalue = sequence_name ~ '_fnc()' %} + {% else %} + {% set nextvalue = sequence_name ~ '.nextval'%} + {% endif %} + + {{ return(nextvalue) }} + +{% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/dbt_overrides/conditional_truncate.sql b/dbt/macros/dbt_overrides/conditional_truncate.sql new file mode 100644 index 0000000..000866a --- /dev/null +++ b/dbt/macros/dbt_overrides/conditional_truncate.sql @@ -0,0 +1,10 @@ +{% macro conditional_truncate(target_relation, source_relation) %} +DECLARE + row_count NUMBER; +BEGIN + SELECT COUNT(*) INTO row_count FROM {{ source_relation }}; + IF row_count > 0 THEN + EXECUTE IMMEDIATE 'TRUNCATE TABLE ' || '{{ target_relation }}'; + END IF; +END; +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/dbt_overrides/generate_schema_name.sql b/dbt/macros/dbt_overrides/generate_schema_name.sql new file mode 100644 index 0000000..a2c4d9e --- /dev/null +++ b/dbt/macros/dbt_overrides/generate_schema_name.sql @@ -0,0 +1,14 @@ +{% macro generate_schema_name(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + {%- if custom_schema_name is none -%} + + {{ default_schema }} + + {%- else -%} + + {{ custom_schema_name | trim }} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt/macros/dbt_overrides/oracle__snapshot_hash_arguments.sql b/dbt/macros/dbt_overrides/oracle__snapshot_hash_arguments.sql new file mode 100644 index 0000000..0591640 --- /dev/null +++ b/dbt/macros/dbt_overrides/oracle__snapshot_hash_arguments.sql @@ -0,0 +1,9 @@ +{% macro oracle__snapshot_hash_arguments(args) -%} + {# Oracle-specific implementation with NULL-safe SCD_ID generation #} + LOWER(RAWTOHEX(STANDARD_HASH( + {%- for arg in args -%} + coalesce(cast({{ arg }} as varchar(4000)), '__DBT_NULL__') + {%- if not loop.last %} || '|' || {%- endif -%} + {%- endfor -%} + , 'SHA256'))) +{%- endmacro %} \ No newline at end of file diff --git a/dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql b/dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql new file mode 100644 index 0000000..d07366c --- /dev/null +++ b/dbt/macros/dbt_overrides/oracle__snapshot_staging_table.sql @@ -0,0 +1,214 @@ +{% macro oracle__snapshot_staging_table(strategy, source_sql, target_relation) -%} + {#-- + Custom Oracle snapshot implementation that ensures unified timestamps across all snapshot operations. + + SOLUTION: Using strategy.updated_at throughout + - strategy.updated_at generates a single TO_TIMESTAMP literal at Jinja compile time + - This literal is reused everywhere, replacing multiple snapshot_get_time() calls + - Ensures identical timestamps for all records created in the same snapshot run + - Eliminates microsecond differences that occur when unique_key changes (DELETE+INSERT pattern) + + Changes from original dbt-oracle v1.9.4 snapshot.sql: + - Line 12: new_scd_id uses strategy.updated_at instead of snapshot_get_time() + - Lines 127-129: deletes CTE uses strategy.updated_at instead of snapshot_get_time() (3 places) + - Lines 174-175: deletion_records CTE uses strategy.updated_at instead of snapshot_get_time() (2 places) + --#} + + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% if strategy.hard_deletes == 'new_record' %} + {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, strategy.updated_at]) %} + {% endif %} + + with snapshot_query as ( + + {{ source_sql }} + + ), + + snapshotted_data as ( + + select {{ target_relation }}.*, + {{ unique_key_fields(strategy.unique_key) }} + from {{ target_relation }} + where + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + ), + + insertions_source_data as ( + + select + snapshot_query.*, + {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ oracle__get_dbt_valid_to_current(strategy, columns) }}, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} + + from snapshot_query + ), + + updates_source_data as ( + + select + snapshot_query.*, + {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} + + from snapshot_query + ), + + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + + deletes_source_data as ( + + select + snapshot_query.*, + {{ unique_key_fields(strategy.unique_key) }} + from snapshot_query + ), + {% endif %} + + insertions as ( + + select + 'insert' as dbt_change_type, + source_data.* + {%- if strategy.hard_deletes == 'new_record' -%} + ,'False' as {{ columns.dbt_is_deleted }} + {%- endif %} + + from insertions_source_data source_data + left outer join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }}) + {%- if strategy.hard_deletes == 'new_record' -%} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and snapshotted_data.{{ columns.dbt_is_deleted }} = 'True') + {%- endif %} + ) + + ), + + updates as ( + + select + 'update' as dbt_change_type, + source_data.*, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + + from updates_source_data source_data + join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where ( + {{ strategy.row_changed }} + ) + {%- if strategy.hard_deletes == 'new_record' -%} + or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + {%- endif %} + ) + + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' -%} + , + + deletes as ( + + select + 'delete' as dbt_change_type, + source_data.*, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + + from snapshotted_data + left join deletes_source_data source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + {%- if strategy.hard_deletes == 'new_record' %} + and not ( + --avoid updating the record's valid_to if the latest entry is marked as deleted + snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + and + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + ) + {%- endif %} + + ) + {%- endif %} + + {%- if strategy.hard_deletes == 'new_record' %} + {% set source_sql_cols = get_column_schema_from_query(source_sql) %} + , + deletion_records as ( + + select + 'insert' as dbt_change_type, + {%- for col in source_sql_cols -%} + snapshotted_data.{{ adapter.quote(col.column) }}, + {% endfor -%} + {%- if strategy.unique_key | is_list -%} + {%- for key in strategy.unique_key -%} + snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }}, + {% endfor -%} + {%- else -%} + snapshotted_data.dbt_unique_key as dbt_unique_key, + {% endif -%} + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }}, + {{ new_scd_id }} as {{ columns.dbt_scd_id }}, + 'True' as {{ columns.dbt_is_deleted }} + from snapshotted_data + left join deletes_source_data source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + and not ( + --avoid inserting a new record if the latest one is marked as deleted + snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + and + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + ) + ) + {%- endif %} + + select * from insertions + union all + select * from updates + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + union all + select * from deletes + {%- endif %} + {%- if strategy.hard_deletes == 'new_record' %} + union all + select * from deletion_records + {%- endif %} + +{%- endmacro %} + + \ No newline at end of file diff --git a/dbt/macros/dbt_overrides/unique_key_join_on.sql b/dbt/macros/dbt_overrides/unique_key_join_on.sql new file mode 100644 index 0000000..d0546ec --- /dev/null +++ b/dbt/macros/dbt_overrides/unique_key_join_on.sql @@ -0,0 +1,14 @@ +{% macro unique_key_join_on(unique_key, identifier, from_identifier) %} + {% if unique_key | is_list %} + {% for key in unique_key %} + {% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {# Use Oracle-specific NULL-safe equality matching oracle__equals macro #} + ({{ source_unique_key }} = {{ target_unique_key }} OR ({{ source_unique_key }} IS NULL AND {{ target_unique_key }} IS NULL)) + {%- if not loop.last %} and {%- endif %} + {% endfor %} + {% else %} + {# Use Oracle-specific NULL-safe equality for single unique_key as well #} + ({{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key OR ({{ identifier }}.dbt_unique_key IS NULL AND {{ from_identifier }}.dbt_unique_key IS NULL)) + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/error_handlers/check_null.sql b/dbt/macros/error_handlers/check_null.sql new file mode 100644 index 0000000..a30b2ea --- /dev/null +++ b/dbt/macros/error_handlers/check_null.sql @@ -0,0 +1,7 @@ +{% macro check_null(var_name, var_value) %} + {% if var_value == None or var_value | string == '' %} + {{ elog("Macro local variable " ~ var_name ~ " is empty!", "ERR", 0)}} + {{ exceptions.raise_compiler_error("Variable " ~ var_name ~ " is empty!") }} + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/error_handlers/check_variable.sql b/dbt/macros/error_handlers/check_variable.sql new file mode 100644 index 0000000..03d5875 --- /dev/null +++ b/dbt/macros/error_handlers/check_variable.sql @@ -0,0 +1,7 @@ +{% macro check_variable(var_name) %} + + {% if var(var_name, none) is none %} + {{ exceptions.raise_compiler_error("The required global variable " ~ var_name ~ " hasn't been provided!") }} + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/get_main_task_name.sql b/dbt/macros/get_main_task_name.sql new file mode 100644 index 0000000..78235f6 --- /dev/null +++ b/dbt/macros/get_main_task_name.sql @@ -0,0 +1,5 @@ +{% macro get_main_task_name(source_model_name) %} + {# return({{ source_model_name | split('__')[-1] }}) #} + {% set ret = "substr('" ~ source_model_name ~ "', 1, instr('" ~ source_model_name ~ "', '_', -1) -1)" %} + {{ return(ret) }} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/has_tag.sql b/dbt/macros/has_tag.sql new file mode 100644 index 0000000..862e9b9 --- /dev/null +++ b/dbt/macros/has_tag.sql @@ -0,0 +1,7 @@ +{% macro has_tag(tag_name, tag_to_run) %} + {% if tag_name == tag_to_run %} + {{ return(true) }} + {% else %} + {{ return(false) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/logging/elog.sql b/dbt/macros/logging/elog.sql new file mode 100644 index 0000000..f1bddea --- /dev/null +++ b/dbt/macros/logging/elog.sql @@ -0,0 +1,23 @@ +{% macro elog(message, level, depth) %} +{% if execute %} + {% set valid_levels = ["INFO", "WARN", "ERR"] %} + {% if level not in valid_levels %} + {{ exceptions.raise_compiler_error("Invalid log level: " ~ level ~ ". Allowed values: " ~ valid_levels | join(", ")) }} + {% endif %} + + {% if not depth is number or depth < 0 %} + {{ exception.raise_compiler_error("Depth must be a non-negatibe integer!") }} + {% endif %} + + {% if level == "INFO" %} + {% set start_msg = "\033[0m [INFO]: " ~ "*" * depth * 2 %} + {% elif level == "WARN" %} + {% set start_msg = "\033[33m [WARN]: " ~ "*" * depth * 2 %} + {% elif level == "ERR" %} + {% set start_msg = "\033[31m [ERROR]: " ~ "*" * depth * 2 %} + {% endif %} + + {{ log(start_msg | trim ~ " " ~ message, info=True)}} + +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/m_current_timestamp.sql b/dbt/macros/m_current_timestamp.sql new file mode 100644 index 0000000..fe1c3b4 --- /dev/null +++ b/dbt/macros/m_current_timestamp.sql @@ -0,0 +1,22 @@ +{% macro m_current_timestamp(type) %} + +{% if type | lower == 'db' %} + + {% if execute %} + + {% set get_timestamp %} + SELECT to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS') FROM DUAL + {% endset %} + + {% set result = run_query(get_timestamp) %} + {% set current_timestamp = result.columns[0].values()[0] %} + + {% endif %} + +{% else %} + {% set current_timestamp = modules.datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") %} +{% endif %} + +{{ return(current_timestamp) }} + +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/materialization/table_if_rows_exist.sql b/dbt/macros/materialization/table_if_rows_exist.sql new file mode 100644 index 0000000..389c25a --- /dev/null +++ b/dbt/macros/materialization/table_if_rows_exist.sql @@ -0,0 +1,64 @@ +{% materialization table_if_rows_exist, adapter='oracle' %} + + {%- set target_relation = this -%} + + -- Count the number of rows in the source table + {#-- Use the compiled SQL of the model to count rows --#} + {%- set count_query %} + select count(*) as row_count from ( + {{ model.compiled_sql }} + ) + {%- endset %} + + {#-- Execute the count query --#} + {%- set result = run_query(count_query) -%} + {%- set num_rows = result.columns[0].values()[0]|int -%} + + {#-- Proceed only if there are rows --#} + {%- if num_rows > 0 -%} + {{ log(num_rows ~ ' rows found in source data. Executing model.', info=True) }} + + {#-- collect hooks --#} + {% set pre_hooks = control_model_start() %} + {% set post_hooks = control_model_end('TRG') %} + + {#-- run pre_hooks --#} + {% do run_hooks(pre_hooks, inside_transaction=True) %} + + {#-- Define a temporary relation with a unique name --#} + {%- set tmp_relation = api.Relation.create( + schema=target_relation.schema, + identifier=target_relation.identifier + '_tmp' + ) %} + + {#-- Create a temporary table with the model's SQL --#} + {% set ctas_sql = 'CREATE TABLE ' ~ tmp_relation ~ ' AS ' ~ model.compiled_sql %} + {% call statement('main') %} + {{ ctas_sql }} + {% endcall %} + + {#-- Drop the existing table if it exists --#} + {%- if adapter.get_relation(target_relation.database, target_relation.schema, target_relation.identifier) is not none -%} + {{ log('Dropping target table ' ~ target_relation.identifier, info=True) }} + {% do run_query('DROP TABLE ' ~ target_relation.schema ~ '.' ~ target_relation.identifier) %} + {%- endif -%} + + {#-- Rename the temporary table to the target name --#} + {{ log('Renaming temporary table to target table', info=True) }} + {% do run_query('ALTER TABLE ' ~ tmp_relation ~ ' RENAME TO ' ~ target_relation.identifier) %} + + {#-- run post_hooks --#} + {% do run_hooks(post_hooks, inside_transaction=True) %} + {%- else -%} + {#-- Do nothing; leave the target table untouched --#} + {{ log('No rows in source data. Skipping model execution.', info=True) }} + {% call statement('main') %} + SELECT 1 FROM DUAL + {% endcall %} + {%- endif -%} + + {#-- Return the target relation to dbt --#} + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} + \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_default.sql b/dbt/macros/mopdb/get_sql_t_collateral_default.sql new file mode 100644 index 0000000..356097d --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_default.sql @@ -0,0 +1,268 @@ +{% macro get_sql_t_collateral_default() %} + -- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} + select 'MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, -- to find + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, -- to find + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + ISSUER_CAS AS ISSUER_CRED_ASSESS_SOURCE, + ISSUER_RATING_ENUM_VALUE, + ISSUER_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(18 CHAR)) AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, -- to find + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + ISSUER_CRED_PROVIDER AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} + WHERE A_WORKFLOW_HISTORY_KEY IS NULL + + UNION + + select 'NON_MARKET_ASSET' AS DATASET, + CAST (NULL AS DATE) AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS, + ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR, + DEBTOR_NAME, + DEBTOR_ID, + DEBTOR_TYPE AS DEBTOR_GROUP, + DEBTOR_CLASS, + DEBTOR_RESIDENCE, + DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE, + DEBTOR_RATING_ENUM_VALUE, + DEBTOR_RATING_NUMBER_VALUE, + GUAR_NAME AS GUARANTOR_NAME, + GUAR_ID AS GUARANTOR_ID, + GUAR_RESIDENCE AS GUARANTOR_RESIDENCE, + GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE, + GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE, + GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE, + NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + MATURITY_DATE, + GUAR_CLASS AS GUARANTOR_CLASS, + CAST (NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + INTEREST_PAYMENT_TYPE, + RESIDUAL_MATURITY, + BUCKET_SIZE, + DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER, + GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + SECURED_FLAG AS SECURED, + DENOMINATION AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT, + DEBTOR_ID_TYPE AS DEBTOR_IDTYPE, + CAP, + REFERENCE_RATE, + REFERENCE_RATE_COMMENT, + RESET_PERIOD_MORE_ONE_YEAR, + OA_ID, + CONTRACT_ID, + INSTRMNT_ID, + 'C2D_ACC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }} + WHERE A_WORKFLOW_HISTORY_KEY IS NULL + + UNION + + select 'NON_MARKET_DECC' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + NOM_AMT_SUBMITTED AS COLL_VALUE_BEFORE_HAIRCUTS, + NOM_AMT_SUBMITTED AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, -- to find + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, -- check + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, -- check + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, -- to find + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, --check + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, -- check + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, -- check + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, -- check + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, -- check + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} + WHERE A_WORKFLOW_HISTORY_KEY IS NULL + + UNION + + select 'NON_MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS, + ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR, + DEBTOR_NAME, + DEBTOR_ID, + DEBTOR_TYPE AS DEBTOR_GROUP, + DEBTOR_CLASS, + DEBTOR_RESIDENCE, + DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE, + DEBTOR_RATING_ENUM_VALUE, + DEBTOR_RATING_NUMBER_VALUE, + GUAR_NAME AS GUARANTOR_NAME, + GUAR_ID AS GUARANTOR_ID, + GUAR_RESIDENCE AS GUARANTOR_RESIDENCE, + GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE, + GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE, + GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE, + NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + MATURITY_DATE, + GUAR_CLASS AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + INTEREST_PAYMENT_TYPE, + RESIDUAL_MATURITY, + BUCKET_SIZE, + DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER, + GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + SECURED_FLAG AS SECURED, + DENOMINATION AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT, + DEBTOR_ID_TYPE AS DEBTOR_IDTYPE, + CAP, + REFERENCE_RATE, + REFERENCE_RATE_COMMENT, + RESET_PERIOD_MORE_ONE_YEAR, + OA_ID, + CONTRACT_ID, + INSTRMNT_ID, + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} + WHERE A_WORKFLOW_HISTORY_KEY IS NULL +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_ma_dissem.sql b/dbt/macros/mopdb/get_sql_t_collateral_ma_dissem.sql new file mode 100644 index 0000000..7d45d81 --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_ma_dissem.sql @@ -0,0 +1,100 @@ +{% macro get_sql_t_collateral_ma_dissem() %} + -- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} + select 'MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, -- to find + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, -- to find + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + ISSUER_CAS AS ISSUER_CRED_ASSESS_SOURCE, + ISSUER_RATING_ENUM_VALUE, + ISSUER_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(18 CHAR)) AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, -- to find + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + ISSUER_CRED_PROVIDER AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} + WHERE (REPORTING_NCB, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ this }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ') }} + ) + AND DATASET = 'MARKET_ASSET' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_nma_acc.sql b/dbt/macros/mopdb/get_sql_t_collateral_nma_acc.sql new file mode 100644 index 0000000..ef7a330 --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_nma_acc.sql @@ -0,0 +1,71 @@ +{% macro get_sql_t_collateral_nma_acc() %} + -- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }} + + select 'NON_MARKET_ASSET' AS DATASET, + CAST (NULL AS DATE) AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST (NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS, + ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR, + DEBTOR_NAME, + DEBTOR_ID, + DEBTOR_TYPE AS DEBTOR_GROUP, + DEBTOR_CLASS, + DEBTOR_RESIDENCE, + DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE, + DEBTOR_RATING_ENUM_VALUE, + DEBTOR_RATING_NUMBER_VALUE, + GUAR_NAME AS GUARANTOR_NAME, + GUAR_ID AS GUARANTOR_ID, + GUAR_RESIDENCE AS GUARANTOR_RESIDENCE, + GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE, + GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE, + GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE, + NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + MATURITY_DATE, + GUAR_CLASS AS GUARANTOR_CLASS, + CAST (NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + INTEREST_PAYMENT_TYPE, + RESIDUAL_MATURITY, + BUCKET_SIZE, + DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER, + GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + SECURED_FLAG AS SECURED, + DENOMINATION AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT, + DEBTOR_ID_TYPE AS DEBTOR_IDTYPE, + CAP, + REFERENCE_RATE, + REFERENCE_RATE_COMMENT, + RESET_PERIOD_MORE_ONE_YEAR, + OA_ID, + CONTRACT_ID, + INSTRMNT_ID, + 'C2D_ACC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }} + WHERE (reporting_ncb, SNAPSHOT_DATE) IN ( + SELECT DISTINCT reporting_ncb, snapshot_date + FROM {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ') }}) + AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_nma_decc_dissem.sql b/dbt/macros/mopdb/get_sql_t_collateral_nma_decc_dissem.sql new file mode 100644 index 0000000..9e4e97b --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_nma_decc_dissem.sql @@ -0,0 +1,101 @@ +{% macro get_sql_t_collateral_nma_decc_dissem() %} + -- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} + + select 'NON_MARKET_DECC' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + NOM_AMT_SUBMITTED AS COLL_VALUE_BEFORE_HAIRCUTS, + NOM_AMT_SUBMITTED AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, -- to find + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, -- check + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, -- check + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, -- to find + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, --check + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, -- check + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, -- check + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, -- check + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, -- check + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} + WHERE (reporting_ncb, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ this }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} + ) + AND DATASET = 'NON_MARKET_DECC' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_nma_dissem.sql b/dbt/macros/mopdb/get_sql_t_collateral_nma_dissem.sql new file mode 100644 index 0000000..da145ec --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_nma_dissem.sql @@ -0,0 +1,106 @@ +{% macro get_sql_t_collateral_nma_dissem() %} + -- depends_on: {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} + + select 'NON_MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS, + ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR, + DEBTOR_NAME, + DEBTOR_ID, + DEBTOR_TYPE AS DEBTOR_GROUP, + DEBTOR_CLASS, + DEBTOR_RESIDENCE, + DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE, + DEBTOR_RATING_ENUM_VALUE, + DEBTOR_RATING_NUMBER_VALUE, + GUAR_NAME AS GUARANTOR_NAME, + GUAR_ID AS GUARANTOR_ID, + GUAR_RESIDENCE AS GUARANTOR_RESIDENCE, + GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE, + GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE, + GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE, + NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + MATURITY_DATE, + GUAR_CLASS AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + INTEREST_PAYMENT_TYPE, + RESIDUAL_MATURITY, + BUCKET_SIZE, + DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER, + GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + SECURED_FLAG AS SECURED, + DENOMINATION AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT, + DEBTOR_ID_TYPE AS DEBTOR_IDTYPE, + CAP, + REFERENCE_RATE, + REFERENCE_RATE_COMMENT, + RESET_PERIOD_MORE_ONE_YEAR, + OA_ID, + CONTRACT_ID, + INSTRMNT_ID, + 'C2D_UC' AS SOURCE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + from {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} + WHERE (reporting_ncb, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ this }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ ref('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ') }} + ) + AND DATASET = 'NON_MARKET_ASSET' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + AND A_WORKFLOW_HISTORY_KEY = MAX_A_WORKFLOW_HISTORY_KEY + AND (reporting_ncb, SNAPSHOT_DATE) NOT IN ( + SELECT DISTINCT REPORTING_NCB, SNAPSHOT_DATE + FROM {{ this }} + WHERE SOURCE = 'C2D_ACC' + ) +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/mopdb/get_sql_t_collateral_prehook.sql b/dbt/macros/mopdb/get_sql_t_collateral_prehook.sql new file mode 100644 index 0000000..94b12e9 --- /dev/null +++ b/dbt/macros/mopdb/get_sql_t_collateral_prehook.sql @@ -0,0 +1,126 @@ +{% macro get_sql_t_collateral_prehook( model, source_model ) %} + {% if var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM' %} + + DELETE FROM {{ model }} + WHERE DATASET = 'MARKET_ASSET' + AND SOURCE = 'C2D_UC' + AND (NCB, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ source_model }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ model }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ source_model }} + ) + AND DATASET = 'MARKET_ASSET' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + + {% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC' %} + + DELETE FROM {{ model }} + WHERE DATASET = 'NON_MARKET_ASSET' + AND (NCB, SNAPSHOT_DATE) IN ( + SELECT DISTINCT reporting_ncb, snapshot_date + FROM {{ source_model }} + + {% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM' %} + + DELETE FROM {{ model }} + WHERE DATASET = 'NON_MARKET_DECC' + AND SOURCE = 'C2D_UC' + AND (NCB, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ source_model }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ model }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ source_model }} + ) + AND DATASET = 'NON_MARKET_DECC' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + {% elif var('tag_to_run') == 'm_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM' %} + + DELETE FROM {{ model }} + WHERE DATASET = 'NON_MARKET_ASSET' + AND SOURCE = 'C2D_UC' + AND (NCB, SNAPSHOT_DATE) in ( + SELECT DISTINCT + src.reporting_ncb, + src.snapshot_date + FROM + {{ source_model }} src + full outer join (SELECT DISTINCT + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + FROM + {{ model }} + WHERE + ( ncb, + snapshot_date ) IN ( + SELECT DISTINCT + reporting_ncb, + snapshot_date + FROM + {{ source_model }} + ) + AND DATASET = 'NON_MARKET_ASSET' + AND SOURCE = 'C2D_UC' + GROUP BY + ncb, + snapshot_date, + DATABASE_ENTRY_TIMESTAMP + ) trg on src.reporting_ncb = trg.ncb + and TRUNC(src.snapshot_date) = TRUNC(trg.SNAPSHOT_DATE) + WHERE + src.max_a_workflow_history_key = src.a_workflow_history_key + and src.FILE_CREATION_DATE > NVL(trg.DATABASE_ENTRY_TIMESTAMP,src.FILE_CREATION_DATE - 1) + ) + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/macros/oracle/list_relations_without_caching.sql b/dbt/macros/oracle/list_relations_without_caching.sql new file mode 100644 index 0000000..164950b --- /dev/null +++ b/dbt/macros/oracle/list_relations_without_caching.sql @@ -0,0 +1,46 @@ +# +# This macro overwrites the macro with the same name in the dbt-oracle adapter, which was found in +# /lib/python3.11/site-packages/dbt/include/oracle/macros/adapters.sql +# Reason: The SQL query in the original macro had severe performance issues when accessing sys.all_views +# +{% macro oracle__list_relations_without_caching(schema_relation) %} + {% call statement('list_relations_without_caching', fetch_result=True) -%} + with tables as + (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, + owner table_schema, + table_name, + case + when iot_type = 'Y' + then 'IOT' + when temporary = 'Y' + then 'TEMP' + else 'BASE TABLE' + end table_type + from sys.all_tables + where upper(table_name) not in ( + select upper(mview_name) + from sys.all_mviews + where upper(owner) = upper('{{ schema_relation.schema }}') + ) + union all + select SYS_CONTEXT('userenv', 'DB_NAME'), + owner, + object_name, + object_type + from sys.all_objects + WHERE object_type in ('VIEW', 'MATERIALIZED VIEW') + ) + select table_catalog as "database_name" + ,table_name as "name" + ,table_schema as "schema_name" + ,case table_type + when 'BASE TABLE' then 'table' + when 'VIEW' then 'view' + when 'MATERIALIZED VIEW' then 'materialized_view' + end as "kind" + from tables + where table_type in ('BASE TABLE', 'VIEW', 'MATERIALIZED VIEW') + and upper(table_schema) = upper('{{ schema_relation.schema }}') + {% endcall %} + {{ return(load_result('list_relations_without_caching').table) }} +{% endmacro %} diff --git a/dbt/models/control_tables/_sources.yml b/dbt/models/control_tables/_sources.yml new file mode 100644 index 0000000..caf5ad5 --- /dev/null +++ b/dbt/models/control_tables/_sources.yml @@ -0,0 +1,38 @@ +version: 2 + +sources: + - name: control_tables + schema: ct_mrds + tables: + - name: A_WORKFLOW_HISTORY + identifier: A_WORKFLOW_HISTORY + + - name: A_TASK_HISTORY + identifier: A_TASK_HISTORY + + - name: A_TASK_HISTORY_SOURCE + identifier: A_TASK_HISTORY_SOURCE + + - name: A_TASK_HISTORY_TARGET + identifier: A_TASK_HISTORY_TARGET + + - name: A_MOPDB_LOAD_HISTORY + identifier: A_MOPDB_LOAD_HISTORY + + - name: A_DWH_LOAD_HISTORY + identifier: A_DWH_LOAD_HISTORY + + - name: control_sequences + schema: ct_mrds + tables: + - name: A_WORKFLOW_HISTORY_SEQ + identifier: A_WORKFLOW_HISTORY_KEY_SEQ + + - name: A_TASK_HISTORY_SEQ + identifier: A_TASK_HISTORY_KEY_SEQ + + - name: A_TASK_HISTORY_SOURCE_SEQ + identifier: A_TASK_HISTORY_SOURCE_KEY_SEQ + + - name: A_TASK_HISTORY_TARGET_SEQ + identifier: A_TASK_HISTORY_TARGET_KEY_SEQ diff --git a/dbt/models/example/my_first_dbt_model.sql b/dbt/models/example/my_first_dbt_model.sql new file mode 100644 index 0000000..f31a12d --- /dev/null +++ b/dbt/models/example/my_first_dbt_model.sql @@ -0,0 +1,27 @@ + +/* + Welcome to your first dbt model! + Did you know that you can also configure models directly within SQL files? + This will override configurations stated in dbt_project.yml + + Try changing "table" to "view" below +*/ + +{{ config(materialized='table') }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +/* + Uncomment the line below to remove records with null `id` values +*/ + +-- where id is not null diff --git a/dbt/models/example/my_second_dbt_model.sql b/dbt/models/example/my_second_dbt_model.sql new file mode 100644 index 0000000..c91f879 --- /dev/null +++ b/dbt/models/example/my_second_dbt_model.sql @@ -0,0 +1,6 @@ + +-- Use the `ref` function to select from other models + +select * +from {{ ref('my_first_dbt_model') }} +where id = 1 diff --git a/dbt/models/example/schema.yml b/dbt/models/example/schema.yml new file mode 100644 index 0000000..2a53081 --- /dev/null +++ b/dbt/models/example/schema.yml @@ -0,0 +1,21 @@ + +version: 2 + +models: + - name: my_first_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + tests: + - unique + - not_null diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ.sql new file mode 100644 index 0000000..e31daaa --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING"], + alias = "T_CEPH_SQ", + schema = "C2D", + meta={ + "author": "Shane Dalisay", + "created": "2026-01-13", + "last_updated_by": "Shane Dalisay", + "last_updated": "2026-01-13 - ticket MARS-193" + } +) }} + +SELECT DISTINCT + /*+ PARALLEL(OU_CEPH_PRICING,4) */ + OU_CEPH_PRICING.A_KEY, + OU_CEPH_PRICING.A_WORKFLOW_HISTORY_KEY, + CAST(OU_CEPH_PRICING.PRICE_DATE AS TIMESTAMP(6)) AS PRICE_DATE, + CAST(OU_CEPH_PRICING.ISIN_CODE AS VARCHAR2(200 BYTE)) AS ISIN_CODE, + CAST(OU_CEPH_PRICING.PRICE AS NUMBER(28,10)) AS PRICE, + CAST(OU_CEPH_PRICING.WARNING_CODE AS VARCHAR2(20 BYTE)) AS WARNING_CODE, + CAST(OU_CEPH_PRICING.ACCRUED_INTEREST AS NUMBER(28,10)) AS ACCRUED_INTEREST, + CAST(OU_CEPH_PRICING.POOL_FACTOR AS NUMBER(28,10)) AS POOL_FACTOR, + CAST(OU_CEPH_PRICING.PRICE_NATURE AS VARCHAR2(20 BYTE)) AS PRICE_NATURE, + CAST(OU_CEPH_PRICING.WAL AS NUMBER(28,10)) AS WAL, + CAST(OU_CEPH_PRICING.CLEAN_PRICE_WO_MARKDOWN AS NUMBER(28,10)) AS CLEAN_PRICE_WO_MARKDOWN, + CAST(OU_CEPH_PRICING.ACCRUED_INTEREST_WO_MARKDOWN AS NUMBER(28,10)) AS ACCRUED_INTEREST_WO_MARKDOWN, + CAST(OU_CEPH_PRICING.THEORETICAL_PRICE AS NUMBER(28,10)) AS THEORETICAL_PRICE +FROM + {{ source("ods","PRICING")}} OU_CEPH_PRICING +WHERE + OU_CEPH_PRICING.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_CEPH_PRICING", + get_main_task_name(model.name) + ) + }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ.sql new file mode 100644 index 0000000..c8097a6 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM_SQ.sql @@ -0,0 +1,41 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM"], + alias='UC_MA_DISSEM_SQ', + schema='C2D' +) }} + +SELECT /*+ PARALLEL(OU_C2D_UC_MA_DISSEM,4) */ +MAX(OU_C2D_UC_MA_DISSEM.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_C2D_UC_MA_DISSEM.REPORTING_NCB,OU_C2D_UC_MA_DISSEM.SNAPSHOT_DATE ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_MA_DISSEM.A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_MA_DISSEM.REPORTING_NCB, +OU_C2D_UC_MA_DISSEM.SNAPSHOT_DATE, +OU_C2D_UC_MA_DISSEM.FILE_CREATION_DATE, +OU_C2D_UC_MA_DISSEM.MFI_ID, +OU_C2D_UC_MA_DISSEM.ISIN_CODE, +OU_C2D_UC_MA_DISSEM.OTHER_REG_NO, +OU_C2D_UC_MA_DISSEM.NOM_AMT_SUBMITTED, +OU_C2D_UC_MA_DISSEM.COLL_BEFORE_HAIRCUTS, +OU_C2D_UC_MA_DISSEM.COLL_AFTER_HAIRCUTS, +OU_C2D_UC_MA_DISSEM.TYPE_OF_SYSTEM, +OU_C2D_UC_MA_DISSEM.TYPE_OF_OPERATION, +OU_C2D_UC_MA_DISSEM.DOM_OR_XBORDER, +OU_C2D_UC_MA_DISSEM.ISSUER_CAS, +OU_C2D_UC_MA_DISSEM.ISSUER_CRED_PROVIDER, +OU_C2D_UC_MA_DISSEM.ISSUER_CLASS, +OU_C2D_UC_MA_DISSEM.ISSUER_RATING_ENUM_VALUE, +OU_C2D_UC_MA_DISSEM.ISSUER_RATING_NUMBER_VALUE, +OU_C2D_UC_MA_DISSEM.NCB_COMMENT, +OU_C2D_UC_MA_DISSEM.MOBILISATION_CHANNEL, +OU_C2D_UC_MA_DISSEM.CCB, +OU_C2D_UC_MA_DISSEM.INVESTOR_SSS, +OU_C2D_UC_MA_DISSEM.INTERMEDIARY_SSS, +OU_C2D_UC_MA_DISSEM.ISSUER_SSS, +OU_C2D_UC_MA_DISSEM.TRIPARTY_AGENT, +OU_C2D_UC_MA_DISSEM.SUSPECT_ID, +OU_C2D_UC_MA_DISSEM.QUALITY_CHECK_STATUS, +OU_C2D_UC_MA_DISSEM.ERROR_CODE, +OU_C2D_UC_MA_DISSEM.ERROR_MESSAGE, +OU_C2D_UC_MA_DISSEM.ERROR_POSITION_IN_FILE +FROM +{{ source('ou_c2d','UC_MA_DISSEM') }} OU_C2D_UC_MA_DISSEM +WHERE A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key('w_OU_C2D_UC_DISSEM', get_main_task_name(model.name)) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ.sql new file mode 100644 index 0000000..10c528e --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC_SQ.sql @@ -0,0 +1,72 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC"], + alias='UC_NMA_ACC_SQ', + schema='C2D' +) }} + +SELECT /*+ PARALLEL(OU_C2D_UC_NMA_ACC,4) */ +MAX(OU_C2D_UC_NMA_ACC.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_C2D_UC_NMA_ACC.REPORTING_NCB,OU_C2D_UC_NMA_ACC.SNAPSHOT_DATE ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_ACC.A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_ACC.REPORTING_NCB, +OU_C2D_UC_NMA_ACC.SNAPSHOT_DATE, +OU_C2D_UC_NMA_ACC.MFI_ID, +OU_C2D_UC_NMA_ACC.OTHER_REG_NO, +OU_C2D_UC_NMA_ACC.TYPE_OF_SYSTEM, +OU_C2D_UC_NMA_ACC.TYPE_OF_OPERATION, +OU_C2D_UC_NMA_ACC.DOM_OR_XBORDER, +OU_C2D_UC_NMA_ACC.NON_MKT_ASSET_TYPE, +OU_C2D_UC_NMA_ACC.MATURITY_DATE, +OU_C2D_UC_NMA_ACC.INTEREST_PAYMENT_TYPE, +OU_C2D_UC_NMA_ACC.CAP, +OU_C2D_UC_NMA_ACC.REFERENCE_RATE, +OU_C2D_UC_NMA_ACC.REFERENCE_RATE_COMMENT, +OU_C2D_UC_NMA_ACC.COLL_BEFORE_HAIRCUTS, +OU_C2D_UC_NMA_ACC.COLL_AFTER_HAIRCUTS, +OU_C2D_UC_NMA_ACC.NO_AGGR_DEBTORS, +OU_C2D_UC_NMA_ACC.ELIGIBLE_VIA_GUAR, +OU_C2D_UC_NMA_ACC.DEBTOR_TYPE, +OU_C2D_UC_NMA_ACC.DEBTOR_NAME, +OU_C2D_UC_NMA_ACC.DEBTOR_ID_TYPE, +OU_C2D_UC_NMA_ACC.DEBTOR_ID, +OU_C2D_UC_NMA_ACC.DEBTOR_CLASS, +OU_C2D_UC_NMA_ACC.DEBTOR_RESIDENCE, +OU_C2D_UC_NMA_ACC.DEBTOR_CAS, +OU_C2D_UC_NMA_ACC.DEBTOR_CRED_PROV, +OU_C2D_UC_NMA_ACC.DEBTOR_RATING_ENUM_VALUE, +OU_C2D_UC_NMA_ACC.DEBTOR_RATING_NUMBER_VALUE, +OU_C2D_UC_NMA_ACC.GUAR_TYPE, +OU_C2D_UC_NMA_ACC.GUAR_NAME, +OU_C2D_UC_NMA_ACC.GUAR_ID_TYPE, +OU_C2D_UC_NMA_ACC.GUAR_ID, +OU_C2D_UC_NMA_ACC.GUAR_CLASS, +OU_C2D_UC_NMA_ACC.GUAR_RESIDENCE, +OU_C2D_UC_NMA_ACC.GUAR_CRED_CAS, +OU_C2D_UC_NMA_ACC.GUAR_CRED_PROV, +OU_C2D_UC_NMA_ACC.GUAR_RATING_ENUM_VALUE, +OU_C2D_UC_NMA_ACC.GUAR_RATING_NUMBER_VALUE, +OU_C2D_UC_NMA_ACC.NO_AGGR_ASSETS, +OU_C2D_UC_NMA_ACC.DENOMINATION, +OU_C2D_UC_NMA_ACC.SECURED_FLAG, +OU_C2D_UC_NMA_ACC.RESIDUAL_MATURITY, +OU_C2D_UC_NMA_ACC.BUCKET_SIZE, +OU_C2D_UC_NMA_ACC.NCB_COMMENT, +OU_C2D_UC_NMA_ACC.VALUATION_METHODOLOGY, +OU_C2D_UC_NMA_ACC.NOM_AMT_SUBMITTED, +OU_C2D_UC_NMA_ACC.RESET_PERIOD_MORE_ONE_YEAR, +OU_C2D_UC_NMA_ACC.MOBILISATION_CHANNEL, +OU_C2D_UC_NMA_ACC.CCB, +OU_C2D_UC_NMA_ACC.INVESTOR_SSS, +OU_C2D_UC_NMA_ACC.INTERMEDIARY_SSS, +OU_C2D_UC_NMA_ACC.ISSUER_SSS, +OU_C2D_UC_NMA_ACC.OA_ID, +OU_C2D_UC_NMA_ACC.CONTRACT_ID, +OU_C2D_UC_NMA_ACC.INSTRMNT_ID +FROM +{{ source('ou_c2d','UC_NMA_ACC') }} OU_C2D_UC_NMA_ACC +WHERE A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key('w_ODS_C2D_UC_ACC', get_main_task_name(model.name)) }} + + + + + + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql new file mode 100644 index 0000000..0dd5086 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql @@ -0,0 +1,40 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM"], + alias='UC_NMA_DECC_DISSEM_SQ', + schema='C2D' +) }} + +SELECT /*+ PARALLEL(OU_C2D_UC_NMA_DECC_DISSEM,4) */ +MAX(OU_C2D_UC_NMA_DECC_DISSEM.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_C2D_UC_NMA_DECC_DISSEM.REPORTING_NCB,OU_C2D_UC_NMA_DECC_DISSEM.SNAPSHOT_DATE ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_DECC_DISSEM.A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_DECC_DISSEM.REPORTING_NCB, +OU_C2D_UC_NMA_DECC_DISSEM.SNAPSHOT_DATE, +OU_C2D_UC_NMA_DECC_DISSEM.FILE_CREATION_DATE, +OU_C2D_UC_NMA_DECC_DISSEM.MFI_ID, +OU_C2D_UC_NMA_DECC_DISSEM.ISIN_CODE, +OU_C2D_UC_NMA_DECC_DISSEM.NOM_AMT_SUBMITTED, +OU_C2D_UC_NMA_DECC_DISSEM.TYPE_OF_SYSTEM, +OU_C2D_UC_NMA_DECC_DISSEM.TYPE_OF_OPERATION, +OU_C2D_UC_NMA_DECC_DISSEM.DOM_OR_XBORDER, +OU_C2D_UC_NMA_DECC_DISSEM.NON_MKT_ASSET_TYPE, +OU_C2D_UC_NMA_DECC_DISSEM.NCB_COMMENT, +OU_C2D_UC_NMA_DECC_DISSEM.MOBILISATION_CHANNEL, +OU_C2D_UC_NMA_DECC_DISSEM.CCB, +OU_C2D_UC_NMA_DECC_DISSEM.INVESTOR_SSS, +OU_C2D_UC_NMA_DECC_DISSEM.INTERMEDIARY_SSS, +OU_C2D_UC_NMA_DECC_DISSEM.ISSUER_SSS, +OU_C2D_UC_NMA_DECC_DISSEM.TRIPARTY_AGENT, +OU_C2D_UC_NMA_DECC_DISSEM.SUSPECT_ID, +OU_C2D_UC_NMA_DECC_DISSEM.QUALITY_CHECK_STATUS, +OU_C2D_UC_NMA_DECC_DISSEM.ERROR_CODE, +OU_C2D_UC_NMA_DECC_DISSEM.ERROR_MESSAGE, +OU_C2D_UC_NMA_DECC_DISSEM.ERROR_POSITION_IN_FILE +FROM +{{ source('ou_c2d','UC_NMA_DECC_DISSEM') }} OU_C2D_UC_NMA_DECC_DISSEM +WHERE A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key('w_ODS_C2D_UC_DISSEMI', get_main_task_name(model.name)) }} + + + + + + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ.sql new file mode 100644 index 0000000..6657c6e --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM_SQ.sql @@ -0,0 +1,77 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM"], + alias='UC_NMA_DISSEM_SQ', + schema='C2D' +) }} + +SELECT /*+ PARALLEL(OU_C2D_UC_NMA_DISSEM,4) */ +MAX(OU_C2D_UC_NMA_DISSEM.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_C2D_UC_NMA_DISSEM.REPORTING_NCB,OU_C2D_UC_NMA_DISSEM.SNAPSHOT_DATE ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_DISSEM.A_WORKFLOW_HISTORY_KEY, +OU_C2D_UC_NMA_DISSEM.REPORTING_NCB, +OU_C2D_UC_NMA_DISSEM.SNAPSHOT_DATE, +OU_C2D_UC_NMA_DISSEM.FILE_CREATION_DATE, +OU_C2D_UC_NMA_DISSEM.MFI_ID, +OU_C2D_UC_NMA_DISSEM.OTHER_REG_NO, +OU_C2D_UC_NMA_DISSEM.TYPE_OF_SYSTEM, +OU_C2D_UC_NMA_DISSEM.TYPE_OF_OPERATION, +OU_C2D_UC_NMA_DISSEM.DOM_OR_XBORDER, +OU_C2D_UC_NMA_DISSEM.NON_MKT_ASSET_TYPE, +OU_C2D_UC_NMA_DISSEM.MATURITY_DATE, +OU_C2D_UC_NMA_DISSEM.INTEREST_PAYMENT_TYPE, +OU_C2D_UC_NMA_DISSEM.CAP, +OU_C2D_UC_NMA_DISSEM.REFERENCE_RATE, +OU_C2D_UC_NMA_DISSEM.REFERENCE_RATE_COMMENT, +OU_C2D_UC_NMA_DISSEM.COLL_BEFORE_HAIRCUTS, +OU_C2D_UC_NMA_DISSEM.COLL_AFTER_HAIRCUTS, +OU_C2D_UC_NMA_DISSEM.NO_AGGR_DEBTORS, +OU_C2D_UC_NMA_DISSEM.ELIGIBLE_VIA_GUAR, +OU_C2D_UC_NMA_DISSEM.DEBTOR_TYPE, +OU_C2D_UC_NMA_DISSEM.DEBTOR_NAME, +OU_C2D_UC_NMA_DISSEM.DEBTOR_ID_TYPE, +OU_C2D_UC_NMA_DISSEM.DEBTOR_ID, +OU_C2D_UC_NMA_DISSEM.DEBTOR_CLASS, +OU_C2D_UC_NMA_DISSEM.DEBTOR_RESIDENCE, +OU_C2D_UC_NMA_DISSEM.DEBTOR_CAS, +OU_C2D_UC_NMA_DISSEM.DEBTOR_CRED_PROV, +OU_C2D_UC_NMA_DISSEM.DEBTOR_RATING_ENUM_VALUE, +OU_C2D_UC_NMA_DISSEM.DEBTOR_RATING_NUMBER_VALUE, +OU_C2D_UC_NMA_DISSEM.GUAR_TYPE, +OU_C2D_UC_NMA_DISSEM.GUAR_NAME, +OU_C2D_UC_NMA_DISSEM.GUAR_ID_TYPE, +OU_C2D_UC_NMA_DISSEM.GUAR_ID, +OU_C2D_UC_NMA_DISSEM.GUAR_CLASS, +OU_C2D_UC_NMA_DISSEM.GUAR_RESIDENCE, +OU_C2D_UC_NMA_DISSEM.GUAR_CRED_CAS, +OU_C2D_UC_NMA_DISSEM.GUAR_CRED_PROV, +OU_C2D_UC_NMA_DISSEM.GUAR_RATING_ENUM_VALUE, +OU_C2D_UC_NMA_DISSEM.GUAR_RATING_NUMBER_VALUE, +OU_C2D_UC_NMA_DISSEM.NO_AGGR_ASSETS, +OU_C2D_UC_NMA_DISSEM.DENOMINATION, +OU_C2D_UC_NMA_DISSEM.SECURED_FLAG, +OU_C2D_UC_NMA_DISSEM.RESIDUAL_MATURITY, +OU_C2D_UC_NMA_DISSEM.BUCKET_SIZE, +OU_C2D_UC_NMA_DISSEM.NCB_COMMENT, +OU_C2D_UC_NMA_DISSEM.VALUATION_METHODOLOGY, +OU_C2D_UC_NMA_DISSEM.NOM_AMT_SUBMITTED, +OU_C2D_UC_NMA_DISSEM.RESET_PERIOD_MORE_ONE_YEAR, +OU_C2D_UC_NMA_DISSEM.MOBILISATION_CHANNEL, +OU_C2D_UC_NMA_DISSEM.CCB, +OU_C2D_UC_NMA_DISSEM.INVESTOR_SSS, +OU_C2D_UC_NMA_DISSEM.INTERMEDIARY_SSS, +OU_C2D_UC_NMA_DISSEM.ISSUER_SSS, +OU_C2D_UC_NMA_DISSEM.SUSPECT_ID, +OU_C2D_UC_NMA_DISSEM.QUALITY_CHECK_STATUS, +OU_C2D_UC_NMA_DISSEM.ERROR_CODE, +OU_C2D_UC_NMA_DISSEM.ERROR_MESSAGE, +OU_C2D_UC_NMA_DISSEM.ERROR_POSITION_IN_FILE, +OU_C2D_UC_NMA_DISSEM.OA_ID, +OU_C2D_UC_NMA_DISSEM.CONTRACT_ID, +OU_C2D_UC_NMA_DISSEM.INSTRMNT_ID +FROM +{{ source('ou_c2d','UC_NMA_DISSEM') }} OU_C2D_UC_NMA_DISSEM +WHERE A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key('w_ODS_C2D_UC_DISSEMI', get_main_task_name(model.name)) }} + + + + + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ.sql new file mode 100644 index 0000000..7597fe8 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ.sql @@ -0,0 +1,47 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_UC_AGGREGATED"], + alias='UC_AGG_MA_DISSEM_SQ', + schema='C2D' +) }} + +SELECT DISTINCT + A_WORKFLOW_HISTORY_KEY , + REPORTING_NCB , + SNAPSHOT_DATE , + FILE_CREATION_DATE , + MFI_ID , + ISIN_CODE , + OTHER_REG_NO , + NOM_AMT_SUBMITTED , + COLL_BEFORE_HAIRCUTS , + COLL_AFTER_HAIRCUTS , + TYPE_OF_SYSTEM , + TYPE_OF_OPERATION , + DOM_OR_XBORDER , + ISSUER_CAS , + ISSUER_CRED_PROVIDER , + ISSUER_CLASS , + ISSUER_RATING_ENUM_VALUE , + ISSUER_RATING_NUMBER_VALUE , + NCB_COMMENT , + MOBILISATION_CHANNEL , + CCB , + INVESTOR_SSS , + INTERMEDIARY_SSS , + ISSUER_SSS , + TRIPARTY_AGENT , + SUSPECT_ID , + QUALITY_CHECK_STATUS , + ERROR_CODE , + ERROR_MESSAGE , + ERROR_POSITION_IN_FILE + FROM + {{ source('ou_c2d','UC_MA_DISSEM') }} + WHERE (A_WORKFLOW_HISTORY_KEY,REPORTING_NCB) IN + (SELECT MAX(A_WORKFLOW_HISTORY_KEY), REPORTING_NCB + FROM {{ source('ou_c2d','UC_MA_DISSEM') }} + WHERE (SNAPSHOT_DATE, REPORTING_NCB) IN + (SELECT MAX(SNAPSHOT_DATE), REPORTING_NCB + FROM {{ source('ou_c2d','UC_MA_DISSEM') }} + GROUP BY REPORTING_NCB) + GROUP BY REPORTING_NCB) \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql new file mode 100644 index 0000000..dd72659 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ.sql @@ -0,0 +1,42 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_UC_AGGREGATED"], + alias='UC_AGG_NMA_DECC_DISSEM_SQ', + schema='C2D' +) }} + +SELECT DISTINCT + A_WORKFLOW_HISTORY_KEY, + REPORTING_NCB , + SNAPSHOT_DATE , + FILE_CREATION_DATE , + MFI_ID , + ISIN_CODE , + NOM_AMT_SUBMITTED , + TYPE_OF_SYSTEM , + TYPE_OF_OPERATION , + DOM_OR_XBORDER , + NON_MKT_ASSET_TYPE , + NCB_COMMENT , + MOBILISATION_CHANNEL , + CCB , + INVESTOR_SSS , + INTERMEDIARY_SSS , + ISSUER_SSS , + TRIPARTY_AGENT , + SUSPECT_ID , + QUALITY_CHECK_STATUS , + ERROR_CODE , + ERROR_MESSAGE , + ERROR_POSITION_IN_FILE +FROM + {{ source('ou_c2d','UC_NMA_DECC_DISSEM') }} OU_C2D_UC_NMA_DECC_DISSEM -- IU_ODB. +WHERE (A_WORKFLOW_HISTORY_KEY,REPORTING_NCB) IN + (SELECT MAX(A_WORKFLOW_HISTORY_KEY), + REPORTING_NCB + FROM {{ source('ou_c2d','UC_NMA_DECC_DISSEM') }} OU_C2D_UC_NMA_DECC_DISSEM + WHERE (SNAPSHOT_DATE, REPORTING_NCB) IN + (SELECT MAX(SNAPSHOT_DATE), + REPORTING_NCB + FROM {{ source('ou_c2d','UC_NMA_DECC_DISSEM') }} OU_C2D_UC_NMA_DECC_DISSEM + GROUP BY REPORTING_NCB) + GROUP BY REPORTING_NCB) \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ.sql new file mode 100644 index 0000000..0c7066f --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ.sql @@ -0,0 +1,81 @@ +{{ config(materialized='table', + tags=["m_MOPDB_C2D_UC_AGGREGATED"], + alias='UC_AGG_NMA_DISSEM_SQ', + schema='C2D' +) }} + +SELECT DISTINCT + OU_C2D_UC_NMA_DISSEM.A_WORKFLOW_HISTORY_KEY, + OU_C2D_UC_NMA_DISSEM.REPORTING_NCB, + OU_C2D_UC_NMA_DISSEM.SNAPSHOT_DATE, + OU_C2D_UC_NMA_DISSEM.FILE_CREATION_DATE, + OU_C2D_UC_NMA_DISSEM.MFI_ID, + OU_C2D_UC_NMA_DISSEM.OTHER_REG_NO, + OU_C2D_UC_NMA_DISSEM.TYPE_OF_SYSTEM, + OU_C2D_UC_NMA_DISSEM.TYPE_OF_OPERATION, + OU_C2D_UC_NMA_DISSEM.DOM_OR_XBORDER, + OU_C2D_UC_NMA_DISSEM.NON_MKT_ASSET_TYPE, + OU_C2D_UC_NMA_DISSEM.MATURITY_DATE, + OU_C2D_UC_NMA_DISSEM.INTEREST_PAYMENT_TYPE, + OU_C2D_UC_NMA_DISSEM.CAP, + OU_C2D_UC_NMA_DISSEM.REFERENCE_RATE, + OU_C2D_UC_NMA_DISSEM.REFERENCE_RATE_COMMENT, + OU_C2D_UC_NMA_DISSEM.COLL_BEFORE_HAIRCUTS, + OU_C2D_UC_NMA_DISSEM.COLL_AFTER_HAIRCUTS, + OU_C2D_UC_NMA_DISSEM.NO_AGGR_DEBTORS, + OU_C2D_UC_NMA_DISSEM.ELIGIBLE_VIA_GUAR, + OU_C2D_UC_NMA_DISSEM.DEBTOR_TYPE, + OU_C2D_UC_NMA_DISSEM.DEBTOR_NAME, + OU_C2D_UC_NMA_DISSEM.DEBTOR_ID_TYPE, + OU_C2D_UC_NMA_DISSEM.DEBTOR_ID, + OU_C2D_UC_NMA_DISSEM.DEBTOR_CLASS, + OU_C2D_UC_NMA_DISSEM.DEBTOR_RESIDENCE, + OU_C2D_UC_NMA_DISSEM.DEBTOR_CAS, + OU_C2D_UC_NMA_DISSEM.DEBTOR_CRED_PROV, + OU_C2D_UC_NMA_DISSEM.DEBTOR_RATING_ENUM_VALUE, + OU_C2D_UC_NMA_DISSEM.DEBTOR_RATING_NUMBER_VALUE, + OU_C2D_UC_NMA_DISSEM.GUAR_TYPE, + OU_C2D_UC_NMA_DISSEM.GUAR_NAME, + OU_C2D_UC_NMA_DISSEM.GUAR_ID_TYPE, + OU_C2D_UC_NMA_DISSEM.GUAR_ID, + OU_C2D_UC_NMA_DISSEM.GUAR_CLASS, + OU_C2D_UC_NMA_DISSEM.GUAR_RESIDENCE, + OU_C2D_UC_NMA_DISSEM.GUAR_CRED_CAS, + OU_C2D_UC_NMA_DISSEM.GUAR_CRED_PROV, + OU_C2D_UC_NMA_DISSEM.GUAR_RATING_ENUM_VALUE, + OU_C2D_UC_NMA_DISSEM.GUAR_RATING_NUMBER_VALUE, + OU_C2D_UC_NMA_DISSEM.NO_AGGR_ASSETS, + OU_C2D_UC_NMA_DISSEM.DENOMINATION, + OU_C2D_UC_NMA_DISSEM.SECURED_FLAG, + OU_C2D_UC_NMA_DISSEM.RESIDUAL_MATURITY, + OU_C2D_UC_NMA_DISSEM.BUCKET_SIZE, + OU_C2D_UC_NMA_DISSEM.NCB_COMMENT, + OU_C2D_UC_NMA_DISSEM.VALUATION_METHODOLOGY, + OU_C2D_UC_NMA_DISSEM.NOM_AMT_SUBMITTED, + OU_C2D_UC_NMA_DISSEM.RESET_PERIOD_MORE_ONE_YEAR, + OU_C2D_UC_NMA_DISSEM.MOBILISATION_CHANNEL, + OU_C2D_UC_NMA_DISSEM.CCB, + OU_C2D_UC_NMA_DISSEM.INVESTOR_SSS, + OU_C2D_UC_NMA_DISSEM.INTERMEDIARY_SSS, + OU_C2D_UC_NMA_DISSEM.ISSUER_SSS, + OU_C2D_UC_NMA_DISSEM.SUSPECT_ID, + OU_C2D_UC_NMA_DISSEM.QUALITY_CHECK_STATUS, + OU_C2D_UC_NMA_DISSEM.ERROR_CODE, + OU_C2D_UC_NMA_DISSEM.ERROR_MESSAGE, + OU_C2D_UC_NMA_DISSEM.ERROR_POSITION_IN_FILE, + OU_C2D_UC_NMA_DISSEM.OA_ID, + OU_C2D_UC_NMA_DISSEM.CONTRACT_ID, + OU_C2D_UC_NMA_DISSEM.INSTRMNT_ID +FROM + {{ source('ou_c2d','UC_NMA_DISSEM') }} OU_C2D_UC_NMA_DISSEM +WHERE + (OU_C2D_UC_NMA_DISSEM.A_WORKFLOW_HISTORY_KEY, OU_C2D_UC_NMA_DISSEM.REPORTING_NCB ) IN + (SELECT MAX(OU_C2D_UC_NMA_DISSEM.A_WORKFLOW_HISTORY_KEY), + OU_C2D_UC_NMA_DISSEM.REPORTING_NCB + FROM {{ source('ou_c2d','UC_NMA_DISSEM') }} OU_C2D_UC_NMA_DISSEM + WHERE (OU_C2D_UC_NMA_DISSEM.SNAPSHOT_DATE, OU_C2D_UC_NMA_DISSEM.REPORTING_NCB) IN + (SELECT MAX(OU_C2D_UC_NMA_DISSEM.SNAPSHOT_DATE), + OU_C2D_UC_NMA_DISSEM.REPORTING_NCB + FROM {{ source('ou_c2d','UC_NMA_DISSEM') }} OU_C2D_UC_NMA_DISSEM + GROUP BY OU_C2D_UC_NMA_DISSEM.REPORTING_NCB) + GROUP BY OU_C2D_UC_NMA_DISSEM.REPORTING_NCB) \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ.sql new file mode 100644 index 0000000..c4a42e3 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ.sql @@ -0,0 +1,172 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY"], + alias = "DEBT_DAILY_SQ", + schema = "CSDB", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-17", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-10-03" + } +) }} + +WITH LOAD_HISTORY AS +( + {{ get_ODS_wf_start_max_key("w_ODS_CSDB_DEBT_DAILY", get_main_task_name(model.name) ) }} +), +DATE_LIST_MATCHING as + ( + SELECT distinct + OU_CSDB_DEBT_DAILY.A_WORKFLOW_HISTORY_KEY as A_WORKFLOW_HISTORY_KEY , + LOAD_HISTORY.MAX_ODS_WORKFLOW_KEY_DAY as MAX_ODS_WORKFLOW_KEY_DAY, + LOAD_HISTORY.ODS_WORKFLOW_START as ODS_WORKFLOW_START + FROM + {{ source("ods","DEBT_DAILY")}} OU_CSDB_DEBT_DAILY + INNER JOIN + LOAD_HISTORY + ON LOAD_HISTORY.A_WORKFLOW_HISTORY_KEY = OU_CSDB_DEBT_DAILY.A_WORKFLOW_HISTORY_KEY + ) + +SELECT /*+ PARALLEL(ou_csdb_debt_daily,4) */ + OU_CSDB_DEBT_DAILY.A_KEY, + OU_CSDB_DEBT_DAILY.NEWUPDATED, + OU_CSDB_DEBT_DAILY.IDLOADDATE_DIM, + OU_CSDB_DEBT_DAILY.EXTERNALCODE_ISIN, + OU_CSDB_DEBT_DAILY.EXTERNALCODETYPE_NC, + OU_CSDB_DEBT_DAILY.EXTERNALCODE_NATIONAL, + OU_CSDB_DEBT_DAILY.IDIRINSTRUMENT, + OU_CSDB_DEBT_DAILY.SHORTNAME, + OU_CSDB_DEBT_DAILY.IDIRDEPOSITORY, + OU_CSDB_DEBT_DAILY.IDIRDEBTTYPE, + OU_CSDB_DEBT_DAILY.IDIRASSETSECTYPE, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_CFI, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAI, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAI_DM, + OU_CSDB_DEBT_DAILY.IDIRCURRENCY_NOMINAL, + OU_CSDB_DEBT_DAILY.AMOUNTISSUED, + OU_CSDB_DEBT_DAILY.AMOUNTOUTSTANDING, + OU_CSDB_DEBT_DAILY.AMOUNTOUTSTANDING_EUR, + OU_CSDB_DEBT_DAILY.POOLFACTOR, + OU_CSDB_DEBT_DAILY.ISSUEPRICE, + OU_CSDB_DEBT_DAILY.IDISSUEDATE, + OU_CSDB_DEBT_DAILY.IDIRCOUPONTYPE, + OU_CSDB_DEBT_DAILY.IDIRCOUPONFREQUENCY, + OU_CSDB_DEBT_DAILY.IDIRCURRENCY_COUPON, + OU_CSDB_DEBT_DAILY.COUPONRATE, + OU_CSDB_DEBT_DAILY.COUPONDATE, + OU_CSDB_DEBT_DAILY.IDIRREDEMPTIONTYPE, + OU_CSDB_DEBT_DAILY.IDIRREDEMPTIONFREQUENCY, + OU_CSDB_DEBT_DAILY.IDIRCURRENCY_REDEMPTION, + OU_CSDB_DEBT_DAILY.REDEMPTIONPRICE, + OU_CSDB_DEBT_DAILY.IDMATURITYDATE, + OU_CSDB_DEBT_DAILY.IDIRORGANISATIONALIASTYPE_IS, + OU_CSDB_DEBT_DAILY.ISSUERSOURCECODE, + OU_CSDB_DEBT_DAILY.ISSUEREXTERNALCODE_MFI, + OU_CSDB_DEBT_DAILY.ISSUEREXTERNALCODE_BIC, + OU_CSDB_DEBT_DAILY.ISSUEREXTERNALCODE_BEI, + OU_CSDB_DEBT_DAILY.IDIRORGANISATION_ISSUER, + OU_CSDB_DEBT_DAILY.ISSUERNAME, + OU_CSDB_DEBT_DAILY.IDIRCOUNTRY, + OU_CSDB_DEBT_DAILY.IDIRCOUNTRY_DM, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAO, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAO_DM, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_NACE, + OU_CSDB_DEBT_DAILY.PUBLICATIONPRICEDATE, + OU_CSDB_DEBT_DAILY.PUBLICATIONPRICE, + OU_CSDB_DEBT_DAILY.PUBLICATIONPRICETYPE, + OU_CSDB_DEBT_DAILY.PUBLICATIONPRICEQUOTATIONBASIS, + OU_CSDB_DEBT_DAILY.MONTHLYAVERAGEPRICE, + OU_CSDB_DEBT_DAILY.ACCRUALSTARTDATE, + OU_CSDB_DEBT_DAILY.DEBTACCRUALDEBTOR, + OU_CSDB_DEBT_DAILY.DEBTACCRUALDEBTOR_DM, + OU_CSDB_DEBT_DAILY.DEBTACCRUALCREDITOR, + OU_CSDB_DEBT_DAILY.DEBTACCRUALCREDITOR_TYP, + OU_CSDB_DEBT_DAILY.ACCRUEDINTEREST, + OU_CSDB_DEBT_DAILY.YTMNONOPTIONADJUSTED, + OU_CSDB_DEBT_DAILY.ESCB_ISSUER_IDENT, + OU_CSDB_DEBT_DAILY.ESCB_ISSUER_IDENT_TYP, + OU_CSDB_DEBT_DAILY.IDUDCMPPARTY, + OU_CSDB_DEBT_DAILY.AMOUNTOUTSTANDINGTYPE, + OU_CSDB_DEBT_DAILY.MARKETCAPITALISATION, + OU_CSDB_DEBT_DAILY.MARKETCAPITALISATION_EUR, + OU_CSDB_DEBT_DAILY.VA_SECURITYSTATUS, + OU_CSDB_DEBT_DAILY.VA_INSTRSUPPLEMENTARYCLASS, + OU_CSDB_DEBT_DAILY.VA_RESIDUALMATURITYCLASS, + OU_CSDB_DEBT_DAILY.VA_ISINSEC, + OU_CSDB_DEBT_DAILY.VA_ISELIGIBLEFOREADB, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAI10, + OU_CSDB_DEBT_DAILY.IDIRCLASSIFICATIONCODE_ESAO10, + OU_CSDB_DEBT_DAILY.IDIRDEBTTYPE_N, + OU_CSDB_DEBT_DAILY.SENIORITY, + OU_CSDB_DEBT_DAILY.ISSUEREXTERNALCODE_LEI, + OU_CSDB_DEBT_DAILY.A_WORKFLOW_HISTORY_KEY, + OU_CSDB_DEBT_DAILY.INSTR_ESA2010_CLASS_VALUETYPE, + OU_CSDB_DEBT_DAILY.ISS_ESA2010_CLASS_VALUETYPE, + OU_CSDB_DEBT_DAILY.SEC_STATUS_DATE, + OU_CSDB_DEBT_DAILY.GROUP_TYPE, + OU_CSDB_DEBT_DAILY.HAS_EMBEDDED_OPTION, + OU_CSDB_DEBT_DAILY.VOLUME_TRADED, + OU_CSDB_DEBT_DAILY.PRIMARY_LISTING_NAME, + OU_CSDB_DEBT_DAILY.PRIM_LISTING_RESIDENCY_COUNTRY, + OU_CSDB_DEBT_DAILY.INSTR_PORTFOLIO_FLAGS, + OU_CSDB_DEBT_DAILY.BOND_DURATION, + OU_CSDB_DEBT_DAILY.RESIDUAL_MATURITY, + OU_CSDB_DEBT_DAILY.ORIGINAL_MATURITY, + OU_CSDB_DEBT_DAILY.CFIN_CLASSIFICATION, + OU_CSDB_DEBT_DAILY.COUPONFIRSTPAYMENTDATE, + OU_CSDB_DEBT_DAILY.COUPONLASTPAYMENTDATE, + OU_CSDB_DEBT_DAILY.COUPONRATEUNDERLYINGCODE_ISIN, + OU_CSDB_DEBT_DAILY.COUPONRATESPREAD, + OU_CSDB_DEBT_DAILY.COUPONRATEMULTIPLIER, + OU_CSDB_DEBT_DAILY.COUPONRATECAP, + OU_CSDB_DEBT_DAILY.COUPONRATEFLOOR, + OU_CSDB_DEBT_DAILY.IDISSUEDATE_TRANCHE, + OU_CSDB_DEBT_DAILY.ISSUEPRICE_TRANCHE, + OU_CSDB_DEBT_DAILY.VA_ISPRIVATEPLACEMENT, + OU_CSDB_DEBT_DAILY.RIAD_CODE, + OU_CSDB_DEBT_DAILY.RIAD_OUID , + OU_CSDB_DEBT_DAILY.ESG1, + OU_CSDB_DEBT_DAILY.ESG2, + OU_CSDB_DEBT_DAILY.ESG3, + OU_CSDB_DEBT_DAILY.STRIP, + OU_CSDB_DEBT_DAILY.DEPOSITORY_RECEIPT, + OU_CSDB_DEBT_DAILY.RULE_144A, + OU_CSDB_DEBT_DAILY.REG_S, + OU_CSDB_DEBT_DAILY.WARRANT, + OU_CSDB_DEBT_DAILY.CSEC_RELEVANCE_STOCK, + OU_CSDB_DEBT_DAILY.CSEC_RELEVANCE_GROSS_ISSUANCE, + OU_CSDB_DEBT_DAILY.CSEC_RELEVANCE_REDEMPTION, + OU_CSDB_DEBT_DAILY.ACCRUING_COUPON, + OU_CSDB_DEBT_DAILY.ACCRUING_DISCOUNT, + CAST(NULL as varchar2(255)) AS PLACEHOLDER32, + CAST(NULL as varchar2(255)) AS PLACEHOLDER33, + CAST(NULL as varchar2(255)) AS PLACEHOLDER34, + CAST(NULL as varchar2(255)) AS PLACEHOLDER35, + CAST(NULL as varchar2(255)) AS PLACEHOLDER36, + CAST(NULL as varchar2(255)) AS PLACEHOLDER37, + CAST(NULL as varchar2(255)) AS PLACEHOLDER38, + CAST(NULL as varchar2(255)) AS PLACEHOLDER39, + CAST(NULL as varchar2(255)) AS PLACEHOLDER40, + CAST(NULL as varchar2(255)) AS PLACEHOLDER41, + CAST(NULL as varchar2(255)) AS PLACEHOLDER42, + CAST(NULL as varchar2(255)) AS PLACEHOLDER43, + CAST(NULL as varchar2(255)) AS PLACEHOLDER44, + CAST(NULL as varchar2(255)) AS PLACEHOLDER45, + CAST(NULL as varchar2(255)) AS PLACEHOLDER46, + CAST(NULL as varchar2(255)) AS PLACEHOLDER47, + CAST(NULL as varchar2(255)) AS PLACEHOLDER48, + CAST(NULL as varchar2(255)) AS PLACEHOLDER49, + CAST(NULL as varchar2(255)) AS PLACEHOLDER50, + TRUNC(DATE_LIST_MATCHING.ODS_WORKFLOW_START) AS WORKFLOW_START, + MAX(OU_CSDB_DEBT_DAILY.A_WORKFLOW_HISTORY_KEY) OVER (PARTITION BY TO_DATE(TO_CHAR(DATE_LIST_MATCHING.ODS_WORKFLOW_START,'DD/MM/YYYY'),'DD/MM/YYYY')) as MAX_KEY +FROM + {{ source("ods","DEBT_DAILY")}} OU_CSDB_DEBT_DAILY +INNER JOIN + DATE_LIST_MATCHING + ON DATE_LIST_MATCHING.A_WORKFLOW_HISTORY_KEY = OU_CSDB_DEBT_DAILY.A_WORKFLOW_HISTORY_KEY + AND DATE_LIST_MATCHING.A_WORKFLOW_HISTORY_KEY + in (select min(A_WORKFLOW_HISTORY_KEY) from DATE_LIST_MATCHING) + + + \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ.sql new file mode 100644 index 0000000..1f347c9 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ.sql @@ -0,0 +1,130 @@ +{{ + config( + materialized="table", + tags=["m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT"], + alias="DEBT_SQ", + schema="CSDB", + ) +}} +SELECT + /*+ PARALLEL(csdb_sq,4) */ + ou_csdb_debt.A_KEY, + ou_csdb_debt.NEWUPDATED, + ou_csdb_debt.IDLoadDATE_DIM, + ou_csdb_debt.EXTERNALCODE_ISIN, + ou_csdb_debt.EXTERNALCODETYPE_NC, + ou_csdb_debt.EXTERNALCODE_NATIONAL, + ou_csdb_debt.IDIRINSTRUMENT, + ou_csdb_debt.SHORTNAME, + CAST('' AS VARCHAR2(1 CHAR)) as IDIRDEPOSITORY, + ou_csdb_debt.IDIRDEBTTYPE, + ou_csdb_debt.IDIRASSETSECTYPE, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_CFI, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAI, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAI_DM, + ou_csdb_debt.IDIRCURRENCY_NOMINAL, + ou_csdb_debt.AMOUNTISSUED, + ou_csdb_debt.AMOUNTOUTSTANDING, + ou_csdb_debt.AMOUNTOUTSTANDING_EUR, + ou_csdb_debt.POOLFACTOR, + ou_csdb_debt.ISSUEPRICE, + ou_csdb_debt.IDISSUEDATE, + ou_csdb_debt.IDIRCOUPONTYPE, + ou_csdb_debt.IDIRCOUPONFREQUENCY, + ou_csdb_debt.IDIRCURRENCY_COUPON, + ou_csdb_debt.COUPONRATE, + ou_csdb_debt.COUPONDATE, + ou_csdb_debt.IDIRREDEMPTIONTYPE, + ou_csdb_debt.IDIRREDEMPTIONFREQUENCY, + ou_csdb_debt.IDIRCURRENCY_REDEMPTION, + ou_csdb_debt.REDEMPTIONPRICE, + ou_csdb_debt.IDMATURITYDATE, + ou_csdb_debt.IDIRORGANISATIONALIASTYPE_IS, + ou_csdb_debt.ISSUERSOURCECODE, + ou_csdb_debt.ISSUEREXTERNALCODE_MFI, + ou_csdb_debt.ISSUEREXTERNALCODE_BIC, + ou_csdb_debt.ISSUEREXTERNALCODE_BEI, + ou_csdb_debt.IDIRORGANISATION_ISSUER, + ou_csdb_debt.ISSUERNAME, + ou_csdb_debt.IDIRCOUNTRY, + ou_csdb_debt.IDIRCOUNTRY_DM, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAO, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAO_DM, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_NACE, + ou_csdb_debt.PUBLICATIONPRICEDATE, + ou_csdb_debt.PUBLICATIONPRICE, + ou_csdb_debt.PUBLICATIONPRICETYPE, + ou_csdb_debt.PUBLICATIONPRICEQUOTATIONBASIS, + ou_csdb_debt.MONTHLYAVERAGEPRICE, + ou_csdb_debt.ACCRUALSTARTDATE, + ou_csdb_debt.DEBTACCRUALDEBTOR, + ou_csdb_debt.DEBTACCRUALDEBTOR_DM, + ou_csdb_debt.DEBTACCRUALCREDITOR, + ou_csdb_debt.DEBTACCRUALCREDITOR_TYP, + ou_csdb_debt.ACCRUEDINTEREST, + ou_csdb_debt.YTMNONOPTIONADJUSTED, + ou_csdb_debt.ESCB_ISSUER_IDENT, + ou_csdb_debt.VA_ESCBCODETYPE, + ou_csdb_debt.IDUDCMPPARTY, + ou_csdb_debt.AMOUNTOUTSTANDINGTYPE, + ou_csdb_debt.MARKETCAPITALISATION, + ou_csdb_debt.MARKETCAPITALISATION_EUR, + ou_csdb_debt.VA_SECURITYSTATUS, + ou_csdb_debt.VA_INSTRSUPPLEMENTARYCLASS, + ou_csdb_debt.VA_RESIDUALMATURITYCLASS, + ou_csdb_debt.VA_ISINSEC, + ou_csdb_debt.VA_ISELIGIBLEFOREADB, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAI10, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_ESAO10, + ou_csdb_debt.IDIRDEBTTYPE_N, + ou_csdb_debt.SENIORITY, + ou_csdb_debt.ISSUEREXTERNALCODE_LEI, + ou_csdb_debt.A_WORKFLOW_HISTORY_KEY, + ou_csdb_debt.INSTR_ESA2010_CLASS_VALUETYPE, + ou_csdb_debt.ISS_ESA2010_CLASS_VALUETYPE, + ou_csdb_debt.VA_SECURITYSTATUSDATE, + ou_csdb_debt.GROUP_TYPE, + ou_csdb_debt.HASEMBEDDEDOPTION, + ou_csdb_debt.VOLUMETRADED, + ou_csdb_debt.PRIMARYLISTINGNAME, + ou_csdb_debt.PRIMARYLISTINGCOUNTRY, + ou_csdb_debt.VA_INSTRPORTFLAGS, + ou_csdb_debt.VA_BONDDURATION, + ou_csdb_debt.RESIDUALMATURITY, + ou_csdb_debt.ORIGINAL_MATURITY, + ou_csdb_debt.IDIRCLASSIFICATIONCODE_CFIN, + ou_csdb_debt.COUPONFIRSTPAYMENTDATE, + ou_csdb_debt.COUPONLASTPAYMENTDATE, + ou_csdb_debt.COUPONRATEUNDERLYINGCODE_ISIN, + ou_csdb_debt.COUPONRATESPREAD, + ou_csdb_debt.COUPONRATEMULTIPLIER, + ou_csdb_debt.COUPONRATECAP, + ou_csdb_debt.COUPONRATEFLOOR, + ou_csdb_debt.IDISSUEDATE_TRANCHE, + ou_csdb_debt.ISSUEPRICE_TRANCHE, + ou_csdb_debt.VA_ISPRIVATEPLACEMENT, + ou_csdb_debt.RIAD_CODE, + ou_csdb_debt.RIAD_OUID, + ou_csdb_debt.ESG1, + ou_csdb_debt.ESG2, + ou_csdb_debt.ESG3, + ou_csdb_debt.STRIP, + ou_csdb_debt.DEPOSITORY_RECEIPT, + ou_csdb_debt.RULE_144A, + ou_csdb_debt.REG_S, + ou_csdb_debt.WARRANT, + ou_csdb_debt.CSEC_RELEVANCE_STOCK, + ou_csdb_debt.CSEC_RELEVANCE_GROSS_ISSUANCE, + ou_csdb_debt.CSEC_RELEVANCE_REDEMPTION, + ou_csdb_debt.ACCRUING_COUPON, + ou_csdb_debt.ACCRUING_DISCOUNT, + wh.WORKFLOW_START +FROM + {{ source("ods", "DEBT") }} ou_csdb_debt + JOIN ct_mrds.a_workflow_history wh ON ou_csdb_debt.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY +WHERE + wh.SERVICE_NAME = 'ODS' + AND ou_csdb_debt.A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_CSDB_DEBT", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ.sql new file mode 100644 index 0000000..5637d81 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL"], + alias = "INSTR_DESC_FULL_SQ", + schema = "CSDB", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-10", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-10-03" + } +) }} + +WITH LOAD_HISTORY AS +( + {{ get_ODS_wf_start_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name) + )}} +) +SELECT + LOAD_HISTORY.ODS_WORKFLOW_START, + OU_CSDB_INSTR_DESC_FULL.IDIRINSTRUMENT, + OU_CSDB_INSTR_DESC_FULL.ISIN, + OU_CSDB_INSTR_DESC_FULL.MOO_INSTR_ID, + OU_CSDB_INSTR_DESC_FULL.SNP_INSTR_ID, + OU_CSDB_INSTR_DESC_FULL.FITCH_IDENTIFIER, + OU_CSDB_INSTR_DESC_FULL.DBRS_IDENTIFIER, + OU_CSDB_INSTR_DESC_FULL.EA_STATUS, + OU_CSDB_INSTR_DESC_FULL.IS_TMS, + OU_CSDB_INSTR_DESC_FULL.DBRS_COVERED_BOND_PROGRAM, + OU_CSDB_INSTR_DESC_FULL.FITCH_PRG_IDENTIFIER, + OU_CSDB_INSTR_DESC_FULL.MOO_DEAL_NUMBER, + OU_CSDB_INSTR_DESC_FULL.SNP_PROGRAM_ID, + OU_CSDB_INSTR_DESC_FULL.IDIRDEBTTYPE, + OU_CSDB_INSTR_DESC_FULL.SNP_DEBT_TYPE, + OU_CSDB_INSTR_DESC_FULL.MOODY_SENIORITY, + OU_CSDB_INSTR_DESC_FULL.FITCH_DEBT_LEVEL_CODE, + OU_CSDB_INSTR_DESC_FULL.DBRS_RANK_TYPE, + OU_CSDB_INSTR_DESC_FULL.DBRS_SECURITY_TYPE, + OU_CSDB_INSTR_DESC_FULL.SCO_DEBT_TYPE, + OU_CSDB_INSTR_DESC_FULL.SCO_INSTR_ID, + OU_CSDB_INSTR_DESC_FULL.SCO_COVERED_BOND_PROGRAM, + OU_CSDB_INSTR_DESC_FULL.SCO_CATEGORY, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER15, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER16, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER17, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER18, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER19, + OU_CSDB_INSTR_DESC_FULL.PLACEHOLDER20, + OU_CSDB_INSTR_DESC_FULL.A_WORKFLOW_HISTORY_KEY, + LOAD_HISTORY.MAX_ODS_WORKFLOW_KEY_DAY +FROM + {{ source("ods","INSTR_DESC_FULL")}} OU_CSDB_INSTR_DESC_FULL +INNER JOIN + LOAD_HISTORY + ON LOAD_HISTORY.A_WORKFLOW_HISTORY_KEY = OU_CSDB_INSTR_DESC_FULL.A_WORKFLOW_HISTORY_KEY + AND OU_CSDB_INSTR_DESC_FULL.A_WORKFLOW_HISTORY_KEY + in (select min(LOAD_HISTORY.A_WORKFLOW_HISTORY_KEY) from LOAD_HISTORY) + \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ.sql new file mode 100644 index 0000000..d57c5ab --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ.sql @@ -0,0 +1,65 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL"], + alias = "INSTR_RAT_FULL_SQ", + schema = "CSDB", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-14", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-09-29" + } +) }} + + +SELECT + OU_CSDB_INSTR_RAT_FULL.A_KEY, + OU_CSDB_INSTR_RAT_FULL.RDB_INSTR_ID, + OU_CSDB_INSTR_RAT_FULL.SOURCE, + OU_CSDB_INSTR_RAT_FULL.RATING_SCHEME, + OU_CSDB_INSTR_RAT_FULL.RATING, + OU_CSDB_INSTR_RAT_FULL.RATING_DATE, + OU_CSDB_INSTR_RAT_FULL.TIME_HORIZON, + OU_CSDB_INSTR_RAT_FULL.CURRENCY_TYPE, + OU_CSDB_INSTR_RAT_FULL.NOTES, + OU_CSDB_INSTR_RAT_FULL.VALID_FROM, + OU_CSDB_INSTR_RAT_FULL.VALID_UNTIL, + OU_CSDB_INSTR_RAT_FULL.RDB_RATINGS_ID, + OU_CSDB_INSTR_RAT_FULL.WATCHLIST, + OU_CSDB_INSTR_RAT_FULL.WATCHLIST_DATE, + OU_CSDB_INSTR_RAT_FULL.OUTLOOK, + OU_CSDB_INSTR_RAT_FULL.OUTLOOK_DATE, + OU_CSDB_INSTR_RAT_FULL.RATING_ACTION, + OU_CSDB_INSTR_RAT_FULL.RATING_ACTION_DATE, + OU_CSDB_INSTR_RAT_FULL.IS_PRELIMINARY, + OU_CSDB_INSTR_RAT_FULL.RATING_RAW, + OU_CSDB_INSTR_RAT_FULL.RATING_TYPE, + OU_CSDB_INSTR_RAT_FULL.ENDORSEMENT_INDICATOR, + OU_CSDB_INSTR_RAT_FULL.LAST_REVIEW_DATE, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER6, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER7, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER8, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER9, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER10, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER11, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER12, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER13, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER14, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER15, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER16, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER17, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER18, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER19, + OU_CSDB_INSTR_RAT_FULL.PLACEHOLDER20, + OU_CSDB_INSTR_RAT_FULL.A_WORKFLOW_HISTORY_KEY +FROM + {{ source("ods","INSTR_RAT_FULL")}} OU_CSDB_INSTR_RAT_FULL +WHERE + OU_CSDB_INSTR_RAT_FULL.A_WORKFLOW_HISTORY_KEY IN + {{ + filter_workflow_history_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name)) + }} + + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ.sql new file mode 100644 index 0000000..39edd19 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL"], + alias = "ISSUER_DESC_FULL_SQ", + schema = "CSDB", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-15", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-10-01" + } +) }} + +WITH LOAD_HISTORY AS +( + {{ get_ODS_wf_start_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name) + )}} +) +SELECT + LOAD_HISTORY.ODS_WORKFLOW_START, + OU_CSDB_ISSUER_DESC_FULL.RDB_ISSUER_ID, + OU_CSDB_ISSUER_DESC_FULL.ISSUERNAME, + OU_CSDB_ISSUER_DESC_FULL.COUNTRY_DOMICILE, + OU_CSDB_ISSUER_DESC_FULL.IS_SOVEREIGN, + OU_CSDB_ISSUER_DESC_FULL.MOODY_IDENTIFIER, + OU_CSDB_ISSUER_DESC_FULL.SNP_ISSUER_ID, + OU_CSDB_ISSUER_DESC_FULL.FITCH_IDENTIFIER, + OU_CSDB_ISSUER_DESC_FULL.DBRS_IDENTIFIER, + OU_CSDB_ISSUER_DESC_FULL.LEI_ISSUER_ID, + OU_CSDB_ISSUER_DESC_FULL.RIAD_CODE, + OU_CSDB_ISSUER_DESC_FULL.RIAD_OUID, + OU_CSDB_ISSUER_DESC_FULL.CLASH_GROUP_STATUS, + OU_CSDB_ISSUER_DESC_FULL.SCO_ISSUER_ID, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER5, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER6, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER7, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER8, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER9, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER10, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER11, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER12, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER13, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER14, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER15, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER16, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER17, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER18, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER19, + OU_CSDB_ISSUER_DESC_FULL.PLACEHOLDER20, + OU_CSDB_ISSUER_DESC_FULL.A_WORKFLOW_HISTORY_KEY, + LOAD_HISTORY.MAX_ODS_WORKFLOW_KEY_DAY +FROM + {{ source("ods","ISSUER_DESC_FULL")}} OU_CSDB_ISSUER_DESC_FULL +INNER JOIN + LOAD_HISTORY + ON LOAD_HISTORY.A_WORKFLOW_HISTORY_KEY = OU_CSDB_ISSUER_DESC_FULL.A_WORKFLOW_HISTORY_KEY + AND OU_CSDB_ISSUER_DESC_FULL.A_WORKFLOW_HISTORY_KEY + in (select min(LOAD_HISTORY.A_WORKFLOW_HISTORY_KEY) from LOAD_HISTORY) \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ.sql new file mode 100644 index 0000000..33620c4 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL"], + alias = "ISSUER_RAT_FULL_SQ", + schema = "CSDB", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-16", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-09-29" + } +) }} + + +SELECT + OU_CSDB_ISSUER_RAT_FULL.A_KEY, + OU_CSDB_ISSUER_RAT_FULL.RDB_ISSUER_ID, + OU_CSDB_ISSUER_RAT_FULL.SOURCE, + OU_CSDB_ISSUER_RAT_FULL.RATING_SCHEME, + OU_CSDB_ISSUER_RAT_FULL.RATING, + OU_CSDB_ISSUER_RAT_FULL.RATING_DATE, + OU_CSDB_ISSUER_RAT_FULL.TIME_HORIZON, + OU_CSDB_ISSUER_RAT_FULL.CURRENCY_TYPE, + OU_CSDB_ISSUER_RAT_FULL.NOTES, + OU_CSDB_ISSUER_RAT_FULL.VALID_FROM, + OU_CSDB_ISSUER_RAT_FULL.VALID_UNTIL, + OU_CSDB_ISSUER_RAT_FULL.RDB_RATINGS_ID, + OU_CSDB_ISSUER_RAT_FULL.OUTLOOK, + OU_CSDB_ISSUER_RAT_FULL.OUTLOOK_DATE, + OU_CSDB_ISSUER_RAT_FULL.WATCHLIST, + OU_CSDB_ISSUER_RAT_FULL.WATCHLIST_DATE, + OU_CSDB_ISSUER_RAT_FULL.RATING_ACTION, + OU_CSDB_ISSUER_RAT_FULL.RATING_ACTION_DATE, + OU_CSDB_ISSUER_RAT_FULL.IS_PRELIMINARY, + OU_CSDB_ISSUER_RAT_FULL.RATING_RAW, + OU_CSDB_ISSUER_RAT_FULL.RATING_TYPE, + OU_CSDB_ISSUER_RAT_FULL.ENDORSEMENT_INDICATOR, + OU_CSDB_ISSUER_RAT_FULL.LAST_REVIEW_DATE, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER6, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER7, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER8, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER9, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER10, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER11, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER12, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER13, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER14, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER15, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER16, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER17, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER18, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER19, + OU_CSDB_ISSUER_RAT_FULL.PLACEHOLDER20, + OU_CSDB_ISSUER_RAT_FULL.A_WORKFLOW_HISTORY_KEY +FROM + {{ source("ods","ISSUER_RAT_FULL")}} OU_CSDB_ISSUER_RAT_FULL +WHERE + OU_CSDB_ISSUER_RAT_FULL.A_WORKFLOW_HISTORY_KEY = + {{ + filter_workflow_history_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name)) + }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ.sql new file mode 100644 index 0000000..7befb4e --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ.sql @@ -0,0 +1,61 @@ +{{ config(materialized='table', + schema='CSDB', + alias='RATINGS_RT_INSTRUMENT_RATING_SQ', + tags=["m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT"], +) }} + +SELECT +A_KEY, +A_WORKFLOW_HISTORY_KEY, +IDRT_INSTRUMENT_RATING, +IDRT_INSTRUMENT, +IDRT_POPULATION, +IDIRINSTRRATINGCLASS, +IDIRINSTRUMENT, +SOURCE, +RATING_SCHEME, +TIME_HORIZON, +CURRENCY_TYPE, +RATING_REGION_TYPE, +RATING, +RATING_RAW, +RATING_DATE, +RATING_ACTION, +RATING_ACTION_DATE, +OUTLOOK, +OUTLOOK_DATE, +WATCHLIST, +WATCHLIST_DATE, +VALIDFROM, +VALIDUNTIL, +IS_VALID, +IS_CURRENT, +EADB_REFERENCE, +IS_PRELIMINARY, +NOTES, +SUFFIX, +ENDORSEMENT_INDICATOR, +ATTR3, +ATTR4, +ATTR5, +KEYATTR1, +LAST_REVIEW_DATE, +KEYATTR3, +KEYATTR4, +KEYATTR5, +CORRECTFROM, +CORRECTUNTIL, +BATCH_FIRST_CREATED_DATE, +BATCH_LAST_UPDATED_DATE, +IS_SOLICITED, +TEC_SOURCE_SYSTEM, +TEC_DATASET, +TEC_SURROGATE_KEY, +TEC_CRC, +TEC_INGESTION_DATE, +TEC_EXECUTION_DATE, +TEC_RUN_ID, +TEC_BUSINESS_DATE +FROM +{{ source('ods','RATINGS_RT_INSTRUMENT_RATING') }} OU_CSDB_RATINGS_RT_INSTRUMENT_RATING +WHERE OU_CSDB_RATINGS_RT_INSTRUMENT_RATING.A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_max_key('w_ODS_CSDB_RATINGS_DEVO', get_main_task_name(model.name)) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ.sql new file mode 100644 index 0000000..6514f59 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ.sql @@ -0,0 +1,60 @@ +{{ config(materialized='table', + schema='CSDB', + alias='RATINGS_RT_PARTY_RATING_SQ', + tags=["m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING"], +) }} + +SELECT +A_KEY, +A_WORKFLOW_HISTORY_KEY, +IDRT_PARTY_RATING, +IDRT_PARTY, +IDRT_POPULATION, +IDIRPARTYRATINGCLASS, +SOURCE, +RATING_SCHEME, +TIME_HORIZON, +CURRENCY_TYPE, +RATING_REGION_TYPE, +RATING, +RATING_RAW, +RATING_DATE, +RATING_ACTION, +RATING_ACTION_DATE, +OUTLOOK, +OUTLOOK_DATE, +WATCHLIST, +WATCHLIST_DATE, +VALIDFROM, +VALIDUNTIL, +IS_VALID, +IS_CURRENT, +EADB_REFERENCE, +IS_PRELIMINARY, +NOTES, +SUFFIX, +ENDORSEMENT_INDICATOR, +ATTR3, +ATTR4, +ATTR5, +KEYATTR1, +LAST_REVIEW_DATE, +KEYATTR3, +KEYATTR4, +KEYATTR5, +CORRECTFROM, +CORRECTUNTIL, +BATCH_FIRST_CREATED_DATE, +BATCH_LAST_UPDATED_DATE, +IS_SOLICITED, +TEC_SOURCE_SYSTEM, +TEC_DATASET, +TEC_SURROGATE_KEY, +TEC_CRC, +TEC_INGESTION_DATE, +TEC_EXECUTION_DATE, +TEC_RUN_ID, +TEC_BUSINESS_DATE +FROM +{{ source('ou_csdb','RATINGS_RT_PARTY_RATING') }} OU_CSDB_RATINGS_RT_PARTY_RATING +WHERE OU_CSDB_RATINGS_RT_PARTY_RATING.A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_max_key('w_ODS_CSDB_RATINGS_DEVO', get_main_task_name(model.name)) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ.sql new file mode 100644 index 0000000..f9e5f82 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION"], + alias = "T_ELA_INFO_SQ", + schema = "ELA", + meta={ + "author": "Adelina Borisova", + "created": "2025-02-20", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-02-20" + } +) }} + +SELECT /*+ PARALLEL(OU_C2D_ELA_INFO_REPLICATION,4) */ + OU_C2D_ELA_INFO_REPLICATION.A_KEY, + OU_C2D_ELA_INFO_REPLICATION.A_WORKFLOW_HISTORY_KEY, + CAST(OU_C2D_ELA_INFO_REPLICATION.RIAD_CODE AS VARCHAR2(30 CHAR)) AS RIAD_CODE, + CAST(OU_C2D_ELA_INFO_REPLICATION.INSTITUTION_NAME AS VARCHAR2(200 CHAR)) AS MFI_NAME, + OU_C2D_ELA_INFO_REPLICATION.ELA_MATURITY_DATE, + OU_C2D_ELA_INFO_REPLICATION.ELA_VALUE_DATE, + CAST(OU_C2D_ELA_INFO_REPLICATION.ELA_BASE AS NUMBER(28,10)) AS ELA_BASE, + CAST(OU_C2D_ELA_INFO_REPLICATION.ELA_DENOMINATION AS VARCHAR2(3 CHAR)) AS ELA_DENOMINATION, + CAST(OU_C2D_ELA_INFO_REPLICATION.ELA AS NUMBER(28,10)) AS ELA, + CAST(OU_C2D_ELA_INFO_REPLICATION.INTEREST_RATE_APPLIED AS NUMBER(28,10)) AS INTEREST_RATE_APPLIED, + CAST(OU_C2D_ELA_INFO_REPLICATION.ISIN_CODE AS VARCHAR2(100 CHAR)) AS ISIN_CODE, + CAST(OU_C2D_ELA_INFO_REPLICATION.NOMINAL_AMOUNT_SUBMITTED AS NUMBER(28,10)) AS NOMINAL_AMOUNT_SUBMITTED, + CAST(OU_C2D_ELA_INFO_REPLICATION.COLLATERAL_VALUE_BEFORE_HAIRCU AS NUMBER(28,10)) AS COLL_BEFORE_HAIRCUTS, + CAST(OU_C2D_ELA_INFO_REPLICATION.COLLATERAL_VALUE_AFTER_HAIRCUT AS NUMBER(28,10)) AS COLL_AFTER_HAIRCUTS, + CAST(OU_C2D_ELA_INFO_REPLICATION.HAIRCUT AS NUMBER(28,10)) AS HAIRCUT, + CAST(OU_C2D_ELA_INFO_REPLICATION.ELA_ASSET_GROUP AS VARCHAR2(100 CHAR)) AS ELA_ASSET_GROUP, + CAST(OU_C2D_ELA_INFO_REPLICATION.DENOMINATION AS VARCHAR2(3 CHAR)) AS DENOMINATION, + CAST(OU_C2D_ELA_INFO_REPLICATION.ASSET_TYPE AS VARCHAR2(4 CHAR)) AS ASSET_TYPE, + CAST(OU_C2D_ELA_INFO_REPLICATION.DOMESTIC_OR_XBORDER AS VARCHAR2(20 CHAR)) AS DOM_OR_CROSS, + CAST(OU_C2D_ELA_INFO_REPLICATION.ABS_TYPE AS VARCHAR2(40 CHAR)) AS ABS_TYPE, + CAST(OU_C2D_ELA_INFO_REPLICATION.NUMBER_OF_AGGREG_ASSETS AS NUMBER(28,0)) AS NUMBER_OF_AGGREGATED_ASSETS, + CAST(OU_C2D_ELA_INFO_REPLICATION.NUMBER_OF_AGGREG_DEBTORS AS NUMBER(28,0)) AS NUMBER_OF_AGGREGATED_DEBTORS, + CAST(OU_C2D_ELA_INFO_REPLICATION.GUARANTEE AS VARCHAR2(200 CHAR)) AS GUARANTEE, + CAST(OU_C2D_ELA_INFO_REPLICATION.ISSUER_CODE AS VARCHAR2(30 CHAR)) AS ISSUER_CODE, + CAST(OU_C2D_ELA_INFO_REPLICATION.ISSUER_NAME AS VARCHAR2(200 CHAR)) AS ISSUER_NAME, + CAST(OU_C2D_ELA_INFO_REPLICATION.ISSUER_RESIDENCE AS VARCHAR2(3 CHAR)) AS ISSUER_RESIDENCE, + CAST(OU_C2D_ELA_INFO_REPLICATION.ISSUER_GROUP AS VARCHAR2(4 CHAR)) AS ISSUER_GROUP, + CAST(OU_C2D_ELA_INFO_REPLICATION.RATING_OF_ASSET AS VARCHAR2(100 CHAR)) AS RATING_OF_ASSET, + CAST(OU_C2D_ELA_INFO_REPLICATION.RATING_OF_THE_IS AS VARCHAR2(100 CHAR)) AS RATING_OF_THE_IS, + CAST(OU_C2D_ELA_INFO_REPLICATION.RATING_OF_THE_GU AS VARCHAR2(100 CHAR)) AS RATING_OF_THE_GU, + CAST(OU_C2D_ELA_INFO_REPLICATION.PRICE_INFORMATION AS VARCHAR2(100 CHAR)) AS PRICE_INFORMATION, + CAST(OU_C2D_ELA_INFO_REPLICATION.VALUATION_METHODOLOGY AS VARCHAR2(15 CHAR)) AS VALUATION_METHODOLOGY, + CAST(OU_C2D_ELA_INFO_REPLICATION.TYPE_OF_OPERATION AS VARCHAR2(30 CHAR)) AS TYPE_OF_OPERATION, + CAST(OU_C2D_ELA_INFO_REPLICATION.NCB_COMMENT AS VARCHAR2(200 CHAR)) AS NCB_COMMENT, + OU_C2D_ELA_INFO_REPLICATION.SNAPSHOT_DATE, + CAST(OU_C2D_ELA_INFO_REPLICATION.REPORTING_NCB AS VARCHAR2(2 CHAR)) AS REPORTING_NCB, + CAST(OU_C2D_ELA_INFO_REPLICATION.ID AS NUMBER(28,0)) AS ID, + CAST(OU_C2D_ELA_INFO_REPLICATION.VERSION AS VARCHAR2(20 CHAR)) AS VERSION, + CAST(OU_C2D_ELA_INFO_REPLICATION.IS_CORRECTION AS VARCHAR2(1 CHAR)) AS IS_CORRECTION, + CAST(OU_C2D_ELA_INFO_REPLICATION.META_INFORMATION_ID AS NUMBER(19,0)) AS META_INFORMATION_ID, + CAST(OU_C2D_ELA_INFO_REPLICATION.META_INFORMATION_TYPE AS VARCHAR2(50 CHAR)) AS META_INFORMATION_TYPE, + OU_C2D_ELA_INFO_REPLICATION.USED_SNAPSHOT_DATE, + OU_C2D_ELA_INFO_REPLICATION.PRICING_DATE, + CAST(NULL AS NUMBER(38,0)) AS RIAD_ENTTY_ID +FROM + {{ source("ods","ELA_INFO_REPLICATION")}} OU_C2D_ELA_INFO_REPLICATION + WHERE OU_C2D_ELA_INFO_REPLICATION.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_C2D_ELA", + get_main_task_name(model.name) + ) + }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ.sql new file mode 100644 index 0000000..903884f --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ.sql @@ -0,0 +1,21 @@ + +{{ config( + materialized = "table", + tags = ["m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN"], + alias = "POP_LIST_SQ", + schema = "FMCO_REPORTS", +) }} + + +SELECT /*+ PARALLEL(EA_RIAD_MFI_ISIN,4) */ +OU_C2D_EA_RIAD_MFI_ISIN.A_KEY, +OU_C2D_EA_RIAD_MFI_ISIN.A_WORKFLOW_HISTORY_KEY, +OU_C2D_EA_RIAD_MFI_ISIN.SNAPSHOT_DATE, +OU_C2D_EA_RIAD_MFI_ISIN.MFI_ID, +OU_C2D_EA_RIAD_MFI_ISIN.ISIN_CODE +FROM + {{ source("ou_c2d", "EA_RIAD_MFI_ISIN") }} OU_C2D_EA_RIAD_MFI_ISIN +WHERE + OU_C2D_EA_RIAD_MFI_ISIN.A_WORKFLOW_HISTORY_KEY +IN + {{ filter_workflow_history_key("w_ODS_C2D_MFI_ISIN_TRANSFER",get_main_task_name(model.name) ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ.sql new file mode 100644 index 0000000..5e542f5 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET"], + alias = "T_BALANCESHEET_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-03-31", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + } +) }} + +SELECT + /*+ PARALLEL(OU_LM_BALANCESHEET_HEADER,4) */ + MAX(OU_LM_BALANCESHEET_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_LM_BALANCESHEET_HEADER.REFERENCE_DATE) as MAX_KEY, + OU_LM_BALANCESHEET_HEADER.A_KEY, + OU_LM_BALANCESHEET_HEADER.A_WORKFLOW_HISTORY_KEY, + OU_LM_BALANCESHEET_HEADER.REFERENCE_DATE, + CAST(OU_LM_BALANCESHEET_HEADER.VERSION AS NUMBER(10,0)) AS VERSION, + CAST(OU_LM_BALANCESHEET_HEADER.COUNTRY AS VARCHAR2(3 CHAR)) AS COUNTRY, + CAST(OU_LM_BALANCESHEET_ITEM.ITEM_LEVEL AS NUMBER(10,0)) AS ITEM_LEVEL, + CAST(OU_LM_BALANCESHEET_ITEM.POSITION AS NUMBER(10,0)) AS POSITION, + CAST(OU_LM_BALANCESHEET_ITEM.ITEM_TYPE AS VARCHAR2(10 CHAR)) AS ITEM_TYPE, + CAST(OU_LM_BALANCESHEET_ITEM.FULLY_QUALIFIED_POSITION AS VARCHAR2(200 CHAR)) AS FULLY_QUALIFIED_POSITION, + CAST(OU_LM_BALANCESHEET_ITEM.NAME AS VARCHAR2(200 CHAR)) AS NAME, + CAST(OU_LM_BALANCESHEET_ITEM.AMOUNT AS NUMBER(28,8)) AS AMOUNT +FROM + {{ source("ods","BALANCESHEET_HEADER")}} OU_LM_BALANCESHEET_HEADER, + {{ source("ods","BALANCESHEET_ITEM")}} OU_LM_BALANCESHEET_ITEM +WHERE + OU_LM_BALANCESHEET_HEADER.A_KEY = OU_LM_BALANCESHEET_ITEM.A_HEADER_FK + AND OU_LM_BALANCESHEET_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_BALANCESHEET_ITEM.A_WORKFLOW_HISTORY_KEY + AND OU_LM_BALANCESHEET_HEADER.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_LM_BALANCESHEET", + get_main_task_name(model.name) + ) + }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ.sql new file mode 100644 index 0000000..66ce6c2 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS"], + alias = "T_CURRENT_ACCOUNTS_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-01", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + }, +) }} + +SELECT /*+ PARALLEL(OU_MRR_MRR_IND_CURR_ACC_HEADER,4) */ + MAX(OU_MRR_MRR_IND_CURR_ACC_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_MRR_MRR_IND_CURR_ACC_HEADER.REFERENCE_DATE ) as MAX_KEY, + OU_MRR_MRR_IND_CURR_ACC_HEADER.A_KEY, + OU_MRR_MRR_IND_CURR_ACC_HEADER.A_WORKFLOW_HISTORY_KEY, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.REVISION_NUMBER AS NUMBER(10,0)) AS REVISION_NUMBER, + OU_MRR_MRR_IND_CURR_ACC_HEADER.REFERENCE_DATE, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.CURRENT_ACCOUNT_BS_TOTAL AS NUMBER(28,8)) AS CURRENT_ACCOUNT_BS_TOTAL, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.MRR_FORECAST_TOTAL AS NUMBER(28,8)) AS MRR_FORECAST_TOTAL, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.CURRENT_ACCOUNT_MESSAGE_TOTAL AS NUMBER(28,8)) AS CURRENT_ACCOUNT_MESSAGE_TOTAL, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.MRR_MESSAGE_TOTAL AS NUMBER(28,8)) AS MRR_MESSAGE_TOTAL, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.PERIODICITY AS VARCHAR2(50 CHAR)) AS PERIODICITY, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.COUNTRY AS VARCHAR2(200 CHAR)) AS COUNTRY, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.MFI_CODE AS VARCHAR2(255 CHAR)) AS MFI_CODE, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.BANK_NAME AS VARCHAR2(500 CHAR)) AS BANK_NAME, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.CURRENT_ACCOUNT AS NUMBER(28,8)) AS CURRENT_ACCOUNT, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.MINIMUM_RESERVE_REQUIREMENT AS NUMBER(28,8)) AS MINIMUM_RESERVE_REQUIREMENT, + CAST(OU_MRR_MRR_IND_CURR_ACC_ITEM.COMMENT_ AS VARCHAR2(4000 CHAR)) AS COMMENT_, + CAST(OU_MRR_MRR_IND_CURR_ACC_HEADER.FREE_TEXT AS VARCHAR2(1000 CHAR)) AS FREE_TEXT +FROM + {{ source("ods","MRR_IND_CURRENT_ACCOUNT_HEADER") }} OU_MRR_MRR_IND_CURR_ACC_HEADER, + {{ source("ods","MRR_IND_CURRENT_ACCOUNT_ITEM") }} OU_MRR_MRR_IND_CURR_ACC_ITEM +WHERE + OU_MRR_MRR_IND_CURR_ACC_HEADER.A_KEY=OU_MRR_MRR_IND_CURR_ACC_ITEM.A_HEADER_FK + AND OU_MRR_MRR_IND_CURR_ACC_HEADER.A_WORKFLOW_HISTORY_KEY= OU_MRR_MRR_IND_CURR_ACC_ITEM.A_WORKFLOW_HISTORY_KEY + AND OU_MRR_MRR_IND_CURR_ACC_HEADER.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_LM_CURRENT_ACCOUNTS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ.sql new file mode 100644 index 0000000..cf08847 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT"], + alias = "T_FC_ADH_ADJUSTMENTS_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-04", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + }, +) }} + +SELECT /*+ PARALLEL(OU_LM_ADHOC_ADJ_HEADER,4) */ + MAX(OU_LM_ADHOC_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_LM_ADHOC_ADJ_HEADER.ADJUSTMENT_DATE) AS MAX_KEY, + MAX(OU_LM_ADHOC_ADJ_HEADER.A_KEY) OVER ( PARTITION BY OU_LM_ADHOC_ADJ_HEADER.ADJUSTMENT_DATE) AS MAX_A_KEY, + OU_LM_ADHOC_ADJ_HEADER.A_KEY, + OU_LM_ADHOC_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY, + OU_LM_ADHOC_ADJ_HEADER.ADJUSTMENT_DATE, + CAST(OU_LM_ADHOC_ADJ_HEADER.VERSION AS NUMBER(10,0)) AS VERSION, + CAST(OU_LM_ADHOC_ADJ_ITEM_HEADER.COUNTRY AS VARCHAR2(3)) AS COUNTRY, + OU_LM_ADHOC_ADJ_ITEM_HEADER.EFFECTIVE_DATE, + OU_LM_ADHOC_ADJ_ITEM_HEADER.LAST_DATE_NOT_FORECAST, + CAST(OU_LM_ADHOC_ADJ_ITEM.FORECAST_NAME AS VARCHAR2(50)) AS FORECAST_NAME, + CAST(OU_LM_ADHOC_ADJ_ITEM.ADJUSTMENT_AMOUNT AS NUMBER(25,8)) AS ADJUSTMENT_AMOUNT +FROM + {{ source("ods","ADHOC_ADJ_HEADER")}} OU_LM_ADHOC_ADJ_HEADER +INNER JOIN + {{ source("ods","ADHOC_ADJ_ITEM_HEADER")}} OU_LM_ADHOC_ADJ_ITEM_HEADER + ON OU_LM_ADHOC_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_ADHOC_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY + AND OU_LM_ADHOC_ADJ_HEADER.A_KEY=OU_LM_ADHOC_ADJ_ITEM_HEADER.A_HEADER_FK +LEFT JOIN + {{ source("ods","ADHOC_ADJ_ITEM")}} OU_LM_ADHOC_ADJ_ITEM + ON OU_LM_ADHOC_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_ADHOC_ADJ_ITEM.A_WORKFLOW_HISTORY_KEY + AND OU_LM_ADHOC_ADJ_ITEM_HEADER.A_KEY=OU_LM_ADHOC_ADJ_ITEM.A_HEADER_FK +WHERE + OU_LM_ADHOC_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_LM_ADHOC_ADJUSTMENT_MSG", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ.sql new file mode 100644 index 0000000..fe134d1 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT"], + alias = "T_FC_CSM_ADJUSTMENTS_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-08", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + } +) }} + +SELECT /*+ PARALLEL(OU_LM_CSM_ADJ_HEADER,4) */ + MAX(OU_LM_CSM_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_LM_CSM_ADJ_HEADER.YEAR, OU_LM_CSM_ADJ_HEADER.MONTH ) AS MAX_KEY, + MAX(OU_LM_CSM_ADJ_HEADER.A_KEY) OVER ( PARTITION BY OU_LM_CSM_ADJ_HEADER.YEAR, OU_LM_CSM_ADJ_HEADER.MONTH ) AS MAX_A_KEY, + OU_LM_CSM_ADJ_HEADER.A_KEY, + OU_LM_CSM_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY, + CAST(OU_LM_CSM_ADJ_HEADER.YEAR AS VARCHAR2(4)) AS YEAR, + CAST(OU_LM_CSM_ADJ_HEADER.MONTH AS VARCHAR2(2)) AS MONTH, + CAST(OU_LM_CSM_ADJ_HEADER.VERSION AS NUMBER(10,0)) AS VERSION, + CAST(OU_LM_CSM_ADJ_ITEM_HEADER.COUNTRY AS VARCHAR2(3)) AS COUNTRY, + OU_LM_CSM_ADJ_ITEM_HEADER.EFFECTIVE_DATE, + OU_LM_CSM_ADJ_ITEM_HEADER.LAST_DATE_NOT_FORECAST, + CAST(OU_LM_CSM_ADJ_ITEM.FORECAST_NAME AS VARCHAR2(50)) AS FORECAST_NAME, + CAST(OU_LM_CSM_ADJ_ITEM.ADJUSTMENT_AMOUNT AS NUMBER(25,8)) AS ADJUSTMENT_AMOUNT +FROM + {{ source("ods","CSM_ADJ_HEADER")}} OU_LM_CSM_ADJ_HEADER +INNER JOIN + {{ source("ods","CSM_ADJ_ITEM_HEADER")}} OU_LM_CSM_ADJ_ITEM_HEADER + ON OU_LM_CSM_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_CSM_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY + AND OU_LM_CSM_ADJ_HEADER.A_KEY = OU_LM_CSM_ADJ_ITEM_HEADER.A_HEADER_FK +LEFT JOIN + {{ source("ods","CSM_ADJ_ITEM")}} OU_LM_CSM_ADJ_ITEM + ON OU_LM_CSM_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY =OU_LM_CSM_ADJ_ITEM.A_WORKFLOW_HISTORY_KEY + AND OU_LM_CSM_ADJ_ITEM_HEADER.A_KEY = OU_LM_CSM_ADJ_ITEM.A_HEADER_FK +WHERE + OU_LM_CSM_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_LM_CSM_ADJUSTMENT_MSG", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ.sql new file mode 100644 index 0000000..91c7c9b --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ.sql @@ -0,0 +1,44 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT"], + alias = "T_FC_QRE_ADJUSTMENTS_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-02", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + }, +) }} + + +SELECT /*+ PARALLEL(OU_LM_QR_ADJ_HEADER,4) */ + MAX(OU_LM_QR_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_LM_QR_ADJ_HEADER.YEAR, OU_LM_QR_ADJ_HEADER.QUARTER) as MAX_KEY, + --MAX(OU_LM_QR_ADJ_HEADER.VERSION) OVER ( PARTITION BY OU_LM_QR_ADJ_HEADER.YEAR,OU_LM_QR_ADJ_HEADER.QUARTER ) as MAX_VERSION, + OU_LM_QR_ADJ_HEADER.A_KEY, + OU_LM_QR_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY, + CAST(OU_LM_QR_ADJ_HEADER.YEAR AS VARCHAR2(4)) AS YEAR, + CAST(OU_LM_QR_ADJ_HEADER.QUARTER AS NUMBER(1,0)) AS QUARTER, + CAST(OU_LM_QR_ADJ_HEADER.VERSION AS NUMBER(10,0)) AS VERSION, + CAST(OU_LM_QR_ADJ_ITEM_HEADER.COUNTRY AS VARCHAR2(3)) AS COUNTRY, + OU_LM_QR_ADJ_ITEM_HEADER.EFFECTIVE_DATE, + OU_LM_QR_ADJ_ITEM_HEADER.LAST_DATE_NOT_FORECAST, + CAST(OU_LM_QR_ADJ_ITEM.FORECAST_NAME AS VARCHAR2(50)) AS FORECAST_NAME, + CAST(OU_LM_QR_ADJ_ITEM.ADJUSTMENT_AMOUNT AS NUMBER(25,8)) AS ADJUSTMENT_AMOUNT + +FROM + {{ source("ods","QR_ADJ_HEADER")}} OU_LM_QR_ADJ_HEADER +INNER JOIN + {{ source("ods","QR_ADJ_ITEM_HEADER")}} OU_LM_QR_ADJ_ITEM_HEADER + ON OU_LM_QR_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_QR_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY + AND OU_LM_QR_ADJ_HEADER.A_KEY=OU_LM_QR_ADJ_ITEM_HEADER.A_HEADER_FK +LEFT JOIN + {{ source("ods",'QR_ADJ_ITEM')}} OU_LM_QR_ADJ_ITEM + ON OU_LM_QR_ADJ_ITEM_HEADER.A_WORKFLOW_HISTORY_KEY =OU_LM_QR_ADJ_ITEM.A_WORKFLOW_HISTORY_KEY + AND OU_LM_QR_ADJ_ITEM_HEADER.A_KEY=OU_LM_QR_ADJ_ITEM.A_HEADER_FK +WHERE + OU_LM_QR_ADJ_HEADER.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_key( + "w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ.sql new file mode 100644 index 0000000..4c45bf5 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST"], + alias = "T_FORECAST_SQ", + schema = "LM", + meta={ + "author": "Adelina Borisova", + "created": "2025-03-27", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + }, +) }} + + +SELECT + /*+ PARALLEL(OU_LM_FORECAST_HEADER,4) */ + MAX(OU_LM_FORECAST_HEADER.A_WORKFLOW_HISTORY_KEY) OVER (PARTITION BY OU_LM_FORECAST_HEADER.REFERENCE_DATE) as MAX_KEY, + OU_LM_FORECAST_HEADER.A_KEY, + OU_LM_FORECAST_HEADER.A_WORKFLOW_HISTORY_KEY, + OU_LM_FORECAST_HEADER.REFERENCE_DATE, + CAST(OU_LM_FORECAST_HEADER.REVISION AS NUMBER(10,0)) AS REVISION, + CAST(OU_LM_FORECAST_HEADER.COUNTRY AS VARCHAR2(3 CHAR)) AS COUNTRY, + --CAST(OU_LM_FORECAST_HEADER.FREE_TEXT AS VARCHAR2(4000 CHAR)) AS FREE_TEXT, + OU_LM_FORECAST_ITEM.FORECAST_DATE, + CAST(OU_LM_FORECAST_ITEM.FORECAST_NAME AS VARCHAR2(50 CHAR)) AS FORECAST_NAME, + CAST(OU_LM_FORECAST_ITEM.FORECAST_VALUE AS NUMBER(28,10)) AS FORECAST_VALUE +FROM + + {{ source("ods", "FORECAST_HEADER") }} OU_LM_FORECAST_HEADER, + {{ source("ods", "FORECAST_ITEM") }} OU_LM_FORECAST_ITEM + +WHERE OU_LM_FORECAST_HEADER.A_KEY = OU_LM_FORECAST_ITEM.A_HEADER_FK +AND OU_LM_FORECAST_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_FORECAST_ITEM.A_WORKFLOW_HISTORY_KEY +AND OU_LM_FORECAST_HEADER.A_WORKFLOW_HISTORY_KEY + +IN {{ filter_workflow_history_key( + "w_ODS_LM_FORECAST", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ.sql new file mode 100644 index 0000000..b4f0e9e --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES"], + alias = "T_STANDING_FACILITIES_SQ", + schema = "LM", + meta={ + "author": "Giacomo Scalabrin", + "created": "2025-03", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-11-26 - ticket MARS-1059" + } +) }} + +SELECT + /*+ PARALLEL(ou_lm_standing_facility_header,4) */ + MAX(OU_LM_STANDING_FACILITIES_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( + PARTITION BY OU_LM_STANDING_FACILITIES_HEADER.REF_DATE + ) AS MAX_KEY, + OU_LM_STANDING_FACILITIES_HEADER.A_KEY, + OU_LM_STANDING_FACILITIES_HEADER.A_WORKFLOW_HISTORY_KEY, + OU_LM_STANDING_FACILITIES_HEADER.REF_DATE, + CAST(OU_LM_STANDING_FACILITIES_HEADER.REV_NUMBER AS NUMBER(28,0)) AS REV_NUMBER, + CAST(OU_LM_STANDING_FACILITIES_HEADER.MLF_BS_TOTAL AS NUMBER(28,10)) AS MLF_BS_TOTAL, + CAST(OU_LM_STANDING_FACILITIES_HEADER.DF_BS_TOTAL AS NUMBER(28,10)) AS DF_BS_TOTAL, + CAST(OU_LM_STANDING_FACILITIES_HEADER.MLF_SF_TOTAL AS NUMBER(28,10)) AS MLF_SF_TOTAL, + CAST(OU_LM_STANDING_FACILITIES_HEADER.DF_SF_TOTAL AS NUMBER(28,10)) AS DF_SF_TOTAL, + CAST(OU_LM_STANDING_FACILITIES.COUNTRY AS VARCHAR2(3 CHAR)) AS COUNTRY, + CAST(OU_LM_STANDING_FACILITIES.MFI_ID AS VARCHAR2(500 CHAR)) AS MFI_ID, + CAST(OU_LM_STANDING_FACILITIES.MFI_NAME AS VARCHAR2(500 CHAR)) AS MFI_NAME, + CAST(OU_LM_STANDING_FACILITIES.MARGINAL_LENDING_FACILITY AS NUMBER(28,10)) AS MARGINAL_LENDING_FACILITY, + CAST(OU_LM_STANDING_FACILITIES.DEPOSIT_FACILITY AS NUMBER(28,10)) AS DEPOSIT_FACILITY, + CAST(OU_LM_STANDING_FACILITIES.COMMENT_ AS VARCHAR2(4000 CHAR)) AS COMMENT_ +FROM + {{ source("ods", "STANDING_FACILITIES_HEADER") }} OU_LM_STANDING_FACILITIES_HEADER, + {{ source("ods", "STANDING_FACILITIES") }} OU_LM_STANDING_FACILITIES +WHERE + OU_LM_STANDING_FACILITIES_HEADER.A_KEY = OU_LM_STANDING_FACILITIES.A_SFH_FK + AND OU_LM_STANDING_FACILITIES_HEADER.A_WORKFLOW_HISTORY_KEY = OU_LM_STANDING_FACILITIES.A_WORKFLOW_HISTORY_KEY + AND OU_LM_STANDING_FACILITIES_HEADER.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_LM_STANDING_FACILITIES", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ.sql new file mode 100644 index 0000000..94761e4 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ.sql @@ -0,0 +1,284 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL"], + alias = "T_MPEC_FULL_SQ", + schema = "MPEC", + meta={ + "author": "Adelina Borisova", + "created": "2025-05-08", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-05-08" + } +) }} + +WITH T_Criterion AS ( + SELECT + T1.A_KEY as Criterion_Key, + LISTAGG(DISTINCT criterion, ',') WITHIN GROUP (ORDER BY criterion) as criterion_cont + FROM ( + SELECT DISTINCT + OU_C2D_MPEC_CONTENT_FULL.ID , + OU_C2D_MPEC_CONTENT_FULL.A_KEY, + OU_C2D_MPEC_CONTENT_CRITERION_FULL.A_WORKFLOW_HISTORY_KEY, + criterion + FROM {{ source("ods","MPEC_CONTENT_FULL")}} OU_C2D_MPEC_CONTENT_FULL + INNER JOIN {{ source("ods","MPEC_CONTENT_CRITERION_FULL")}} OU_C2D_MPEC_CONTENT_CRITERION_FULL + ON OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY = OU_C2D_MPEC_CONTENT_CRITERION_FULL.A_WORKFLOW_HISTORY_KEY + AND OU_C2D_MPEC_CONTENT_FULL.A_KEY = OU_C2D_MPEC_CONTENT_CRITERION_FULL.a_mpec_content_fk + ) T1 + GROUP BY T1.a_key +), +WH AS ( + {{ filter_workflow_history_key( + "w_ODS_C2D_MPEC", + get_main_task_name(model.name), + ", WORKFLOW_START" + ) + }} +) + +SELECT + FIL_REC.A_KEY, + FIL_REC.A_WORKFLOW_HISTORY_KEY, + TRUNC(FIL_REC.WORKFLOW_START) AS WORKFLOW_START, + CAST(FIL_REC.HOST AS VARCHAR2(5 CHAR)) AS HOST, + CAST(FIL_REC.ID AS VARCHAR2(255 CHAR)) AS ID, + CAST(FIL_REC.RIAD_CODE AS VARCHAR2(255 CHAR)) AS RIAD_CODE, + CAST(FIL_REC.MPEC_BIC AS VARCHAR2(255 CHAR)) AS MPEC_BIC, + CAST(FIL_REC.RTGS_ACCESS AS VARCHAR2(1 CHAR)) AS RTGS_ACCESS, + CAST(FIL_REC.INTRADAY_CREDIT_FACILITY AS VARCHAR2(1 CHAR)) AS INTRADAY_CREDIT_FACILITY, + CAST(FIL_REC.MRR_TYPE AS VARCHAR2(30 CHAR)) AS MRR_TYPE, + CAST(FIL_REC.MRR_INTERMEDIARY_HOST AS VARCHAR2(5 CHAR)) AS MRR_INTERMEDIARY_HOST, + CAST(FIL_REC.MRR_INTERMEDIARY_ID AS VARCHAR2(255 CHAR)) AS MRR_INTERMEDIARY_ID, + FIL_REC.MRR_AVG_PROV_SUSP_STDT, + FIL_REC.MRR_AVG_PROV_SUSP_ENDT, + FIL_REC.MRR_EXEMPTION_STDT, + FIL_REC.MRR_EXEMPTION_ENDT, + FIL_REC.MRR_EXEMPTION_REORG_STDT, + FIL_REC.MRR_EXEMPTION_REORG_ENDT, + CAST(FIL_REC.PRUDENTIAL_SUPERVISION AS VARCHAR2(100 CHAR)) AS PRUDENTIAL_SUPERVISION, + CAST(FIL_REC.ELIG_DEPOSIT_FACILITY AS VARCHAR2(1 CHAR)) AS ELIG_DEPOSIT_FACILITY, + FIL_REC.DEPOSIT_FACILITY_SUSP_STDT, + FIL_REC.DEPOSIT_FACILITY_SUSP_ENDT, + FIL_REC.DEPOSIT_FACILITY_EXCL_STDT, + FIL_REC.DEPOSIT_FACILITY_EXCL_ENDT, + FIL_REC.DEPOSIT_FACILITY_LIMIT_STDT, + FIL_REC.DEPOSIT_FACILITY_LIMIT_ENDT, + CAST(FIL_REC.ELIG_MARGINAL_LENDING_FACILITY AS VARCHAR2(1 CHAR)) AS ELIG_MARGINAL_LENDING_FACILITY, + FIL_REC.MARG_LEND_FACILITY_SUSP_STDT, + FIL_REC.MARG_LEND_FACILITY_SUSP_ENDT, + FIL_REC.MARG_LEND_FACILITY_EXCL_STDT, + FIL_REC.MARG_LEND_FACILITY_EXCL_ENDT, + FIL_REC.MARG_LEND_FACILITY_LIMIT_STDT, + FIL_REC.MARG_LEND_FACILITY_LIMIT_ENDT, + CAST(FIL_REC.ELIG_ECB_DEBT_CERTIFICATE AS VARCHAR2(1 CHAR)) AS ELIG_ECB_DEBT_CERTIFICATE, + FIL_REC.ECB_DEBT_CERTIF_SUSP_STDT, + FIL_REC.ECB_DEBT_CERTIF_SUSP_ENDT, + FIL_REC.ECB_DEBT_CERTIF_EXCL_STDT, + FIL_REC.ECB_DEBT_CERTIF_EXCL_ENDT, + FIL_REC.ECB_DEBT_CERTIF_LIMIT_STDT, + FIL_REC.ECB_DEBT_CERTIF_LIMIT_ENDT, + CAST(FIL_REC.ELIG_STD_TENDER_OPERATIONS AS VARCHAR2(1 CHAR)) AS ELIG_STD_TENDER_OPERATIONS, + FIL_REC.STD_TENDER_OPE_SUSP_STDT, + FIL_REC.STD_TENDER_OPE_SUSP_ENDT, + FIL_REC.STD_TENDER_OPE_EXCL_STDT, + FIL_REC.STD_TENDER_OPE_EXCL_ENDT, + FIL_REC.STD_TENDER_OPE_LIMIT_STDT, + FIL_REC.STD_TENDER_OPE_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FTRO_ABSORBING AS VARCHAR2(1 CHAR)) AS ELIG_FTRO_ABSORBING, + FIL_REC.FINE_TUN_REVOPE_ABS_SUSP_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_SUSP_ENDT, + FIL_REC.FINE_TUN_REVOPE_ABS_EXCL_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_EXCL_ENDT, + FIL_REC.FINE_TUN_REVOPE_ABS_LIMIT_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FTRO_PROVIDING AS VARCHAR2(1 CHAR)) AS ELIG_FTRO_PROVIDING, + FIL_REC.FINE_TUN_REVOP_PROV_SUSP_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_SUSP_ENDT, + FIL_REC.FINE_TUN_REVOP_PROV_EXCL_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_EXCL_ENDT, + FIL_REC.FINE_TUN_REVOP_PROV_LIMIT_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FIX_TERM_DEPOSIT AS VARCHAR2(1 CHAR)) AS ELIG_FIX_TERM_DEPOSIT, + FIL_REC.FIX_TERM_DEPOSIT_SUSP_STDT, + FIL_REC.FIX_TERM_DEPOSIT_SUSP_ENDT, + FIL_REC.FIX_TERM_DEPOSIT_EXCL_STDT, + FIL_REC.FIX_TERM_DEPOSIT_EXCL_ENDT, + FIL_REC.FIX_TERM_DEPOSIT_LIMIT_STDT, + FIL_REC.FIX_TERM_DEPOSIT_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FX_SWAP_ABSORBING AS VARCHAR2(1 CHAR)) AS ELIG_FX_SWAP_ABSORBING, + FIL_REC.FX_SWAP_ABS_SUSP_STDT, + FIL_REC.FX_SWAP_ABS_SUSP_ENDT, + FIL_REC.FX_SWAP_ABS_EXCL_STDT, + FIL_REC.FX_SWAP_ABS_EXCL_ENDT, + FIL_REC.FX_SWAP_ABS_LIMIT_STDT, + FIL_REC.FX_SWAP_ABS_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FX_SWAP_PROVIDING AS VARCHAR2(1 CHAR)) AS ELIG_FX_SWAP_PROVIDING, + FIL_REC.FX_SWAP_PROV_SUSP_STDT, + FIL_REC.FX_SWAP_PROV_SUSP_ENDT, + FIL_REC.FX_SWAP_PROV_EXCL_STDT, + FIL_REC.FX_SWAP_PROV_EXCL_ENDT, + FIL_REC.FX_SWAP_PROV_LIMIT_STDT, + FIL_REC.FX_SWAP_PROV_LIMIT_ENDT, + FIL_REC.ECB_ENTRY_DATE, + CAST(FIL_REC.STATUS AS VARCHAR2(10 CHAR)) AS STATUS, + CAST(FIL_REC.ACTION AS VARCHAR2(1 CHAR)) AS ACTION, + CAST(FIL_REC.USD_OPERATIONS AS VARCHAR2(1 CHAR)) AS USD_OPERATIONS, + CAST(FIL_REC.DELETION_REASON AS VARCHAR2(30 CHAR)) AS DELETION_REASON, + CAST(FIL_REC.NCB_COMMENT AS VARCHAR2(1 CHAR)) AS NCB_COMMENT, + --FIL_REC.CREATION_TIME , + CAST(FIL_REC.VERIFIED_OPERATIONAL_CRITERIA AS VARCHAR2(255 CHAR)) AS VERIFIED_OPERATIONAL_CRITERIA, + CAST(FIL_REC.CLM_ACCESS AS VARCHAR2(1 CHAR)) AS CLM_ACCESS, + FIL_REC.MAX_KEY + --,FIL_REC.COUNT_KEYS +FROM ( + SELECT /*+ PARALLEL(OU_C2D_MPEC_CONTENT_FULL,4) */ + ROW_NUMBER() OVER (PARTITION BY OU_C2D_MPEC_CONTENT_FULL.HOST, OU_C2D_MPEC_CONTENT_FULL.ID,OU_C2D_MPEC_CONTENT_FULL.ACTION, OU_C2D_MPEC_CONTENT_FULL.ECB_ENTRY_DATE,OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY ORDER BY OU_C2D_MPEC_CONTENT_FULL.A_KEY desc) AS MX_ROW_NUM, + OU_C2D_MPEC_CONTENT_FULL.A_KEY, + OU_C2D_MPEC_CONTENT_FULL.HOST, + OU_C2D_MPEC_CONTENT_FULL.ID, + OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY, + CONCAT(OU_C2D_MPEC_CONTENT_FULL.HOST, OU_C2D_MPEC_CONTENT_FULL.ID) as RIAD_CODE, + OU_C2D_MPEC_CONTENT_FULL.MPEC_BIC, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.RTGS_ACCESS) IS NULL THEN '0' + ELSE '1' + END RTGS_ACCESS, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.INTRADAY_CREDIT_FACILITY) IS NULL THEN '0' + ELSE '1' + END INTRADAY_CREDIT_FACILITY, + OU_C2D_MPEC_CONTENT_FULL.MRR_TYPE, + OU_C2D_MPEC_CONTENT_FULL.MRR_INTERMEDIARY_HOST, + OU_C2D_MPEC_CONTENT_FULL.MRR_INTERMEDIARY_ID, + OU_C2D_MPEC_CONTENT_FULL.MRR_AVG_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.MRR_AVG_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.MRR_EXEMPTION_STDT, + OU_C2D_MPEC_CONTENT_FULL.MRR_EXEMPTION_ENDT, + OU_C2D_MPEC_CONTENT_FULL.MRR_EXEMPTION_REORG_STDT, + OU_C2D_MPEC_CONTENT_FULL.MRR_EXEMPTION_REORG_ENDT, + OU_C2D_MPEC_CONTENT_FULL.PRUDENTIAL_SUPERVISION, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_DEPOSIT_FACILITY) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_DEPOSIT_FACILITY, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.DEPOSIT_FACILITY_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_MARGINAL_LENDING_FACILITY) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_MARGINAL_LENDING_FACILITY, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.MARG_LEND_FACILITY_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_ECB_DEBT_CERTIFICATE) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_ECB_DEBT_CERTIFICATE, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_DEBT_CERTIF_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_STD_TENDER_OPERATIONS) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_STD_TENDER_OPERATIONS, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.STD_TENDER_OPE_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_FTRO_ABSORBING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FTRO_ABSORBING, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOPE_ABS_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_FTRO_PROVIDING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FTRO_PROVIDING, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.FINE_TUN_REVOP_PROV_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_FIX_TERM_DEPOSIT) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FIX_TERM_DEPOSIT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.FIX_TERM_DEPOSIT_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_FX_SWAP_ABSORBING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FX_SWAP_ABSORBING, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_ABS_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.ELIG_FX_SWAP_PROVIDING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FX_SWAP_PROVIDING, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_EXCL_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_EXCL_ENDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_LIMIT_STDT, + OU_C2D_MPEC_CONTENT_FULL.FX_SWAP_PROV_LIMIT_ENDT, + OU_C2D_MPEC_CONTENT_FULL.ECB_ENTRY_DATE, + OU_C2D_MPEC_CONTENT_FULL.STATUS, + OU_C2D_MPEC_CONTENT_FULL.ACTION, + OU_C2D_MPEC_CONTENT_FULL.USD_OPERATIONS, + OU_C2D_MPEC_CONTENT_FULL.DELETION_REASON, + OU_C2D_MPEC_CONTENT_FULL.NCB_COMMENT, + --OU_C2D_MPEC_ADMIN.CREATION_TIME , + T_Criterion.criterion_cont as VERIFIED_OPERATIONAL_CRITERIA , + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT_FULL.CLM_ACCESS) IS NULL THEN '0' + ELSE '1' + END CLM_ACCESS, + --OU_C2D_MPEC_CONTENT_FULL.CLM_ACCESS, + WH.WORKFLOW_START, + CASE + WHEN OU_C2D_MPEC_CONTENT_FULL.ACTION = 'N' THEN 1 + WHEN OU_C2D_MPEC_CONTENT_FULL.ACTION = 'U' THEN 2 + WHEN OU_C2D_MPEC_CONTENT_FULL.ACTION = 'D' THEN 3 + END AS ACTION_ORDER, + MAX(OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY) OVER () AS MAX_KEY + --,COUNT(DISTINCT OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY) OVER () AS COUNT_KEYS + FROM + {{ source("ods","MPEC_CONTENT_FULL")}} OU_C2D_MPEC_CONTENT_FULL + --inner join OU_C2D_MPEC_ADMIN on OU_C2D_MPEC_ADMIN.A_ETL_LOAD_SET_FK = OU_C2D_MPEC_CONTENT_FULL.A_ETL_LOAD_SET_FK + LEFT OUTER JOIN T_Criterion + ON T_Criterion.Criterion_Key = OU_C2D_MPEC_CONTENT_FULL.A_KEY + INNER JOIN WH + ON OU_C2D_MPEC_CONTENT_FULL.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY + --INNER JOIN CT_ODS_A_LOAD_HISTORY on OU_C2D_MPEC_CONTENT_FULL.A_ETL_LOAD_SET_FK=CT_ODS_A_LOAD_HISTORY.A_ETL_LOAD_SET_KEY +) FIL_REC +WHERE + MX_ROW_NUM=1 +ORDER BY + RIAD_CODE, A_KEY asc \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ.sql new file mode 100644 index 0000000..69de0cb --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ.sql @@ -0,0 +1,294 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT"], + alias = "T_MPEC_SQ", + schema = "MPEC", + meta={ + "author": "Adelina Borisova", + "created": "2025-04-25", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-04-25" + } +) }} + + + +WITH T_Criterion AS ( + SELECT + T1.A_KEY as Criterion_Key, + LISTAGG(criterion, ',') WITHIN GROUP (ORDER BY criterion) as criterion_cont + FROM ( + SELECT + OU_C2D_MPEC_CONTENT.ID , + OU_C2D_MPEC_CONTENT.A_KEY, + OU_C2D_MPEC_CONTENT_CRITERION.A_WORKFLOW_HISTORY_KEY, + criterion + FROM {{ source("ods","MPEC_CONTENT")}} OU_C2D_MPEC_CONTENT + INNER JOIN {{ source("ods","MPEC_CONTENT_CRITERION")}} OU_C2D_MPEC_CONTENT_CRITERION + ON OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY = OU_C2D_MPEC_CONTENT_CRITERION.A_WORKFLOW_HISTORY_KEY + AND OU_C2D_MPEC_CONTENT.A_KEY = OU_C2D_MPEC_CONTENT_CRITERION.a_mpec_content_fk + ) T1 + GROUP BY T1.a_key +), +ETL_LAOD_KEY AS ( + SELECT + MIN(OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY) MIN_ETL_LOAD_KEY + FROM {{ source("ods","MPEC_CONTENT")}} OU_C2D_MPEC_CONTENT + --INNER JOIN {{ source("ou_c2d","MPEC_ADMIN")}} OU_C2D_MPEC_ADMIN + -- ON OU_C2D_MPEC_ADMIN.A_WORKFLOW_HISTORY_KEY = OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY + WHERE + OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY + IN ( {{ filter_workflow_history_key( + "w_ODS_C2D_MPEC", + get_main_task_name(model.name) + ) }} + ) + +) +SELECT + FIL_REC.A_KEY, + FIL_REC.A_WORKFLOW_HISTORY_KEY, + FIL_REC.ECB_ENTRY_DATE as VALID_FROM, + CASE + WHEN action='D' THEN ECB_ENTRY_DATE + ELSE NVL(lead(ECB_ENTRY_DATE) OVER (PARTITION BY FIL_REC.host, FIL_REC.id ORDER BY ECB_ENTRY_DATE),TO_DATE('31129999', 'dd-mm-yy')) + END AS VALID_TO, + ROW_NUMBER() OVER (PARTITION BY FIL_REC.host, FIL_REC.id ORDER BY ECB_ENTRY_DATE, FIL_REC.ACTION_ORDER) AS row_num2, + COUNT(A_KEY) OVER (PARTITION BY FIL_REC.host, FIL_REC.id) AS max_row_num3, + CAST(FIL_REC.HOST AS VARCHAR2(5 CHAR)) AS HOST, + CAST(FIL_REC.ID AS VARCHAR2(255 CHAR)) AS ID, + CAST(FIL_REC.RIAD_CODE AS VARCHAR2(255 CHAR)) AS RIAD_CODE, + CAST(FIL_REC.MPEC_BIC AS VARCHAR2(255 CHAR)) AS MPEC_BIC, + CAST(FIL_REC.RTGS_ACCESS AS VARCHAR2(1 CHAR)) AS RTGS_ACCESS, + CAST(FIL_REC.INTRADAY_CREDIT_FACILITY AS VARCHAR2(1 CHAR)) AS INTRADAY_CREDIT_FACILITY, + CAST(FIL_REC.MRR_TYPE AS VARCHAR2(30 CHAR)) AS MRR_TYPE, + CAST(FIL_REC.MRR_INTERMEDIARY_HOST AS VARCHAR2(5 CHAR)) AS MRR_INTERMEDIARY_HOST, + CAST(FIL_REC.MRR_INTERMEDIARY_ID AS VARCHAR2(255 CHAR)) AS MRR_INTERMEDIARY_ID, + FIL_REC.MRR_AVG_PROV_SUSP_STDT, + FIL_REC.MRR_AVG_PROV_SUSP_ENDT, + FIL_REC.MRR_EXEMPTION_STDT, + FIL_REC.MRR_EXEMPTION_ENDT, + FIL_REC.MRR_EXEMPTION_REORG_STDT, + FIL_REC.MRR_EXEMPTION_REORG_ENDT, + CAST(FIL_REC.PRUDENTIAL_SUPERVISION AS VARCHAR2(100 CHAR)) AS PRUDENTIAL_SUPERVISION, + CAST(FIL_REC.ELIG_DEPOSIT_FACILITY AS VARCHAR2(1 CHAR)) AS ELIG_DEPOSIT_FACILITY, + FIL_REC.DEPOSIT_FACILITY_SUSP_STDT, + FIL_REC.DEPOSIT_FACILITY_SUSP_ENDT, + FIL_REC.DEPOSIT_FACILITY_EXCL_STDT, + FIL_REC.DEPOSIT_FACILITY_EXCL_ENDT, + FIL_REC.DEPOSIT_FACILITY_LIMIT_STDT, + FIL_REC.DEPOSIT_FACILITY_LIMIT_ENDT, + CAST(FIL_REC.ELIG_MARGINAL_LENDING_FACILITY AS VARCHAR2(1 CHAR)) AS ELIG_MARGINAL_LENDING_FACILITY, + FIL_REC.MARG_LEND_FACILITY_SUSP_STDT, + FIL_REC.MARG_LEND_FACILITY_SUSP_ENDT, + FIL_REC.MARG_LEND_FACILITY_EXCL_STDT, + FIL_REC.MARG_LEND_FACILITY_EXCL_ENDT, + FIL_REC.MARG_LEND_FACILITY_LIMIT_STDT, + FIL_REC.MARG_LEND_FACILITY_LIMIT_ENDT, + CAST(FIL_REC.ELIG_ECB_DEBT_CERTIFICATE AS VARCHAR2(1 CHAR)) AS ELIG_ECB_DEBT_CERTIFICATE, + FIL_REC.ECB_DEBT_CERTIF_SUSP_STDT, + FIL_REC.ECB_DEBT_CERTIF_SUSP_ENDT, + FIL_REC.ECB_DEBT_CERTIF_EXCL_STDT, + FIL_REC.ECB_DEBT_CERTIF_EXCL_ENDT, + FIL_REC.ECB_DEBT_CERTIF_LIMIT_STDT, + FIL_REC.ECB_DEBT_CERTIF_LIMIT_ENDT, + CAST(FIL_REC.ELIG_STD_TENDER_OPERATIONS AS VARCHAR2(1 CHAR)) AS ELIG_STD_TENDER_OPERATIONS, + FIL_REC.STD_TENDER_OPE_SUSP_STDT, + FIL_REC.STD_TENDER_OPE_SUSP_ENDT, + FIL_REC.STD_TENDER_OPE_EXCL_STDT, + FIL_REC.STD_TENDER_OPE_EXCL_ENDT, + FIL_REC.STD_TENDER_OPE_LIMIT_STDT, + FIL_REC.STD_TENDER_OPE_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FTRO_ABSORBING AS VARCHAR2(1 CHAR)) AS ELIG_FTRO_ABSORBING, + FIL_REC.FINE_TUN_REVOPE_ABS_SUSP_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_SUSP_ENDT, + FIL_REC.FINE_TUN_REVOPE_ABS_EXCL_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_EXCL_ENDT, + FIL_REC.FINE_TUN_REVOPE_ABS_LIMIT_STDT, + FIL_REC.FINE_TUN_REVOPE_ABS_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FTRO_PROVIDING AS VARCHAR2(1 CHAR)) AS ELIG_FTRO_PROVIDING, + FIL_REC.FINE_TUN_REVOP_PROV_SUSP_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_SUSP_ENDT, + FIL_REC.FINE_TUN_REVOP_PROV_EXCL_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_EXCL_ENDT, + FIL_REC.FINE_TUN_REVOP_PROV_LIMIT_STDT, + FIL_REC.FINE_TUN_REVOP_PROV_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FIX_TERM_DEPOSIT AS VARCHAR2(1 CHAR)) AS ELIG_FIX_TERM_DEPOSIT, + FIL_REC.FIX_TERM_DEPOSIT_SUSP_STDT, + FIL_REC.FIX_TERM_DEPOSIT_SUSP_ENDT, + FIL_REC.FIX_TERM_DEPOSIT_EXCL_STDT, + FIL_REC.FIX_TERM_DEPOSIT_EXCL_ENDT, + FIL_REC.FIX_TERM_DEPOSIT_LIMIT_STDT, + FIL_REC.FIX_TERM_DEPOSIT_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FX_SWAP_ABSORBING AS VARCHAR2(1 CHAR)) AS ELIG_FX_SWAP_ABSORBING, + FIL_REC.FX_SWAP_ABS_SUSP_STDT, + FIL_REC.FX_SWAP_ABS_SUSP_ENDT, + FIL_REC.FX_SWAP_ABS_EXCL_STDT, + FIL_REC.FX_SWAP_ABS_EXCL_ENDT, + FIL_REC.FX_SWAP_ABS_LIMIT_STDT, + FIL_REC.FX_SWAP_ABS_LIMIT_ENDT, + CAST(FIL_REC.ELIG_FX_SWAP_PROVIDING AS VARCHAR2(1 CHAR)) AS ELIG_FX_SWAP_PROVIDING, + FIL_REC.FX_SWAP_PROV_SUSP_STDT, + FIL_REC.FX_SWAP_PROV_SUSP_ENDT, + FIL_REC.FX_SWAP_PROV_EXCL_STDT, + FIL_REC.FX_SWAP_PROV_EXCL_ENDT, + FIL_REC.FX_SWAP_PROV_LIMIT_STDT, + FIL_REC.FX_SWAP_PROV_LIMIT_ENDT, + FIL_REC.ECB_ENTRY_DATE, + CAST(FIL_REC.STATUS AS VARCHAR2(10 CHAR)) AS STATUS, + CAST(FIL_REC.ACTION AS VARCHAR2(1 CHAR)) AS ACTION, + CAST(FIL_REC.USD_OPERATIONS AS VARCHAR2(1 CHAR)) AS USD_OPERATIONS, + CAST(FIL_REC.DELETION_REASON AS VARCHAR2(30 CHAR)) AS DELETION_REASON, + CAST(FIL_REC.NCB_COMMENT AS VARCHAR2(1 CHAR)) AS NCB_COMMENT, + FIL_REC.CREATION_TIME, + CAST(FIL_REC.VERIFIED_OPERATIONAL_CRITERIA AS VARCHAR2(255 CHAR)) AS VERIFIED_OPERATIONAL_CRITERIA, + CAST(FIL_REC.CLM_ACCESS AS VARCHAR2(1 CHAR)) AS CLM_ACCESS +FROM ( + SELECT /*+ PARALLEL(OU_C2D_MPEC_CONTENT,4) */ + ROW_NUMBER() OVER (PARTITION BY OU_C2D_MPEC_CONTENT.HOST, OU_C2D_MPEC_CONTENT.ID,OU_C2D_MPEC_CONTENT.ACTION, OU_C2D_MPEC_CONTENT.ECB_ENTRY_DATE,OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY ORDER BY OU_C2D_MPEC_CONTENT.A_KEY desc) AS MX_ROW_NUM, + OU_C2D_MPEC_CONTENT.A_KEY, + OU_C2D_MPEC_CONTENT.HOST, + OU_C2D_MPEC_CONTENT.ID, + OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY, + CONCAT(OU_C2D_MPEC_CONTENT.HOST, OU_C2D_MPEC_CONTENT.ID) as RIAD_CODE, + OU_C2D_MPEC_CONTENT.MPEC_BIC, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.RTGS_ACCESS) IS NULL THEN '0' + ELSE '1' + END RTGS_ACCESS, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.INTRADAY_CREDIT_FACILITY) IS NULL THEN '0' + ELSE '1' + END INTRADAY_CREDIT_FACILITY, + OU_C2D_MPEC_CONTENT.MRR_TYPE, + OU_C2D_MPEC_CONTENT.MRR_INTERMEDIARY_HOST, + OU_C2D_MPEC_CONTENT.MRR_INTERMEDIARY_ID, + OU_C2D_MPEC_CONTENT.MRR_AVG_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT.MRR_AVG_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.MRR_EXEMPTION_STDT, + OU_C2D_MPEC_CONTENT.MRR_EXEMPTION_ENDT, + OU_C2D_MPEC_CONTENT.MRR_EXEMPTION_REORG_STDT, + OU_C2D_MPEC_CONTENT.MRR_EXEMPTION_REORG_ENDT, + OU_C2D_MPEC_CONTENT.PRUDENTIAL_SUPERVISION, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_DEPOSIT_FACILITY) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_DEPOSIT_FACILITY, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_SUSP_STDT, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_EXCL_STDT, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.DEPOSIT_FACILITY_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_MARGINAL_LENDING_FACILITY) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_MARGINAL_LENDING_FACILITY, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_SUSP_STDT, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_EXCL_STDT, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.MARG_LEND_FACILITY_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_ECB_DEBT_CERTIFICATE) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_ECB_DEBT_CERTIFICATE, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_SUSP_STDT, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_EXCL_STDT, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.ECB_DEBT_CERTIF_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_STD_TENDER_OPERATIONS) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_STD_TENDER_OPERATIONS, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_SUSP_STDT, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_EXCL_STDT, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.STD_TENDER_OPE_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_FTRO_ABSORBING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FTRO_ABSORBING, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_SUSP_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_EXCL_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOPE_ABS_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_FTRO_PROVIDING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FTRO_PROVIDING, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_EXCL_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.FINE_TUN_REVOP_PROV_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_FIX_TERM_DEPOSIT) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FIX_TERM_DEPOSIT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_SUSP_STDT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_EXCL_STDT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.FIX_TERM_DEPOSIT_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_FX_SWAP_ABSORBING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FX_SWAP_ABSORBING, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_SUSP_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_EXCL_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_ABS_LIMIT_ENDT, + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.ELIG_FX_SWAP_PROVIDING) IS NULL THEN 'N' + ELSE 'Y' + END ELIG_FX_SWAP_PROVIDING, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_SUSP_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_SUSP_ENDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_EXCL_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_EXCL_ENDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_LIMIT_STDT, + OU_C2D_MPEC_CONTENT.FX_SWAP_PROV_LIMIT_ENDT, + OU_C2D_MPEC_CONTENT.ECB_ENTRY_DATE, + OU_C2D_MPEC_CONTENT.STATUS, + OU_C2D_MPEC_CONTENT.ACTION, + OU_C2D_MPEC_CONTENT.USD_OPERATIONS, + OU_C2D_MPEC_CONTENT.DELETION_REASON, + OU_C2D_MPEC_CONTENT.NCB_COMMENT, + CAST(OU_C2D_MPEC_ADMIN.CREATION_TIME AS TIMESTAMP) AS CREATION_TIME, + T_Criterion.criterion_cont as VERIFIED_OPERATIONAL_CRITERIA , + CASE + WHEN TRIM(OU_C2D_MPEC_CONTENT.CLM_ACCESS) IS NULL THEN '0' + ELSE '1' + END CLM_ACCESS, + CASE + WHEN OU_C2D_MPEC_CONTENT.ACTION = 'N' THEN 1 + WHEN OU_C2D_MPEC_CONTENT.ACTION = 'U' THEN 2 + WHEN OU_C2D_MPEC_CONTENT.ACTION = 'D' THEN 3 + END AS ACTION_ORDER + FROM + {{ source("ods","MPEC_CONTENT")}} OU_C2D_MPEC_CONTENT + INNER JOIN + {{ source("ods","MPEC_ADMIN")}} OU_C2D_MPEC_ADMIN + ON OU_C2D_MPEC_ADMIN.A_WORKFLOW_HISTORY_KEY = OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY + LEFT OUTER JOIN T_Criterion + ON T_Criterion.Criterion_Key = OU_C2D_MPEC_CONTENT.A_KEY + INNER JOIN ETL_LAOD_KEY + ON OU_C2D_MPEC_CONTENT.A_WORKFLOW_HISTORY_KEY = ETL_LAOD_KEY.MIN_ETL_LOAD_KEY + + ) FIL_REC +WHERE + MX_ROW_NUM=1 +ORDER BY + RIAD_CODE, A_KEY ASC \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ.sql new file mode 100644 index 0000000..1137d8a --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL"], + alias = "T_MPEC_MID_FULL_SQ", + schema = "MPEC", + meta={ + "author": "Adelina Borisova", + "created": "2025-05-14", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-05-14" + } +) }} + +SELECT +OU_C2D_MPEC_MID_FULL.A_KEY, +OU_C2D_MPEC_MID_FULL.A_WORKFLOW_HISTORY_KEY, +CAST(OU_C2D_MPEC_MID_FULL.RIAD_CODE AS VARCHAR2(255 CHAR)) AS RIAD_CODE, +CAST(OU_C2D_MPEC_MID_FULL.BIC AS VARCHAR2(255 CHAR)) AS BIC, +CAST(OU_C2D_MPEC_MID_FULL.COUNTRY_OF_REGISTRATION AS VARCHAR2(2 CHAR)) AS COUNTRY_OF_REGISTRATION, +CAST(OU_C2D_MPEC_MID_FULL.NAME AS VARCHAR2(255 CHAR)) AS NAME, +CAST(OU_C2D_MPEC_MID_FULL.BOX AS VARCHAR2(255 CHAR)) AS BOX, +CAST(OU_C2D_MPEC_MID_FULL.ADDRESS AS VARCHAR2(255 CHAR)) AS ADDRESS, +CAST(OU_C2D_MPEC_MID_FULL.POSTAL AS VARCHAR2(255 CHAR)) AS POSTAL, +CAST(OU_C2D_MPEC_MID_FULL.CITY AS VARCHAR2(255 CHAR)) AS CITY, +CAST(OU_C2D_MPEC_MID_FULL.CATEGORY AS VARCHAR2(255 CHAR)) AS CATEGORY, +CAST(OU_C2D_MPEC_MID_FULL.HEAD_COUNTRY_OF_REGISTRATION AS VARCHAR2(2 CHAR)) AS HEAD_COUNTRY_OF_REGISTRATION, +CAST(OU_C2D_MPEC_MID_FULL.HEAD_NAME AS VARCHAR2(255 CHAR)) AS HEAD_NAME, +CAST(OU_C2D_MPEC_MID_FULL.HEAD_RIAD_CODE AS VARCHAR2(255 CHAR)) AS HEAD_RIAD_CODE, +CAST(OU_C2D_MPEC_MID_FULL.RESERVE AS VARCHAR2(1 CHAR)) AS RESERVE, +CAST(OU_C2D_MPEC_MID_FULL.EXEMPT AS VARCHAR2(1 CHAR)) AS EXEMPT, +TRUNC(WH.WORKFLOW_START) AS WORKFLOW_START +FROM + {{ source("ods","MPEC_MID_FULL")}} OU_C2D_MPEC_MID_FULL +INNER JOIN + {{ filter_workflow_history_key( + "w_ODS_C2D_MPEC_MID_FULL", + get_main_task_name(model.name), + ", WORKFLOW_START" + ) }} WH + ON OU_C2D_MPEC_MID_FULL.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql new file mode 100644 index 0000000..e24a784 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK"], + alias = "T_CLSLNK_PSTNZ_CURRENT_SQ", + schema = "RIAD", +) }} + +SELECT + ou_riad_riad_close_link.A_KEY, + ou_riad_riad_close_link.A_WORKFLOW_HISTORY_KEY, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_1, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_2, + ou_riad_riad_close_link.CLSNK_PSTN, + ou_riad_riad_close_link.CLSLNK_SNPSHT_DT, + ou_riad_riad_close_link.CLSLNK_CHLD_ENTTY_RIAD_ID, + ou_riad_riad_close_link.CLSLNK_PRNT_ENTTY_RIAD_ID, + ou_riad_riad_close_link.SHARE_OWNERSHIP, + ou_riad_riad_close_link.PATH_ID +FROM + {{ source("ou_riad", "RIAD_CLOSE_LINK") }} ou_riad_riad_close_link +WHERE + ou_riad_riad_close_link.CLSNK_PSTN = 0 + AND ou_riad_riad_close_link.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_max_key( + "w_ODS_RIAD_CLOSE_LINKS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ.sql new file mode 100644 index 0000000..46b9934 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK"], + alias = "T_CLSLNK_PSTNZ_HIST_ADHOCK_SQ", + schema = "RIAD", +) }} + +SELECT + ou_riad_riad_close_link_adhoc.A_KEY, + ou_riad_riad_close_link_adhoc.A_WORKFLOW_HISTORY_KEY, + ou_riad_riad_close_link_adhoc.CLSLNK_ENTTY_RIAD_ID_1, + ou_riad_riad_close_link_adhoc.CLSLNK_ENTTY_RIAD_ID_2, + ou_riad_riad_close_link_adhoc.CLSNK_PSTN, + ou_riad_riad_close_link_adhoc.CLSLNK_SNPSHT_DT, + ou_riad_riad_close_link_adhoc.CLSLNK_CHLD_ENTTY_RIAD_ID, + ou_riad_riad_close_link_adhoc.CLSLNK_PRNT_ENTTY_RIAD_ID, + ou_riad_riad_close_link_adhoc.SHARE_OWNERSHIP, + ou_riad_riad_close_link_adhoc.PATH_ID +FROM + {{ source("ou_riad", "RIAD_CLOSE_LINK_ADHOC") }} ou_riad_riad_close_link_adhoc + WHERE +ou_riad_riad_close_link_adhoc.CLSNK_PSTN = 0 and +A_WORKFLOW_HISTORY_KEY IN +{{ filter_workflow_history_key("w_ODS_RIAD_CLOSE_LINKS_ADHOC",get_main_task_name(model.name)) }} +UNION ALL + SELECT + ou_riad_riad_close_link_adhoc.A_KEY, + ou_riad_riad_close_link_adhoc.A_WORKFLOW_HISTORY_KEY, + ou_riad_riad_close_link_adhoc.CLSLNK_ENTTY_RIAD_ID_2 AS CLSLNK_ENTTY_RIAD_ID_1, + ou_riad_riad_close_link_adhoc.CLSLNK_ENTTY_RIAD_ID_1 AS CLSLNK_ENTTY_RIAD_ID_2, + ou_riad_riad_close_link_adhoc.CLSNK_PSTN, + ou_riad_riad_close_link_adhoc.CLSLNK_SNPSHT_DT, + ou_riad_riad_close_link_adhoc.CLSLNK_CHLD_ENTTY_RIAD_ID, + ou_riad_riad_close_link_adhoc.CLSLNK_PRNT_ENTTY_RIAD_ID, + ou_riad_riad_close_link_adhoc.SHARE_OWNERSHIP, + ou_riad_riad_close_link_adhoc.PATH_ID +FROM + {{ source("ou_riad", "RIAD_CLOSE_LINK_ADHOC") }} ou_riad_riad_close_link_adhoc + WHERE +ou_riad_riad_close_link_adhoc.CLSNK_PSTN = 0 and +A_WORKFLOW_HISTORY_KEY IN +{{ filter_workflow_history_key("w_ODS_RIAD_CLOSE_LINKS_ADHOC",get_main_task_name(model.name) ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql new file mode 100644 index 0000000..e617f17 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK"], + alias = "T_CLSLNK_PSTNZ_HIST_SQ", + schema = "RIAD", +) }} + +WITH ou_riad_riad_close_link AS ( + SELECT + ou_riad_riad_close_link.A_KEY, + ou_riad_riad_close_link.A_WORKFLOW_HISTORY_KEY, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_1 AS CLSLNK_ENTTY_RIAD_ID_1, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_2 AS CLSLNK_ENTTY_RIAD_ID_2, + ou_riad_riad_close_link.CLSNK_PSTN, + ou_riad_riad_close_link.CLSLNK_SNPSHT_DT, + ou_riad_riad_close_link.CLSLNK_CHLD_ENTTY_RIAD_ID, + ou_riad_riad_close_link.CLSLNK_PRNT_ENTTY_RIAD_ID, + ou_riad_riad_close_link.SHARE_OWNERSHIP, + ou_riad_riad_close_link.PATH_ID + FROM + {{ source("ou_riad", "RIAD_CLOSE_LINK") }} ou_riad_riad_close_link + WHERE + ou_riad_riad_close_link.CLSNK_PSTN = 0 and + {{ filter_workflow_history_max_key( + "w_ODS_RIAD_CLOSE_LINKS", + get_main_task_name(model.name) + ) }} + UNION ALL + SELECT + ou_riad_riad_close_link.A_KEY, + ou_riad_riad_close_link.A_WORKFLOW_HISTORY_KEY, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_2 AS CLSLNK_ENTTY_RIAD_ID_1, + ou_riad_riad_close_link.CLSLNK_ENTTY_RIAD_ID_1 AS CLSLNK_ENTTY_RIAD_ID_2, + ou_riad_riad_close_link.CLSNK_PSTN, + ou_riad_riad_close_link.CLSLNK_SNPSHT_DT, + ou_riad_riad_close_link.CLSLNK_CHLD_ENTTY_RIAD_ID, + ou_riad_riad_close_link.CLSLNK_PRNT_ENTTY_RIAD_ID, + ou_riad_riad_close_link.SHARE_OWNERSHIP, + ou_riad_riad_close_link.PATH_ID + FROM + {{ source("ou_riad", "RIAD_CLOSE_LINK") }} ou_riad_riad_close_link + WHERE + ou_riad_riad_close_link.CLSNK_PSTN = 0 and + {{ filter_workflow_history_max_key( + "w_ODS_RIAD_CLOSE_LINKS", + get_main_task_name(model.name) + ) }} + ) + SELECT * FROM ou_riad_riad_close_link + LEFT JOIN +(SELECT + DISTINCT CLSLNK_SNPSHT_DT as CLSLNK_SNPSHT_DT_mopdb + FROM + RIAD.T_CLSLNK_PSTNZ_HIST + ORDER BY + CLSLNK_SNPSHT_DT DESC + FETCH FIRST + 30 ROWS ONLY + ) clslnk_snpsht_dt +ON clslnk_snpsht_dt.CLSLNK_SNPSHT_DT_mopdb = ou_riad_riad_close_link.CLSLNK_SNPSHT_DT +WHERE +--ISNULL(CLSLNK_SNPSHT_DT_mopdb) +clslnk_snpsht_dt.CLSLNK_SNPSHT_DT_mopdb IS NULL \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql new file mode 100644 index 0000000..544eaec --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = "table", + schema = "RIAD", + alias = "T_ENTTY_C2D_CD_HIST_SQ", + tags = ["m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D"], +) }} + +with load_history as ( + {{ filter_workflow_history_max_key("w_ODS_RIAD_ENTTY_FLTTND_C2D",get_main_task_name(model.name))}} +) + +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, RIAD AS entty_cd, 'RIAD' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where RIAD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, AVID AS entty_cd, 'AVID' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where AVID is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, BIC AS entty_cd, 'BIC' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where BIC is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, LEI AS entty_cd, 'LEI' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where LEI is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, NCB AS entty_cd, 'NCB' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where NCB is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, UCDB_CD AS entty_cd, 'UCDB_CD' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where UCDB_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, TMS_CD AS entty_cd, 'TMS_CD' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where TMS_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, BLMBRG_CD AS entty_cd, 'BLMBRG_CD' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where BLMBRG_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, C2D_CD AS entty_cd, 'C2D_CD' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where C2D_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, EACODELIST AS entty_cd, 'EACODELIST' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where EACODELIST is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, FVC AS entty_cd, 'FVC' AS typ_entty_cd, entry_date, bsnss_vld_frm, bsnss_vld_t, vrsn_vld_frm, vrsn_vld_t + from + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} where FVC is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql new file mode 100644 index 0000000..f9a599d --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ.sql @@ -0,0 +1,70 @@ +{{ config( + materialized = "table", + schema = "RIAD", + alias = "T_ENTTY_C2D_HIST_SQ", + tags = ["m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D"], +) }} +SELECT +OU_RIAD_ENTTY_FLTTND_C2D.A_KEY, +OU_RIAD_ENTTY_FLTTND_C2D.A_WORKFLOW_HISTORY_KEY, +OU_RIAD_ENTTY_FLTTND_C2D.ENTTY_RIAD_ID, +OU_RIAD_ENTTY_FLTTND_C2D.BSNSS_VLD_FRM, +OU_RIAD_ENTTY_FLTTND_C2D.BSNSS_VLD_T, +OU_RIAD_ENTTY_FLTTND_C2D.CNTRY, +OU_RIAD_ENTTY_FLTTND_C2D.DT_BRTH, +OU_RIAD_ENTTY_FLTTND_C2D.DT_CLS, +OU_RIAD_ENTTY_FLTTND_C2D.ENTTY_RIAD_CD, +OU_RIAD_ENTTY_FLTTND_C2D.NM_ENTTY, +OU_RIAD_ENTTY_FLTTND_C2D.ECNMC_ACTVTY, +OU_RIAD_ENTTY_FLTTND_C2D.INSTTTNL_SCTR, +OU_RIAD_ENTTY_FLTTND_C2D.INSTTTNL_SCTR_DTL, +OU_RIAD_ENTTY_FLTTND_C2D.NMBR_EMPLYS, +OU_RIAD_ENTTY_FLTTND_C2D.ENTRPRS_SZ, +OU_RIAD_ENTTY_FLTTND_C2D.TYP_BNKNG_LCNS, +OU_RIAD_ENTTY_FLTTND_C2D.LGL_FRM, +OU_RIAD_ENTTY_FLTTND_C2D.PSTL_BX_CD, +OU_RIAD_ENTTY_FLTTND_C2D.STRT, +OU_RIAD_ENTTY_FLTTND_C2D.PSTL_CD, +OU_RIAD_ENTTY_FLTTND_C2D.CTY, +OU_RIAD_ENTTY_FLTTND_C2D.IS_INCTV, +OU_RIAD_ENTTY_FLTTND_C2D.IS_UCITS_CMPLNT, +OU_RIAD_ENTTY_FLTTND_C2D.INTRNTNL_ORGNSTN_CD, +OU_RIAD_ENTTY_FLTTND_C2D.BLNC_SHT_TTL_DMSTC, +OU_RIAD_ENTTY_FLTTND_C2D.BLNC_SHT_TTL, +OU_RIAD_ENTTY_FLTTND_C2D.BLNC_SHT_TTL_GRP, +OU_RIAD_ENTTY_FLTTND_C2D.NT_ASSTS_DMSTC, +OU_RIAD_ENTTY_FLTTND_C2D.COLLATERALGROUP, +OU_RIAD_ENTTY_FLTTND_C2D.INSTTTNL_SCTR_CNTRL, +OU_RIAD_ENTTY_FLTTND_C2D.CMMNT, +OU_RIAD_ENTTY_FLTTND_C2D.IS_C2D_RLVNT, +OU_RIAD_ENTTY_FLTTND_C2D.IS_RAR_RLVNT, +OU_RIAD_ENTTY_FLTTND_C2D.ARTCL_123_STTS, +OU_RIAD_ENTTY_FLTTND_C2D.CSPP_ASSSSMNT, +OU_RIAD_ENTTY_FLTTND_C2D.ENTRY_DATE, +OU_RIAD_ENTTY_FLTTND_C2D.RIAD, +OU_RIAD_ENTTY_FLTTND_C2D.AVID, +OU_RIAD_ENTTY_FLTTND_C2D.BIC, +OU_RIAD_ENTTY_FLTTND_C2D.LEI, +OU_RIAD_ENTTY_FLTTND_C2D.NCB, +OU_RIAD_ENTTY_FLTTND_C2D.UCDB_CD, +OU_RIAD_ENTTY_FLTTND_C2D.TMS_CD, +OU_RIAD_ENTTY_FLTTND_C2D.BLMBRG_CD, +OU_RIAD_ENTTY_FLTTND_C2D.C2D_CD, +OU_RIAD_ENTTY_FLTTND_C2D.EACODELIST, +OU_RIAD_ENTTY_FLTTND_C2D.FVC, +OU_RIAD_ENTTY_FLTTND_C2D.BRANCHHEAD_ENTTY_RIAD_ID, +OU_RIAD_ENTTY_FLTTND_C2D.DIRECTHEAD_ENTTY_RIAD_ID, +OU_RIAD_ENTTY_FLTTND_C2D.ULTIMATEHEAD_ENTTY_RIAD_CD, +OU_RIAD_ENTTY_FLTTND_C2D.ULTIMATEHEAD_ENTTY_RIAD_ID, +OU_RIAD_ENTTY_FLTTND_C2D.VRSN_VLD_FRM, +OU_RIAD_ENTTY_FLTTND_C2D.VRSN_VLD_T , +OU_RIAD_ENTTY_FLTTND_C2D.FNCL_HLDNG_INVSTMNT_FRM , +OU_RIAD_ENTTY_FLTTND_C2D.IS_CNTRL_GVRNMNT_TRTD_LG, +OU_RIAD_ENTTY_FLTTND_C2D.IS_LCL_GVRNMNT_TRTD_PS , +OU_RIAD_ENTTY_FLTTND_C2D.IS_CNTRL_GVRNMNT_TRTD_PS +FROM + {{ source("ou_riad", "ENTTY_FLTTND_C2D") }} OU_RIAD_ENTTY_FLTTND_C2D +where + A_WORKFLOW_HISTORY_KEY = + {{ filter_workflow_history_max_key("w_ODS_RIAD_ENTTY_FLTTND_C2D",get_main_task_name(model.name))}} + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql new file mode 100644 index 0000000..e7b3700 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql @@ -0,0 +1,55 @@ + +{{ config( + materialized = "table", + schema = "RIAD", + alias = "T_ENTTY_ECMS_CD_HIST_SQ", + tags = ["m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS"], +) }} + +with load_history as ( + {{ filter_workflow_history_max_key("w_ODS_RIAD_ENTTY_FLTTND_ECMS",get_main_task_name(model.name))}} +) + +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, RIAD AS entty_cd, 'RIAD' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where RIAD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, AVID AS entty_cd, 'AVID' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where AVID is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, BIC AS entty_cd, 'BIC' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where BIC is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, LEI AS entty_cd, 'LEI' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where LEI is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, NCB AS entty_cd, 'NCB' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where NCB is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, UCDB_CD AS entty_cd, 'UCDB_CD' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where UCDB_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, TMS_CD AS entty_cd, 'TMS_CD' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where TMS_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, BLMBRG_CD AS entty_cd, 'BLMBRG_CD' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where BLMBRG_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, C2D_CD AS entty_cd, 'C2D_CD' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where C2D_CD is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, EACODELIST AS entty_cd, 'EACODELIST' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where EACODELIST is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) +UNION +select /*+ PARALLEL(8) */ A_WORKFLOW_HISTORY_KEY, entty_riad_id, bsnss_vld_frm, bsnss_vld_t, FVC AS entty_cd, 'FVC' AS typ_entty_cd, entry_date, vrsn_vld_frm, vrsn_vld_t + from {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + where FVC is not null and A_WORKFLOW_HISTORY_KEY = (SELECT A_WORKFLOW_HISTORY_KEY FROM load_history) diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql new file mode 100644 index 0000000..3b89a04 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ.sql @@ -0,0 +1,69 @@ +{{ config( + materialized = "table", + schema = "RIAD", + alias = "T_ENTTY_ECMS_HIST_SQ", + tags = ["m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS"], +) }} + +SELECT +A_KEY , +A_WORKFLOW_HISTORY_KEY , +ENTTY_RIAD_ID , +BSNSS_VLD_FRM , +BSNSS_VLD_T , +CNTRY , +DT_BRTH , +DT_CLS , +ENTTY_RIAD_CD , +NM_ENTTY , +ECNMC_ACTVTY , +INSTTTNL_SCTR , +INSTTTNL_SCTR_DTL , +NMBR_EMPLYS , +ENTRPRS_SZ , +TYP_BNKNG_LCNS , +LGL_FRM , +PSTL_BX_CD , +STRT , +PSTL_CD , +CTY , +IS_INCTV , +IS_UCITS_CMPLNT , +INTRNTNL_ORGNSTN_CD , +BLNC_SHT_TTL_DMSTC , +BLNC_SHT_TTL , +NT_ASSTS_DMSTC , +COLLATERALGROUP , +INSTTTNL_SCTR_CNTRL , +CMMNT , +IS_C2D_RLVNT , +IS_RAR_RLVNT , +ARTCL_123_STTS , +CSPP_ASSSSMNT , +ENTRY_DATE , +RIAD , +AVID , +BIC , +LEI , +NCB , +UCDB_CD , +TMS_CD , +BLMBRG_CD , +C2D_CD , +EACODELIST , +FVC , +BRANCHHEAD_ENTTY_RIAD_ID , +DIRECTHEAD_ENTTY_RIAD_ID , +ULTIMATEHEAD_ENTTY_RIAD_CD , +ULTIMATEHEAD_ENTTY_RIAD_ID , +VRSN_VLD_FRM , +VRSN_VLD_T , +FNCL_HLDNG_INVSTMNT_FRM , +IS_CNTRL_GVRNMNT_TRTD_LG , +IS_LCL_GVRNMNT_TRTD_PS , +IS_CNTRL_GVRNMNT_TRTD_PS + FROM + {{ source("ou_riad", "ENTTY_FLTTND_ECMS") }} + WHERE + A_WORKFLOW_HISTORY_KEY = + {{ filter_workflow_history_max_key("w_ODS_RIAD_ENTTY_FLTTND_ECMS",get_main_task_name(model.name))}} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql new file mode 100644 index 0000000..f5c41b5 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ.sql @@ -0,0 +1,69 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND"], + alias = "T_ENTTY_FLTTND_ESSNTL_CURRENT_SQ", + schema = "RIAD", +) }} + +SELECT +MAX(OU_RIAD_ENTTY_IDNTFRS_FLTTND.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_RIAD_ENTTY_IDNTFRS_FLTTND.ENTTY_RIAD_ID) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.A_WORKFLOW_HISTORY_KEY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ENTTY_RIAD_ID, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BSNSS_VLD_FRM, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BSNSS_VLD_T, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.CNTRY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.DT_BRTH, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.DT_CLS, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ENTTY_RIAD_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.NM_ENTTY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ECNMC_ACTVTY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.INSTTTNL_SCTR, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.INSTTTNL_SCTR_DTL, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.PSTL_BX_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.STRT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.PSTL_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.CTY, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_INCTV, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_UCITS_CMPLNT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.INTRNTNL_ORGNSTN_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BLNC_SHT_TTL_DMSTC, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.NT_ASSTS_DMSTC, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.COLLATERALGROUP, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.INSTTTNL_SCTR_CNTRL, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_C2D_RLVNT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_RAR_RLVNT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ARTCL_123_STTS, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.CSPP_ASSSSMNT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ENTRY_DATE, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.RIAD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.AVID, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BIC, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.LEI, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.NCB, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.REU, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.UCDB_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.VAT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.TMS_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BLMBRG_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.C2D_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ULTIMATEHEAD_ENTTY_RIAD_CD, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ULTIMATEHEAD_ENTTY_RIAD_ID, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.NMBR_EMPLYS, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.ENTRPRS_SZ, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.TYP_BNKNG_LCNS, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.LGL_FRM, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BLNC_SHT_TTL, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.BLNC_SHT_TTL_GRP, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.CMMNT, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.FNCL_HLDNG_INVSTMNT_FRM, +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_CNTRL_GVRNMNT_TRTD_LG , +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_LCL_GVRNMNT_TRTD_PS , +OU_RIAD_ENTTY_IDNTFRS_FLTTND.IS_CNTRL_GVRNMNT_TRTD_PS +FROM + {{ source("ou_riad", "ENTTY_IDNTFRS_FLTTND") }} OU_RIAD_ENTTY_IDNTFRS_FLTTND + WHERE +OU_RIAD_ENTTY_IDNTFRS_FLTTND.A_WORKFLOW_HISTORY_KEY in + {{ filter_workflow_history_max_key( + "w_ODS_RIAD_ENTTY_IDNTFRS_FLTTND", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ.sql new file mode 100644 index 0000000..afe0595 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS"], + alias = "T_ENTTY_RIAD_CD_ALS_SQ", + schema = "RIAD", +) }} + + SELECT +OU_RIAD_ENTTY_RIAD_CD_ALS.A_KEY, +OU_RIAD_ENTTY_RIAD_CD_ALS.A_WORKFLOW_HISTORY_KEY, +OU_RIAD_ENTTY_RIAD_CD_ALS.ENTTY_RIAD_ID, +OU_RIAD_ENTTY_RIAD_CD_ALS.ENTTY_RIAD_CD_ALS, +OU_RIAD_ENTTY_RIAD_CD_ALS.BSNSS_VLD_FRM, +OU_RIAD_ENTTY_RIAD_CD_ALS.BSNSS_VLD_T, +OU_RIAD_ENTTY_RIAD_CD_ALS.VRSN_VLD_FRM, +OU_RIAD_ENTTY_RIAD_CD_ALS.VRSN_VLD_T, +OU_RIAD_ENTTY_RIAD_CD_ALS.ENTRY_DATE +FROM + {{ source("ou_riad", "ENTTY_RIAD_CD_ALS") }} OU_RIAD_ENTTY_RIAD_CD_ALS + where + OU_RIAD_ENTTY_RIAD_CD_ALS.A_WORKFLOW_HISTORY_KEY = + {{ filter_workflow_history_max_key( + "w_ODS_RIAD_ENTTY_FLTTND_C2D", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql new file mode 100644 index 0000000..7f31fe0 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql @@ -0,0 +1,69 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_1_ALL_SQ', + schema='RQSD' +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + DATACOLLECTIONCODE, + DATACOLLECTIONNAME, + DATACOLLECTIONOWNER, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULENAME, + MODULEVERSIONNUMBER, + REPORTINGENTITYCOLLECTIONUNIQUEID, + ENTITYATTRIBUTEREPORTINGCODE, + REPORTINGENTITYNAME, + REPORTINGENTITYENTITYTYPE, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREFERENCEDATE, + OBLIGATIONMODULEREMITTANCEDATE, + RECEIVEDFILERECEIVEDDATE, + OBLIGATIONMODULEEXPECTED, + RECEIVEDFILEVERSIONNUMBER, + REVALIDATIONVERSIONNUMBER, + REVALIDATIONDATE, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + FILESTATUSSETEXTERNALVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + CASE WHEN OBSERVATIONVALUE = 'NULL' THEN NULL ELSE OBSERVATIONVALUE END AS OBSERVATIONVALUE, + CASE WHEN OBSERVATIONTEXTVALUE = 'NULL' THEN NULL ELSE OBSERVATIONTEXTVALUE END AS OBSERVATIONTEXTVALUE, + CASE WHEN OBSERVATIONDATEVALUE = 'NULL' THEN NULL ELSE OBSERVATIONDATEVALUE END AS OBSERVATIONDATEVALUE, + DATAPOINTSETDATAPOINTIDENTIFIER, + DATAPOINTSETLABEL, + OBSRVDESCDATATYPE, + ORDINATECODE, + ORDINATEPOSITION, + TABLENAME, + ISSTOCK, + SCALE, + CURRENCY, + NUMBERTYPE, + ISMANDATORY, + DECIMALPLACES, + SERIESKEY, + TEC_SOURCE_SYSTEM, + TEC_DATASET, + TEC_SURROGATE_KEY, + TEC_CRC, + TEC_INGESTION_DATE, + TEC_VERSION_ID +FROM + {{ source("ou_rqsd", "RQSD_OBSERVATIONS") }} +WHERE + MODULECODE = 'SCOPA' +AND TABLENAME = 'cis' +AND A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_RQSD_PROCESS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql new file mode 100644 index 0000000..839873a --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql @@ -0,0 +1,69 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_1_FIN_ALL_SQ', + schema='RQSD' +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + DATACOLLECTIONCODE, + DATACOLLECTIONNAME, + DATACOLLECTIONOWNER, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULENAME, + MODULEVERSIONNUMBER, + REPORTINGENTITYCOLLECTIONUNIQUEID, + ENTITYATTRIBUTEREPORTINGCODE, + REPORTINGENTITYNAME, + REPORTINGENTITYENTITYTYPE, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREFERENCEDATE, + OBLIGATIONMODULEREMITTANCEDATE, + RECEIVEDFILERECEIVEDDATE, + OBLIGATIONMODULEEXPECTED, + RECEIVEDFILEVERSIONNUMBER, + REVALIDATIONVERSIONNUMBER, + REVALIDATIONDATE, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + FILESTATUSSETEXTERNALVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + CASE WHEN OBSERVATIONVALUE = 'NULL' THEN NULL ELSE OBSERVATIONVALUE END AS OBSERVATIONVALUE, + CASE WHEN OBSERVATIONTEXTVALUE = 'NULL' THEN NULL ELSE OBSERVATIONTEXTVALUE END AS OBSERVATIONTEXTVALUE, + CASE WHEN OBSERVATIONDATEVALUE = 'NULL' THEN NULL ELSE OBSERVATIONDATEVALUE END AS OBSERVATIONDATEVALUE, + DATAPOINTSETDATAPOINTIDENTIFIER, + DATAPOINTSETLABEL, + OBSRVDESCDATATYPE, + ORDINATECODE, + ORDINATEPOSITION, + TABLENAME, + ISSTOCK, + SCALE, + CURRENCY, + NUMBERTYPE, + ISMANDATORY, + DECIMALPLACES, + SERIESKEY, + TEC_SOURCE_SYSTEM, + TEC_DATASET, + TEC_SURROGATE_KEY, + TEC_CRC, + TEC_INGESTION_DATE, + TEC_VERSION_ID +FROM + {{ source("ou_rqsd", "RQSD_OBSERVATIONS") }} +WHERE + MODULECODE = 'SCOPF' +AND TABLENAME = 'cis' +AND A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_RQSD_PROCESS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql new file mode 100644 index 0000000..c7ca5d7 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql @@ -0,0 +1,69 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_2_ALL_SQ', + schema='RQSD' +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + DATACOLLECTIONCODE, + DATACOLLECTIONNAME, + DATACOLLECTIONOWNER, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULENAME, + MODULEVERSIONNUMBER, + REPORTINGENTITYCOLLECTIONUNIQUEID, + ENTITYATTRIBUTEREPORTINGCODE, + REPORTINGENTITYNAME, + REPORTINGENTITYENTITYTYPE, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREFERENCEDATE, + OBLIGATIONMODULEREMITTANCEDATE, + RECEIVEDFILERECEIVEDDATE, + OBLIGATIONMODULEEXPECTED, + RECEIVEDFILEVERSIONNUMBER, + REVALIDATIONVERSIONNUMBER, + REVALIDATIONDATE, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + FILESTATUSSETEXTERNALVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + CASE WHEN OBSERVATIONVALUE = 'NULL' THEN NULL ELSE OBSERVATIONVALUE END AS OBSERVATIONVALUE, + CASE WHEN OBSERVATIONTEXTVALUE = 'NULL' THEN NULL ELSE OBSERVATIONTEXTVALUE END AS OBSERVATIONTEXTVALUE, + CASE WHEN OBSERVATIONDATEVALUE = 'NULL' THEN NULL ELSE OBSERVATIONDATEVALUE END AS OBSERVATIONDATEVALUE, + DATAPOINTSETDATAPOINTIDENTIFIER, + DATAPOINTSETLABEL, + OBSRVDESCDATATYPE, + ORDINATECODE, + ORDINATEPOSITION, + TABLENAME, + ISSTOCK, + SCALE, + CURRENCY, + NUMBERTYPE, + ISMANDATORY, + DECIMALPLACES, + SERIESKEY, + TEC_SOURCE_SYSTEM, + TEC_DATASET, + TEC_SURROGATE_KEY, + TEC_CRC, + TEC_INGESTION_DATE, + TEC_VERSION_ID +FROM + {{ source("ou_rqsd", "RQSD_OBSERVATIONS") }} +WHERE + MODULECODE = 'SCOPA' +AND TABLENAME = 'etwdr' +AND A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_RQSD_PROCESS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql new file mode 100644 index 0000000..fec1fc6 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql @@ -0,0 +1,69 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_2_FIN_ALL_SQ', + schema='RQSD' +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + DATACOLLECTIONCODE, + DATACOLLECTIONNAME, + DATACOLLECTIONOWNER, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULENAME, + MODULEVERSIONNUMBER, + REPORTINGENTITYCOLLECTIONUNIQUEID, + ENTITYATTRIBUTEREPORTINGCODE, + REPORTINGENTITYNAME, + REPORTINGENTITYENTITYTYPE, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREFERENCEDATE, + OBLIGATIONMODULEREMITTANCEDATE, + RECEIVEDFILERECEIVEDDATE, + OBLIGATIONMODULEEXPECTED, + RECEIVEDFILEVERSIONNUMBER, + REVALIDATIONVERSIONNUMBER, + REVALIDATIONDATE, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + FILESTATUSSETEXTERNALVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + CASE WHEN OBSERVATIONVALUE = 'NULL' THEN NULL ELSE OBSERVATIONVALUE END AS OBSERVATIONVALUE, + CASE WHEN OBSERVATIONTEXTVALUE = 'NULL' THEN NULL ELSE OBSERVATIONTEXTVALUE END AS OBSERVATIONTEXTVALUE, + CASE WHEN OBSERVATIONDATEVALUE = 'NULL' THEN NULL ELSE OBSERVATIONDATEVALUE END AS OBSERVATIONDATEVALUE, + DATAPOINTSETDATAPOINTIDENTIFIER, + DATAPOINTSETLABEL, + OBSRVDESCDATATYPE, + ORDINATECODE, + ORDINATEPOSITION, + TABLENAME, + ISSTOCK, + SCALE, + CURRENCY, + NUMBERTYPE, + ISMANDATORY, + DECIMALPLACES, + SERIESKEY, + TEC_SOURCE_SYSTEM, + TEC_DATASET, + TEC_SURROGATE_KEY, + TEC_CRC, + TEC_INGESTION_DATE, + TEC_VERSION_ID +FROM + {{ source("ou_rqsd", "RQSD_OBSERVATIONS") }} +WHERE + MODULECODE = 'SCOPF' +AND TABLENAME = 'etwdr' +AND A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_RQSD_PROCESS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql new file mode 100644 index 0000000..c168206 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ.sql @@ -0,0 +1,69 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_2_ALL_SQ', + schema='RQSD' +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + DATACOLLECTIONCODE, + DATACOLLECTIONNAME, + DATACOLLECTIONOWNER, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULENAME, + MODULEVERSIONNUMBER, + REPORTINGENTITYCOLLECTIONUNIQUEID, + ENTITYATTRIBUTEREPORTINGCODE, + REPORTINGENTITYNAME, + REPORTINGENTITYENTITYTYPE, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREFERENCEDATE, + OBLIGATIONMODULEREMITTANCEDATE, + RECEIVEDFILERECEIVEDDATE, + OBLIGATIONMODULEEXPECTED, + RECEIVEDFILEVERSIONNUMBER, + REVALIDATIONVERSIONNUMBER, + REVALIDATIONDATE, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + FILESTATUSSETEXTERNALVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + CASE WHEN OBSERVATIONVALUE = 'NULL' THEN NULL ELSE OBSERVATIONVALUE END AS OBSERVATIONVALUE, + CASE WHEN OBSERVATIONTEXTVALUE = 'NULL' THEN NULL ELSE OBSERVATIONTEXTVALUE END AS OBSERVATIONTEXTVALUE, + CASE WHEN OBSERVATIONDATEVALUE = 'NULL' THEN NULL ELSE OBSERVATIONDATEVALUE END AS OBSERVATIONDATEVALUE, + DATAPOINTSETDATAPOINTIDENTIFIER, + DATAPOINTSETLABEL, + OBSRVDESCDATATYPE, + ORDINATECODE, + ORDINATEPOSITION, + TABLENAME, + ISSTOCK, + SCALE, + CURRENCY, + NUMBERTYPE, + ISMANDATORY, + DECIMALPLACES, + SERIESKEY, + TEC_SOURCE_SYSTEM, + TEC_DATASET, + TEC_SURROGATE_KEY, + TEC_CRC, + TEC_INGESTION_DATE, + TEC_VERSION_ID +FROM + {{ source("ou_rqsd", "RQSD_OBSERVATIONS") }} +WHERE + MODULECODE = 'RQSDC' +AND TABLENAME = 'dct' +AND A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_key( + "w_ODS_RQSD_PROCESS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ.sql new file mode 100644 index 0000000..020ff20 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ.sql @@ -0,0 +1,274 @@ +{{ config(materialized='table', + tags=["m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA", "m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL"], + alias='OUTPUT_CUR', + schema='RQSD' +) }} + +WITH cis_table AS ( + select + ann_1_1_ref_date, + ctp_mfi_id, + ctp_name, + ctp_country, + ctp_legal_status, + ctp_lprt, + ctp_measure_start_date, + ann_1_1_etwdr_mfi_id, + ann_1_1_etwdr_lei, + obligationmodulereferencedate as scopf_obligationmodulereferencedate, + reportingentitycollectionuniqueid as scopf_reportingentitycollectionuniqueid, + receivedfilesystemfilename as ann_1_1_receivedfilesystemfilename, + receivedfileversionnumber as scopf_receivedfileversionnumber, + receivedfilereceiveddate as scopf_receivedfilereceiveddate, + revalidationdate as scopf_revalidationdate, + ann_1_1_submitter_comment, + concat(ann_1_1_etwdr_mfi_id, ann_1_1_etwdr_lei) as ann_1_1_etwdr_comp_key + from {{ ref('m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW') }} +), etwdr_table AS ( + select + ann_1_2_ref_date, + etwdr_mfi_id, + etwdr_lei, + etwdr_name, + etwdr_country, + etwdr_in_eu, + etwdr_source, + etwdr_submitter, + etwdr_comp_key, + obligationmodulereferencedate as ann_1_2_obligationmodulereferencedate, + reportingentitycollectionuniqueid as ann_1_2_reportingentitycollectionuniqueid, + receivedfilesystemfilename as ann_1_2_receivedfilesystemfilename, + receivedfileversionnumber as ann_1_2_receivedfileversionnumber, + receivedfilereceiveddate as ann_1_2_receivedfilereceiveddate, + revalidationdate as ann_1_2_revalidationdate, + ann_1_2_submitter_comment + from {{ ref('m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW') }} +), annex_1 AS ( + select * from cis_table cis + left join etwdr_table etw + on cis.scopf_obligationmodulereferencedate = etw.ann_1_2_obligationmodulereferencedate + and cis.ann_1_1_etwdr_comp_key = etw.etwdr_comp_key +), dct_table AS ( + select + ref_date, + currency, + reported_by_supervisor, + confirmed_by_supervisor, + exp_corep_cons, + exp_corep_solo, + exp_finrep_cons, + exp_finrep_solo, + exp_lev_cons, + exp_lev_solo, + exp_lcr_cons, + exp_lcr_solo, + exp_nsfr_cons, + exp_nsfr_solo, + cons_cet1_amt, + cons_tier1_amt, + cons_tot_cap_amt, + cons_cet1_ratio, + cons_tier1_ratio, + cons_tot_cap_ratio, + cons_risk_wght_assets, + solo_cet1_amt, + solo_tier1_amt, + solo_tot_cap_amt, + solo_cet1_ratio, + solo_tier1_ratio, + solo_tot_cap_ratio, + solo_risk_wght_assets, + cons_tot_assets, + solo_tot_assets, + cons_lev_ratio_full, + cons_lev_ratio_trans, + cons_lev_ratio_req, + cons_lev_ratio_adj, + solo_lev_ratio_full, + solo_lev_ratio_trans, + solo_lev_ratio_req, + solo_lev_ratio_adj, + cons_lc_ratio, + solo_lc_ratio, + cons_nsfr_ratio, + solo_nsfr_ratio, + submitter_comment, + mfi_id, + legal_entity_id, + inst_comp_key, + obligationmodulereferencedate as dct_obligationmodulereferencedate, + reportingentitycollectionuniqueid as dct_reportingentitycollectionuniqueid, + receivedfileversionnumber as dct_receivedfileversionnumber, + receivedfilereceiveddate as dct_receivedfilereceiveddate, + revalidationdate as dct_revalidationdate + from {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW') }} +), annex_joined AS ( + select * from annex_1 an1 + left join dct_table dct + on an1.scopf_obligationmodulereferencedate = dct.dct_obligationmodulereferencedate + and an1.ann_1_1_etwdr_comp_key = dct.inst_comp_key +), mpec_table AS ( + select + CASE + -- Check if the current timestamp (in Europe/Berlin) is before today's 17:40:00 and today is Monday + WHEN TO_CHAR(SYSTIMESTAMP AT TIME ZONE 'Europe/Berlin', 'YYYY-MM-DD HH24:MI:SS') < TO_CHAR(CURRENT_DATE, 'YYYY-MM-DD') || ' 17:40:00' + AND TO_CHAR(SYSTIMESTAMP AT TIME ZONE 'Europe/Berlin', 'D') = '2' + THEN CURRENT_DATE - 3 + + -- Check if the current timestamp (in Europe/Berlin) is before today's 17:40:00 + WHEN TO_CHAR(SYSTIMESTAMP AT TIME ZONE 'Europe/Berlin', 'YYYY-MM-DD HH24:MI:SS') < TO_CHAR(CURRENT_DATE, 'YYYY-MM-DD') || ' 17:40:00' + THEN (SYSTIMESTAMP AT TIME ZONE 'Europe/Berlin') - INTERVAL '1' DAY + + -- Otherwise, return the current timestamp in Europe/Berlin + ELSE SYSTIMESTAMP AT TIME ZONE 'Europe/Berlin' + END AS mpec_snapshot_date, + marginal_lending_facility, + standard_tender_operations, + fine_tuning_providing, + deposit_facility, + fine_tuning_absorbing, + fixed_term_deposit, + fx_swap_absorbing, + fx_swap_providing, + riad_code AS counterparty_id + from {{ source("mpec", "T_MPEC") }} + WHERE is_current = 'Y' +), with_mpec AS ( + select aj.*, + case when marginal_lending_facility = 'Y' or + standard_tender_operations = 'Y' or + fine_tuning_providing = 'Y' then 'LPRT' + when deposit_facility = 'Y' or + fine_tuning_absorbing = 'Y' or + fixed_term_deposit = 'Y' or + fx_swap_absorbing = 'Y' or + fx_swap_providing = 'Y' then 'non-LPRT' + else 'not in MPEC' + end as access_to, + 'FX.Q.' || currency || '.EUR.BL.FX.' || currency || 'EUR.HSTE' as comp_series_key, + mpec_snapshot_date + from annex_joined aj + left join mpec_table mt + on aj.ctp_mfi_id = mt.counterparty_id +), fx_data AS ( + select tec_ingestion_date, + obs_value, + divisor, + series_key, + time_period, + rn + from {{ source("ou_rqsd", "RQSD_FX") }} + where A_WORKFLOW_HISTORY_KEY IN {{ filter_workflow_history_max_key( + "w_ODS_RQSD_PROCESS", get_main_task_name(model.name)) }} +), +with_fx AS ( + SELECT wm.*, fx.*, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(cons_cet1_amt) + WHEN currency is not null THEN TO_NUMBER(cons_cet1_amt) * CAST(obs_value AS NUMBER)/ CAST(divisor AS NUMBER) ELSE null END as cons_cet1_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(cons_tier1_amt) + WHEN currency is not null THEN TO_NUMBER(cons_tier1_amt) * obs_value / divisor ELSE null END as cons_tier1_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(cons_tot_cap_amt) + WHEN currency is not null THEN TO_NUMBER(cons_tot_cap_amt) * obs_value / divisor ELSE null END as cons_tot_cap_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(cons_risk_wght_assets) + WHEN currency is not null THEN TO_NUMBER(cons_risk_wght_assets) * obs_value / divisor ELSE null END as cons_risk_wght_assets_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(solo_cet1_amt) + WHEN currency is not null THEN TO_NUMBER(solo_cet1_amt) * obs_value / divisor ELSE null END as solo_cet1_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(solo_tier1_amt) + WHEN currency is not null THEN TO_NUMBER(solo_tier1_amt) * obs_value / divisor ELSE null END as solo_tier1_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(solo_tot_cap_amt) + WHEN currency is not null THEN TO_NUMBER(solo_tot_cap_amt) * obs_value / divisor ELSE null END as solo_tot_cap_amt_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(solo_risk_wght_assets) + WHEN currency is not null THEN TO_NUMBER(solo_risk_wght_assets) * obs_value / divisor ELSE null END as solo_risk_wght_assets_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(cons_tot_assets) + WHEN currency is not null THEN TO_NUMBER(cons_tot_assets) * obs_value / divisor ELSE null END as cons_tot_assets_eur, + CASE WHEN currency = 'EUR' THEN TO_NUMBER(solo_tot_assets) + WHEN currency is not null THEN TO_NUMBER(solo_tot_assets) * obs_value/ divisor ELSE null END as solo_tot_assets_eur + FROM with_mpec wm + LEFT JOIN fx_data fx + ON wm.comp_series_key = fx.series_key AND wm.ref_date = TO_CHAR(EXTRACT(YEAR FROM fx.time_period)) || 'Q' || TO_CHAR(fx.time_period, 'Q') +) +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(ANN_1_1_REF_DATE AS VARCHAR2(100)) AS ANN_1_1_REF_DATE, + CAST(CTP_MFI_ID AS VARCHAR2(200)) AS CTP_MFI_ID, + CAST(CTP_NAME AS VARCHAR2(200)) AS CTP_NAME, + CAST(CTP_COUNTRY AS VARCHAR2(200)) AS CTP_COUNTRY, + CAST(CTP_LEGAL_STATUS AS VARCHAR2(200)) AS CTP_LEGAL_STATUS, + CAST(CTP_LPRT AS VARCHAR2(200)) AS CTP_LPRT, + CAST(CTP_MEASURE_START_DATE AS VARCHAR2(200)) AS CTP_MEASURE_START_DATE, + CAST(ANN_1_2_REF_DATE AS VARCHAR2(200)) AS ANN_1_2_REF_DATE, + CAST(ETWDR_MFI_ID AS VARCHAR2(200)) AS ETWDR_MFI_ID, + CAST(ETWDR_LEI AS VARCHAR2(200)) AS ETWDR_LEI, + CAST(ETWDR_NAME AS VARCHAR2(200)) AS ETWDR_NAME, + CAST(ETWDR_COUNTRY AS VARCHAR2(200)) AS ETWDR_COUNTRY, + CAST(ETWDR_IN_EU AS VARCHAR2(200)) AS ETWDR_IN_EU, + CAST(ETWDR_SOURCE AS VARCHAR2(200)) AS ETWDR_SOURCE, + CAST(ETWDR_SUBMITTER AS VARCHAR2(200)) AS ETWDR_SUBMITTER, + CAST(REF_DATE AS VARCHAR2(200)) AS REF_DATE, + CAST(CURRENCY AS VARCHAR2(200)) AS CURRENCY, + CAST(REPORTED_BY_SUPERVISOR AS VARCHAR2(200)) AS REPORTED_BY_SUPERVISOR, + CAST(CONFIRMED_BY_SUPERVISOR AS VARCHAR2(200)) AS CONFIRMED_BY_SUPERVISOR, + CAST(EXP_COREP_CONS AS VARCHAR2(200)) AS EXP_COREP_CONS, + CAST(EXP_COREP_SOLO AS VARCHAR2(200)) AS EXP_COREP_SOLO, + CAST(EXP_FINREP_CONS AS VARCHAR2(200)) AS EXP_FINREP_CONS, + CAST(EXP_FINREP_SOLO AS VARCHAR2(200)) AS EXP_FINREP_SOLO, + CAST(EXP_LEV_CONS AS VARCHAR2(200)) AS EXP_LEV_CONS, + CAST(EXP_LEV_SOLO AS VARCHAR2(200)) AS EXP_LEV_SOLO, + CAST(EXP_LCR_CONS AS VARCHAR2(200)) AS EXP_LCR_CONS, + CAST(EXP_LCR_SOLO AS VARCHAR2(200)) AS EXP_LCR_SOLO, + CAST(EXP_NSFR_CONS AS VARCHAR2(200)) AS EXP_NSFR_CONS, + CAST(EXP_NSFR_SOLO AS VARCHAR2(200)) AS EXP_NSFR_SOLO, + CAST(CONS_CET1_AMT AS NUMBER(38,10)) AS CONS_CET1_AMT, + CAST(CONS_CET1_AMT_EUR AS NUMBER(38,10)) AS CONS_CET1_AMT_EUR, + CAST(CONS_TIER1_AMT AS NUMBER(38,10)) AS CONS_TIER1_AMT, + CAST(CONS_TIER1_AMT_EUR AS NUMBER(38,10)) AS CONS_TIER1_AMT_EUR, + CAST(CONS_TOT_CAP_AMT AS NUMBER(38,10)) AS CONS_TOT_CAP_AMT, + CAST(CONS_TOT_CAP_AMT_EUR AS NUMBER(38,10)) AS CONS_TOT_CAP_AMT_EUR, + CAST(CONS_CET1_RATIO AS NUMBER(38,10)) AS CONS_CET1_RATIO, + CAST(CONS_TIER1_RATIO AS NUMBER(38,10)) AS CONS_TIER1_RATIO, + CAST(CONS_TOT_CAP_RATIO AS NUMBER(38,10)) AS CONS_TOT_CAP_RATIO, + CAST(CONS_RISK_WGHT_ASSETS AS NUMBER(38,10)) AS CONS_RISK_WGHT_ASSETS, + CAST(CONS_RISK_WGHT_ASSETS_EUR AS NUMBER(38,10)) AS CONS_RISK_WGHT_ASSETS_EUR, + CAST(SOLO_CET1_AMT AS NUMBER(38,10)) AS SOLO_CET1_AMT, + CAST(SOLO_CET1_AMT_EUR AS NUMBER(38,10)) AS SOLO_CET1_AMT_EUR, + CAST(SOLO_TIER1_AMT AS NUMBER(38,10)) AS SOLO_TIER1_AMT, + CAST(SOLO_TIER1_AMT_EUR AS NUMBER(38,10)) AS SOLO_TIER1_AMT_EUR, + CAST(SOLO_TOT_CAP_AMT AS NUMBER(38,10)) AS SOLO_TOT_CAP_AMT, + CAST(SOLO_TOT_CAP_AMT_EUR AS NUMBER(38,10)) AS SOLO_TOT_CAP_AMT_EUR, + CAST(SOLO_CET1_RATIO AS NUMBER(38,10)) AS SOLO_CET1_RATIO, + CAST(SOLO_TIER1_RATIO AS NUMBER(38,10)) AS SOLO_TIER1_RATIO, + CAST(SOLO_TOT_CAP_RATIO AS NUMBER(38,10)) AS SOLO_TOT_CAP_RATIO, + CAST(SOLO_RISK_WGHT_ASSETS AS NUMBER(38,10)) AS SOLO_RISK_WGHT_ASSETS, + CAST(SOLO_RISK_WGHT_ASSETS_EUR AS NUMBER(38,10)) AS SOLO_RISK_WGHT_ASSETS_EUR, + CAST(CONS_TOT_ASSETS AS NUMBER(38,10)) AS CONS_TOT_ASSETS, + CAST(CONS_TOT_ASSETS_EUR AS NUMBER(38,10)) AS CONS_TOT_ASSETS_EUR, + CAST(SOLO_TOT_ASSETS AS NUMBER(38,10)) AS SOLO_TOT_ASSETS, + CAST(SOLO_TOT_ASSETS_EUR AS NUMBER(38,10)) AS SOLO_TOT_ASSETS_EUR, + CAST(CONS_LEV_RATIO_FULL AS NUMBER(38,10)) AS CONS_LEV_RATIO_FULL, + CAST(CONS_LEV_RATIO_TRANS AS NUMBER(38,10)) AS CONS_LEV_RATIO_TRANS, + CAST(CONS_LEV_RATIO_REQ AS NUMBER(38,10)) AS CONS_LEV_RATIO_REQ, + CAST(CONS_LEV_RATIO_ADJ AS VARCHAR2(200)) AS CONS_LEV_RATIO_ADJ, + CAST(SOLO_LEV_RATIO_FULL AS NUMBER(38,10)) AS SOLO_LEV_RATIO_FULL, + CAST(SOLO_LEV_RATIO_TRANS AS NUMBER(38,10)) AS SOLO_LEV_RATIO_TRANS, + CAST(SOLO_LEV_RATIO_REQ AS NUMBER(38,10)) AS SOLO_LEV_RATIO_REQ, + CAST(SOLO_LEV_RATIO_ADJ AS VARCHAR2(200)) AS SOLO_LEV_RATIO_ADJ, + CAST(CONS_LC_RATIO AS NUMBER(38,10)) AS CONS_LC_RATIO, + CAST(SOLO_LC_RATIO AS NUMBER(38,10)) AS SOLO_LC_RATIO, + CAST(CONS_NSFR_RATIO AS NUMBER(38,10)) AS CONS_NSFR_RATIO, + CAST(SOLO_NSFR_RATIO AS NUMBER(38,10)) AS SOLO_NSFR_RATIO, + CAST(SUBMITTER_COMMENT AS VARCHAR2(200)) AS SUBMITTER_COMMENT, + CAST(ANN_1_1_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_1_SUBMITTER_COMMENT, + CAST(ANN_1_2_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_2_SUBMITTER_COMMENT, + CAST(MPEC_SNAPSHOT_DATE AS VARCHAR2(200)) AS MPEC_SNAPSHOT_DATE, + CAST(ACCESS_TO AS VARCHAR2(200)) AS ACCESS_TO, + CAST(SCOPF_OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(200)) AS SCOPF_OBLIGATIONMODULEREFERENCEDATE, + CAST(SCOPF_REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS SCOPF_REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(SCOPF_RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS SCOPF_RECEIVEDFILEVERSIONNUMBER, + CAST(SCOPF_RECEIVEDFILERECEIVEDDATE AS VARCHAR2(200)) AS SCOPF_RECEIVEDFILERECEIVEDDATE, + CAST(SCOPF_REVALIDATIONDATE AS VARCHAR2(200)) AS SCOPF_REVALIDATIONDATE, + CAST(DCT_OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(200)) AS DCT_OBLIGATIONMODULEREFERENCEDATE, + CAST(DCT_REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS DCT_REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(DCT_RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS DCT_RECEIVEDFILEVERSIONNUMBER, + CAST(DCT_RECEIVEDFILERECEIVEDDATE AS VARCHAR2(200)) AS DCT_RECEIVEDFILERECEIVEDDATE, + CAST(DCT_REVALIDATIONDATE AS VARCHAR2(200)) AS DCT_REVALIDATIONDATE +FROM with_fx diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ.sql new file mode 100644 index 0000000..f8d0c5b --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ.sql @@ -0,0 +1,42 @@ +{{ + config( + materialiazed="table", + schema="T2", + alias="T_MAXIMA_CREDIT_LINE_AMOUNT_SQ", + tag=["m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT"], + meta={ + "author": "Atanas Georgiev", + "created": "2026-02-00", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2026-02-00" + } + ) +}} + +with CTE_MAXIMA_CREDIT_LINE_AMOUNT as ( +SELECT /*+ PARALLEL(T2_MAXIMA_CREDIT_LINE_AMOUNT,4) */ +row_number() OVER (PARTITION BY T2_MAXIMA_CREDIT_LINE_AMOUNT.PARTY_RIAD,T2_MAXIMA_CREDIT_LINE_AMOUNT.PARTY_BIC, T2_MAXIMA_CREDIT_LINE_AMOUNT.BUSINESS_DATE order by T2_MAXIMA_CREDIT_LINE_AMOUNT.A_KEY desc) as MX_ROW_NUM, +T2_MAXIMA_CREDIT_LINE_AMOUNT.A_KEY, +T2_MAXIMA_CREDIT_LINE_AMOUNT.A_WORKFLOW_HISTORY_KEY, +T2_MAXIMA_CREDIT_LINE_AMOUNT.BUSINESS_DATE, +T2_MAXIMA_CREDIT_LINE_AMOUNT.CURRENCY, +T2_MAXIMA_CREDIT_LINE_AMOUNT.SYSTEM_ENTITY, +T2_MAXIMA_CREDIT_LINE_AMOUNT.PARTY_RIAD, +T2_MAXIMA_CREDIT_LINE_AMOUNT.PARTY_BIC, +T2_MAXIMA_CREDIT_LINE_AMOUNT.MRG_LEND_INDIC, +T2_MAXIMA_CREDIT_LINE_AMOUNT.MAXIMA_OF_CREDIT_LINE_AMOUNT +-- CTE_MAXIMA_CREDIT_LINE_AMOUNT.A_ETL_LOAD_SET_FK --- dublirana kolona +FROM + {{ source("ods", "T2_MAXIMA_CREDIT_LINE_AMOUNT") }} T2_MAXIMA_CREDIT_LINE_AMOUNT +) +select * from CTE_MAXIMA_CREDIT_LINE_AMOUNT + WHERE + CTE_MAXIMA_CREDIT_LINE_AMOUNT.MX_ROW_NUM=1 + and + CTE_MAXIMA_CREDIT_LINE_AMOUNT.A_WORKFLOW_HISTORY_KEY --- !!!! A_WORKFLOW_HISTORY_KEY + not in + --- !!! not + {{filter_workflow_history_key ("w_ODS_T2_INTRADAY_CREDIT",get_main_task_name(model.name))}} + + + \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ.sql new file mode 100644 index 0000000..8331be8 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ.sql @@ -0,0 +1,33 @@ +{{ + config( + materilized="table", + schema="T2", + alias="T_PEAK_INTRADAY_CREDIT_USE_SQ", + tag=["m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE"] , + meta={ + "author": "Atanas Georgiev", + "created": "2026-02-00", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2026-02-00" + } + ) +}} + +SELECT /*+ PARALLEL(ods_T2_PEAK_INTRADAY_CREDIT_USE,4) */ +row_number() OVER (PARTITION BY ods_T2_PEAK_INTRADAY_CREDIT_USE.PARTY_RIAD,ods_T2_PEAK_INTRADAY_CREDIT_USE.PARTY_BIC, ods_T2_PEAK_INTRADAY_CREDIT_USE.BUSINESS_DATE + order by ods_T2_PEAK_INTRADAY_CREDIT_USE.A_KEY desc) as MX_ROW_NUM, + ods_T2_PEAK_INTRADAY_CREDIT_USE.A_KEY, +ods_T2_PEAK_INTRADAY_CREDIT_USE.A_WORKFLOW_HISTORY_KEY, +ods_T2_PEAK_INTRADAY_CREDIT_USE.BUSINESS_DATE, +ods_T2_PEAK_INTRADAY_CREDIT_USE.CURRENCY, +ods_T2_PEAK_INTRADAY_CREDIT_USE.SYSTEM_ENTITY, +ods_T2_PEAK_INTRADAY_CREDIT_USE.PARTY_RIAD, +ods_T2_PEAK_INTRADAY_CREDIT_USE.PARTY_BIC, +ods_T2_PEAK_INTRADAY_CREDIT_USE.MRG_LEND_INDIC, +ods_T2_PEAK_INTRADAY_CREDIT_USE.PEAK_IDC_USE +FROM + {{ source("ods", "PEAK_INTRADAY_CREDIT_USE") }} ods_T2_PEAK_INTRADAY_CREDIT_USE + WHERE + ods_T2_PEAK_INTRADAY_CREDIT_USE.A_WORKFLOW_HISTORY_KEY + in + {{filter_workflow_history_key ("w_ODS_T2_INTRADAY_CREDIT",get_main_task_name(model.name))}} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ.sql new file mode 100644 index 0000000..b27e368 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ.sql @@ -0,0 +1,32 @@ +{{config( + materilized="table", + schema="T2", + alias="T_PEAK_LIQUIDITY_NEED_SQ", + tag=["m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_ods_T2_PEAK_LIQUIDITY_NEED"] +)}} + +WITH PEAK_LIQUIDITY_NEED as ( +SELECT /*+ PARALLEL(ods_T2_PEAK_LIQUIDITY_NEED,4) */ +row_number() OVER (PARTITION BY ods_T2_PEAK_LIQUIDITY_NEED.PARTY_RIAD,ods_T2_PEAK_LIQUIDITY_NEED.PARTY_BIC, ods_T2_PEAK_LIQUIDITY_NEED.BUSINESS_DATE + order by ods_T2_PEAK_LIQUIDITY_NEED.A_KEY desc) as MX_ROW_NUM, +ods_T2_PEAK_LIQUIDITY_NEED.A_KEY, +ods_T2_PEAK_LIQUIDITY_NEED.A_WORKFLOW_HISTORY_KEY , +ods_T2_PEAK_LIQUIDITY_NEED.BUSINESS_DATE, +ods_T2_PEAK_LIQUIDITY_NEED.CURRENCY, +ods_T2_PEAK_LIQUIDITY_NEED.SYSTEM_ENTITY, +ods_T2_PEAK_LIQUIDITY_NEED.PARTY_RIAD, +ods_T2_PEAK_LIQUIDITY_NEED.PARTY_BIC, +ods_T2_PEAK_LIQUIDITY_NEED.MRG_LEND_INDIC, +ods_T2_PEAK_LIQUIDITY_NEED.PEAK_LIQUIDITY_NEED, +ods_T2_PEAK_LIQUIDITY_NEED.TIMESTAMP +FROM + {{source("ods","PEAK_LIQUIDITY_NEED")}} ods_T2_PEAK_LIQUIDITY_NEED +) +SELECT + * + FROM PEAK_LIQUIDITY_NEED + WHERE + MX_ROW_NUM = 1 and + PEAK_LIQUIDITY_NEED.A_WORKFLOW_HISTORY_KEY + in + {{filter_workflow_history_key ("w_ODS_T2_INTRADAY_CREDIT",get_main_task_name(model.name))}} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ.sql new file mode 100644 index 0000000..0257f19 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ.sql @@ -0,0 +1,135 @@ +{{ + config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW"], + alias = "T_ACMCURRENCYFLOW_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-11", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-23 Update of data types" + } + ) +}} +SELECT + OU_TMS_ACMCURRENCYFLOW.A_KEY, + OU_TMS_ACMCURRENCYFLOW.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_ACMCURRENCYFLOW.ACCOUNTINGTREATMENT_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNTINGTREATMENT_NAME, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMBOOKKEEPINGTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMBOOKKEEPINGTYPE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMCMMSTATE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCMMSTATE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMCMMVIEWTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCMMVIEWTYPE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMCPTYRELATION_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCPTYRELATION_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMCURRENCYENTRYSTATE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCURRENCYENTRYSTATE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMCURRENCYPOSITION_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCURRENCYPOSITION_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_ACCOUNT_HOLDER_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ACCOUNT_HOLDER_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_BANK_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACMENTRY_BANK_ACCOUNT, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_BANK_ACCOUNT_CCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_BANK_ACCOUNT_CCY_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_BANK_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_BANK_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_BOOK_VALUE AS NUMBER(19,4)) AS ACMENTRY_BOOK_VALUE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_BOOKING_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_BOOKING_CURRENCY_ID, + OU_TMS_ACMCURRENCYFLOW.ACMENTRY_C_PAYMENT_DATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CCY_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_CCY_FLOW_AMOUNT, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CCY_BOOKING_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_CCY_FLOW_BOOKING_AMOUNT, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CCY_GROUP_ID AS NUMBER(19,0)) AS ACMENTRY_CCY_GROUP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CCY_RESULT AS NUMBER(19,4)) AS ACMENTRY_CCY_RESULT, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CLASSIFICATION_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_CLASSIFICATION_ID, + CASE + WHEN OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CMM_GROUP_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CMM_GROUP_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS ACMENTRY_CMM_GROUP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CMM_PAYMENT_METHOD_ID AS VARCHAR2(15 CHAR)) AS ACMENTRY_CMM_PAYMENT_METHOD_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CMM_RECONCILIATION_ID AS NUMBER(19,0)) AS ACMENTRY_CMM_RECONCILIATION_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_COUNTERPARTY_CODE AS VARCHAR2(30 CHAR)) AS ACMENTRY_COUNTERPARTY_CODE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_COUNTERPARTY_GROUP_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_COUNTERPARTY_GROUP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_COUNTERPARTY_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_COUNTERPARTY_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_CURRENCY_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_DATE_BASIS AS VARCHAR2(200 CHAR)) AS ACMENTRY_DATE_BASIS, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_DEAL_PRICE AS NUMBER(28,10)) AS ACMENTRY_DEAL_PRICE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_DEAL_RATE AS NUMBER(28,10)) AS ACMENTRY_DEAL_RATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_DESCRIPTION AS VARCHAR2(150 CHAR)) AS ACMENTRY_DESCRIPTION, + OU_TMS_ACMCURRENCYFLOW.ACMENTRY_EVENT_DATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_EVENT_ID AS NUMBER(19,0)) AS ACMENTRY_EVENT_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_FIGURE_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_FIGURE_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_FX_RATE AS NUMBER(28,10)) AS ACMENTRY_FX_RATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_ID AS NUMBER(19,0)) AS ACMENTRY_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_INSTRUMENT_ID AS VARCHAR2(30 CHAR)) AS ACMENTRY_INSTRUMENT_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_ISSUER_GROUP_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_ISSUER_GROUP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_ISSUER_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ISSUER_ID, + CASE + WHEN OU_TMS_ACMCURRENCYFLOW.ACMENTRY_LEG_GROUP BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_LEG_GROUP as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS ACMENTRY_LEG_GROUP, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_LEG_ID AS VARCHAR2(30 CHAR)) AS ACMENTRY_LEG_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_LOAD_ID AS NUMBER(19,0)) AS ACMENTRY_LOAD_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_NOMINAL_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_NOMINAL_AMOUNT, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_NUMBER_ AS NUMBER(19,0)) AS ACMENTRY_NUMBER_, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_ONE_TIME_COUNTERPARTY_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ONE_TIME_COUNTERPARTY_ID, + CASE + WHEN OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PACKAGE_MAP_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PACKAGE_MAP_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS ACMENTRY_PACKAGE_MAP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PACKAGE_TYPE_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_PACKAGE_TYPE_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_0 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_0, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_1 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_1, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_10 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_10, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_11 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_11, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_12 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_12, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_13 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_13, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_14 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_14, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_15 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_15, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_16 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_16, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_17 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_17, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_18 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_18, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_19 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_19, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_2 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_2, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_3 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_3, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_4 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_4, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_5 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_5, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_6 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_6, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_7 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_7, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_8 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_8, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PARAM_9 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_9, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_PORTFOLIO_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_PORTFOLIO_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_REVERSE_EVENT_ID AS NUMBER(19,0)) AS ACMENTRY_REVERSE_EVENT_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_SOURCE_REFERENCE AS VARCHAR2(30 CHAR)) AS ACMENTRY_SOURCE_REFERENCE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_T_CURRENCY_2_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_T_CURRENCY_2_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRY_T_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_T_CURRENCY_ID, + OU_TMS_ACMCURRENCYFLOW.ACMENTRY_T_MATURITY_DATE, + OU_TMS_ACMCURRENCYFLOW.ACMENTRY_T_OPENING_DATE, + OU_TMS_ACMCURRENCYFLOW.ACMENTRY_T_VALUE_DATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRYORIGIN_ID_USER AS VARCHAR2(50 CHAR)) AS ACMENTRY_ORIGIN_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRYORIGINENTITY_ID_USER AS VARCHAR2(50 CHAR)) AS ACMENTRY_ORIGIN_ENTITY_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRYORIGINGROUP_ID_USER AS VARCHAR2(10 CHAR)) AS ACMENTRY_ORIGIN_GROUP_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMENTRYTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYTYPE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMEVENTTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMEVENTTYPE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMGROUPINGRULE_ID AS NUMBER(19,0)) AS ACMGROUPINGRULE_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMGROUPINGRULE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMGROUPINGRULE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMLEDGER_ID AS NUMBER(19,0)) AS ACMLEDGER_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMLEDGER_ID_USER AS VARCHAR2(30 CHAR)) AS ACMLEDGER_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMMAPPINGRULE_ID AS NUMBER(19,0)) AS ACMMAPPINGRULE_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMMAPPINGRULE_ID_USER AS VARCHAR2(60 CHAR)) AS ACMMAPPINGRULE_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMPERIOD_ID_USER AS VARCHAR2(30 CHAR)) AS ACMPERIOD_ID_USER, + CASE + WHEN OU_TMS_ACMCURRENCYFLOW.ACMPERIODORDERNUMBER_NUMBER BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_ACMCURRENCYFLOW.ACMPERIODORDERNUMBER_NUMBER as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS ACMPERIODORDERNUMBER_ORDER_NUMBER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMPLCATEGORY_ID_USER AS VARCHAR2(30 CHAR)) AS ACMPLCATEGORY_ID_USER, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMTRANSACTIONSIGN_NAME AS VARCHAR2(100 CHAR)) AS ACMTRANSACTIONSIGN_NAME, + OU_TMS_ACMCURRENCYFLOW.ACMVOUCHER_BOOKING_DATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMVOUCHER_ID AS NUMBER(19,0)) AS ACMVOUCHER_ID, + OU_TMS_ACMCURRENCYFLOW.ACMVOUCHER_POSTING_DATE, + CAST(OU_TMS_ACMCURRENCYFLOW.ACMVOUCHER_VOUCHER_NUMBER AS VARCHAR2(30 CHAR)) AS ACMVOUCHER_VOUCHER_NUMBER, + CAST(OU_TMS_ACMCURRENCYFLOW.TRANSACTIONKIND_NAME AS VARCHAR2(255 CHAR)) AS TRANSACTIONKIND_NAME, + CAST(OU_TMS_ACMCURRENCYFLOW.TRANSACTIONTYPE_NAME AS VARCHAR2(255 CHAR)) AS TRANSACTIONTYPE_NAME, + CAST(OU_TMS_ACMCURRENCYFLOW.UMPATHNODE_ID AS VARCHAR2(50 CHAR)) AS UMPATHNODE_ID, + CAST(OU_TMS_ACMCURRENCYFLOW.UMPATHNODE_UPATH AS VARCHAR2(100 CHAR)) AS UMPATHNODE_UPATH +FROM + {{ source("ods","ACMCURRENCYFLOW")}} OU_TMS_ACMCURRENCYFLOW +INNER JOIN + {{ filter_workflow_history_max_key( + "w_ODS_TMS_ACMCURRENCYFLOW", + get_main_task_name(model.name) + ) }} wh + ON OU_TMS_ACMCURRENCYFLOW.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ.sql new file mode 100644 index 0000000..233fc5d --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ.sql @@ -0,0 +1,216 @@ +{{ + config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP"], + alias = "T_ACMENTRYSTATELEDGERGROUP_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-16", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-24" + } + ) +}} + +SELECT + OU_TMS_ACMENTRYSTATELEDGERGROUP.A_KEY, + OU_TMS_ACMENTRYSTATELEDGERGROUP.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACCOUNTINGTREATMENT_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNTINGTREATMENT_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMACCOUNT1_ERP_ACCOUNT AS VARCHAR2(30 CHAR)) AS ACMACCOUNT1_ERP_ACCOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMACCOUNT1_ID_USER AS VARCHAR2(30 CHAR)) AS ACMACCOUNT1_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMACCOUNT1_NAME AS VARCHAR2(170 CHAR)) AS ACMACCOUNT1_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMACCOUNT2_ID_USER AS VARCHAR2(30 CHAR)) AS ACMACCOUNT2_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMBOOKKEEPINGTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMBOOKKEEPINGTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMCMMSTATE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCMMSTATE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMCMMVIEWTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCMMVIEWTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMCOSTCENTER_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCOSTCENTER_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMCPTYRELATION_ID_USER AS VARCHAR2(30 CHAR)) AS ACMCPTYRELATION_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMCTBTYPERULE_ID_USER AS VARCHAR2(60 CHAR)) AS ACMCTBTYPERULE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMDRCR_ID_USER AS VARCHAR2(30 CHAR)) AS ACMDRCR_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ACC_HOLDER_CODE AS VARCHAR2(30 CHAR)) AS ACMENTRY_ACC_HOLDER_CODE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ACC_HOLDER_GROUP_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_ACC_HOLDER_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ACCOUNT_HOLDER_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ACCOUNT_HOLDER_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ACCOUNT_ID AS NUMBER(19,0)) AS ACMENTRY_ACCOUNT_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ADJ_DISCHARGE_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_AGGREGATION_ID AS NUMBER(19,0)) AS ACMENTRY_AGGREGATION_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BANK_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACMENTRY_BANK_ACCOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BANK_ACCOUNT_CCY_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_BANK_ACCOUNT_CCY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BANK_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_BANK_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BATCH_ID AS NUMBER(19,0)) AS ACMENTRY_BATCH_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BOOK_VALUE AS NUMBER(19,4)) AS ACMENTRY_BOOK_VALUE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BOOKING_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_BOOKING_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BOOKING_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_BOOKING_CURRENCY_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_BOOKING_DATE, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_C_PAYMENT_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CCY_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_CCY_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CCY_BOOKING_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_CCY_BOOKING_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CCY_GROUP_ID AS NUMBER(19,0)) AS ACMENTRY_CCY_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CCY_GROUPING_RULE_ID AS NUMBER(19,0)) AS ACMENTRY_CCY_GROUPING_RULE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CCY_RESULT AS NUMBER(19,4)) AS ACMENTRY_CCY_RESULT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CLASSIFICATION_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_CLASSIFICATION_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CMM_GROUP_ID AS NUMBER(10,0)) AS ACMENTRY_CMM_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CMM_PAYMENT_METHOD_ID AS VARCHAR2(15 CHAR)) AS ACMENTRY_CMM_PAYMENT_METHOD_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CMM_RECONCILIATION_ID AS NUMBER(19,0)) AS ACMENTRY_CMM_RECONCILIATION_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CMMSE_ID AS NUMBER(19,0)) AS ACMENTRY_CMMSE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CONTINUATION_NUMBER AS NUMBER(19,0)) AS ACMENTRY_CONTINUATION_NUMBER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_COUNTERPARTY_CODE AS VARCHAR2(30 CHAR)) AS ACMENTRY_COUNTERPARTY_CODE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_COUNTERPARTY_GROUP_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_COUNTERPARTY_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_COUNTERPARTY_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_COUNTERPARTY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_COUNTERPARTY_UNIT_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_COUNTERPARTY_UNIT_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CTB_PREVIOUS_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CTB_SEQUENCE_ID AS NUMBER(19,0)) AS ACMENTRY_CTB_SEQUENCE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_CURRENCY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_DATE_BASIS AS VARCHAR2(200 CHAR)) AS ACMENTRY_DATE_BASIS, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_DEAL_PRICE AS NUMBER(28,10)) AS ACMENTRY_DEAL_PRICE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_DEAL_RATE AS NUMBER(28,10)) AS ACMENTRY_DEAL_RATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_DESCRIPTION AS VARCHAR2(150 CHAR)) AS ACMENTRY_DESCRIPTION, + U_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_EVENT_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_EVENT_ID AS NUMBER(19,0)) AS ACMENTRY_EVENT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_EXPORT_ID AS NUMBER(19,0)) AS ACMENTRY_EXPORT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FAC_REP_GROUP AS VARCHAR2(50 CHAR)) AS ACMENTRY_FAC_REP_GROUP, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FACILITY_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_FACILITY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FIGURE_AMOUNT AS NUMBER(28,4)) AS ACMENTRY_FIGURE_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FIGURE_BOOKING_AMOUNT AS NUMBER(28,4)) AS ACMENTRY_FIGURE_BOOKING_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FIGURE_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_FIGURE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FLAGS AS VARCHAR2(150 CHAR)) AS ACMENTRY_FLAGS, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FLAGS_2 AS VARCHAR2(150 CHAR)) AS ACMENTRY_FLAGS_2, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_FX_RATE AS NUMBER(28,10)) AS ACMENTRY_FX_RATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_BOOK_EFF_BAL AS NUMBER(19,4)) AS ACMENTRY_HEDGE_BOOK_EFF_BAL, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_BOOK_INEFF_BAL AS NUMBER(19,4)) AS ACMENTRY_HEDGE_BOOK_INEFF_BAL, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_DESIGN_PERCENT AS NUMBER(19,4)) AS ACMENTRY_HEDGE_DESIGN_PERCENT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_DOC_EFF_BAL AS NUMBER(19,4)) AS ACMENTRY_HEDGE_DOC_EFF_BAL, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_DOC_INEFF_BAL AS NUMBER(19,4)) AS ACMENTRY_HEDGE_DOC_INEFF_BAL, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_EFF_PERCENT AS NUMBER(28,10)) AS ACMENTRY_HEDGE_EFF_PERCENT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_ID AS NUMBER(10,0)) AS ACMENTRY_HEDGE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_RISK AS VARCHAR2(20 CHAR)) AS ACMENTRY_HEDGE_RISK, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_STRATEGY_CODE AS VARCHAR2(30 CHAR)) AS ACMENTRY_HEDGE_STRATEGY_CODE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_TEST_METHOD AS VARCHAR2(20 CHAR)) AS ACMENTRY_HEDGE_TEST_METHOD, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_HEDGE_TYPE AS VARCHAR2(20 CHAR)) AS ACMENTRY_HEDGE_TYPE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ID AS NUMBER(19,0)) AS ACMENTRY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_INSTRUMENT_ID AS VARCHAR2(30 CHAR)) AS ACMENTRY_INSTRUMENT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ISSUER_GROUP_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_ISSUER_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ISSUER_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ISSUER_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ISSUER_UNIT_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_ISSUER_UNIT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_LEDGER_ID AS NUMBER(19,0)) AS ACMENTRY_LEDGER_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_LEG_GROUP AS NUMBER(10,0)) AS ACMENTRY_LEG_GROUP, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_LEG_ID AS VARCHAR2(30 CHAR)) AS ACMENTRY_LEG_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_LOAD_ID AS NUMBER(19,0)) AS ACMENTRY_LOAD_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_LOGICAL_REFERENCE_ID AS VARCHAR2(30 CHAR)) AS ACMENTRY_LOGICAL_REFERENCE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_MAPPING_DESCRIPTION AS VARCHAR2(150 CHAR)) AS ACMENTRY_MAPPING_DESCRIPTION, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_MATCH_ID AS NUMBER(19,0)) AS ACMENTRY_MATCH_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_MODIFICATION_TIME, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_N_OPTION_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_NOMINAL_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_NOMINAL_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_NUMBER AS NUMBER(19,0)) AS ACMENTRY_NUMBER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ONE_TIME_CPTY_ID AS VARCHAR2(20 CHAR)) AS ACMENTRY_ONE_TIME_CPTY_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ORIG_BOOKING_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_ORIG_BOOKING_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_ORIGINAL_AMOUNT AS NUMBER(19,4)) AS ACMENTRY_ORIGINAL_AMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_OWNER_UNIT_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_OWNER_UNIT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PACKAGE_MAP_ID AS NUMBER(10,0)) AS ACMENTRY_PACKAGE_MAP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PACKAGE_TYPE_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_PACKAGE_TYPE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_0 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_0, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_1 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_1, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_10 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_10, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_11 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_11, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_12 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_12, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_13 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_13, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_14 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_14, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_15 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_15, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_16 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_16, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_17 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_17, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_18 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_18, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_19 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_19, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_2 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_2, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_3 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_3, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_4 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_4, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_5 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_5, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_6 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_6, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_7 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_7, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_8 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_8, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARAM_9 AS VARCHAR2(50 CHAR)) AS ACMENTRY_PARAM_9, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PARENT_ID AS NUMBER(19,0)) AS ACMENTRY_PARENT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PASE_ID AS NUMBER(19,0)) AS ACMENTRY_PASE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PORTFOLIO_ID AS VARCHAR2(50 CHAR)) AS ACMENTRY_PORTFOLIO_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_PRE_VOUCHER_ID AS NUMBER(10,0)) AS ACMENTRY_PRE_VOUCHER_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_REFERENCE_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_REVERSE_EVENT_ID AS NUMBER(19,0)) AS ACMENTRY_REVERSE_EVENT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_SECONDARY_ACCOUNT_ID AS NUMBER(19,0)) AS ACMENTRY_SECONDARY_ACCOUNT_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_SOURCE_REFERENCE AS VARCHAR2(30 CHAR)) AS ACMENTRY_SOURCE_REFERENCE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_STATUS AS NUMBER(10,0)) AS ACMENTRY_STATUS, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_T_CURRENCY_2_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_T_CURRENCY_2_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_T_CURRENCY_ID AS VARCHAR2(4 CHAR)) AS ACMENTRY_T_CURRENCY_ID, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_T_MATURITY_DATE, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_T_OPENING_DATE, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_T_VALUE_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_TAG AS VARCHAR2(50 CHAR)) AS ACMENTRY_TAG, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRY_VOUCHER_ID AS NUMBER(19,0)) AS ACMENTRY_VOUCHER_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYORIGIN_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYORIGIN_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYORIGINENTITY_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYORIGINENTITY_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYORIGINGROUP_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYORIGINGROUP_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYSTATE1_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYSTATE1_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYSTATE2_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYSTATE2_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMENTRYTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMENTRYTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMEVENTTYPE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMEVENTTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMGROUPINGRULE1_ID AS NUMBER(19,0)) AS ACMGROUPINGRULE1_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMGROUPINGRULE1_ID_USER AS VARCHAR2(30 CHAR)) AS ACMGROUPINGRULE1_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMGROUPINGRULE2_ID AS NUMBER(19,0)) AS ACMGROUPINGRULE2_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMGROUPINGRULE2_ID_USER AS VARCHAR2(30 CHAR)) AS ACMGROUPINGRULE2_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMHEDGECOSTCAT_ID_USER AS VARCHAR2(50 CHAR)) AS ACMHEDGECOSTCAT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMHEDGEEFFCAT_ID_USER AS VARCHAR2(50 CHAR)) AS ACMHEDGEEFFCAT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMHEDGELEGCAT_ID_USER AS VARCHAR2(50 CHAR)) AS ACMHEDGELEGCAT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMHEDGEQUALIFCAT_ID_USER AS VARCHAR2(50 CHAR)) AS ACMHEDGEQUALIFCAT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMLEDGER_CHART_ID AS NUMBER(19,0)) AS ACMLEDGER_CHART_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMLEDGER_ID_USER AS VARCHAR2(30 CHAR)) AS ACMLEDGER_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMMAPPINGACTION_ID_USER AS VARCHAR2(30 CHAR)) AS ACMMAPPINGACTION_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMMAPPINGRULE_ID AS NUMBER(19,0)) AS ACMMAPPINGRULE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMMAPPINGRULE_ID_USER AS VARCHAR2(60 CHAR)) AS ACMMAPPINGRULE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMPERIOD_ID_USER AS VARCHAR2(30 CHAR)) AS ACMPERIOD_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMPLCATEGORY_ID_USER AS VARCHAR2(30 CHAR)) AS ACMPLCATEGORY_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMPROJECT_ID_USER AS VARCHAR2(30 CHAR)) AS ACMPROJECT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMREBOOKINGTYPE_ID_USER AS VARCHAR2(50 CHAR)) AS ACMREBOOKINGTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMREVCANCELSTATE1_ID_USER AS VARCHAR2(30 CHAR)) AS ACMREVCANCELSTATE1_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMREVCANCELSTATE2_ID_USER AS VARCHAR2(30 CHAR)) AS ACMREVCANCELSTATE2_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMREVCANCELSTATE3_ID_USER AS VARCHAR2(30 CHAR)) AS ACMREVCANCELSTATE3_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMSIGN1_NAME AS VARCHAR2(100 CHAR)) AS ACMSIGN1_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMSIGN2_NAME AS VARCHAR2(100 CHAR)) AS ACMSIGN2_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMSPECIALTREATMENT_ID_USER AS VARCHAR2(30 CHAR)) AS ACMSPECIALTREATMENT_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMTRANSACTIONSIGN_NAME AS VARCHAR2(100 CHAR)) AS ACMTRANSACTIONSIGN_NAME, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_BOOKING_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_BUNDLE_ID AS NUMBER(19,0)) AS ACMVOUCHER_BUNDLE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_FLAGS AS VARCHAR2(150 CHAR)) AS ACMVOUCHER_FLAGS, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_MODIFICATION_TIME, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_POSTING_DATE, + OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_REQ_BOOKING_DATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_REV_VOUCHER_ID AS NUMBER(19,0)) AS ACMVOUCHER_REV_VOUCHER_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_REVERSAL_BUNDLE_ID AS NUMBER(19,0)) AS ACMVOUCHER_REVERSAL_BUNDLE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHER_VOUCHER_NUMBER AS VARCHAR2(30 CHAR)) AS ACMVOUCHER_VOUCHER_NUMBER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHERSTATE_ID_USER AS VARCHAR2(30 CHAR)) AS ACMVOUCHERSTATE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHERTYPE_ID_USER AS VARCHAR2(64 CHAR)) AS ACMVOUCHERTYPE_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ACMVOUCHERTYPE2_ID_USER AS VARCHAR2(64 CHAR)) AS ACMVOUCHERTYPE2_ID_USER, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.CCY_GROUPING_RULE AS VARCHAR2(255 CHAR)) AS CCY_GROUPING_RULE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.CCY_POSITION AS VARCHAR2(255 CHAR)) AS CCY_POSITION, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.CCY_STATE AS VARCHAR2(255 CHAR)) AS CCY_STATE, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.CLIENT_UNIT_RELATION_ID AS VARCHAR2(255 CHAR)) AS CLIENT_UNIT_RELATION_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.CTB_CATEGORY AS VARCHAR2(255 CHAR)) AS CTB_CATEGORY, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.HEDGEDEDESIGNATIONTYPE_NAME AS VARCHAR2(255 CHAR)) AS HEDGEDEDESIGNATIONTYPE_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.OPTION_SIGN AS VARCHAR2(255 CHAR)) AS OPTION_SIGN, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.ORIGINAL_TIME_BAND AS VARCHAR2(255 CHAR)) AS ORIGINAL_TIME_BAND, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.REPCCY AS VARCHAR2(255 CHAR)) AS REPCCY, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.REPORT_OWNER_UNIT_GROUP_ID AS VARCHAR2(255 CHAR)) AS REPORT_OWNER_UNIT_GROUP_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.REPORTCCYAMOUNT AS VARCHAR2(255 CHAR)) AS REPORTCCYAMOUNT, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.TRANSACTIONKIND_NAME AS VARCHAR2(255 CHAR)) AS TRANSACTIONKIND_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.TRANSACTIONTYPE_NAME AS VARCHAR2(255 CHAR)) AS TRANSACTIONTYPE_NAME, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.UMPATHNODE_ID AS VARCHAR2(50 CHAR)) AS UMPATHNODE_ID, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.UMPATHNODE_UPATH AS VARCHAR2(100 CHAR)) AS UMPATHNODE_UPATH, + CAST(OU_TMS_ACMENTRYSTATELEDGERGROUP.VALUATIONHIERARCHY_NAME AS VARCHAR2(255 CHAR)) AS VALUATIONHIERARCHY_NAME +FROM + {{ source("ods","ACMENTRYSTATELEDGERGROUP")}} OU_TMS_ACMENTRYSTATELEDGERGROUP +INNER JOIN + {{ filter_workflow_history_max_key( + "w_ODS_TMS_ACMENTRYSTATELEDGERGROUP", + get_main_task_name(model.name) + ) }} wh + ON OU_TMS_ACMENTRYSTATELEDGERGROUP.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ.sql new file mode 100644 index 0000000..b336347 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ.sql @@ -0,0 +1,41 @@ +{{ + config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY"], + alias = "T_ACTIVITY_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-25", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-06-25" + } + ) +}} +SELECT + OU_TMS_ACTIVITY.A_KEY, + OU_TMS_ACTIVITY.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_ACTIVITY.GROUP_ID AS VARCHAR2(50 CHAR)) AS GROUP_ID, + CAST(OU_TMS_ACTIVITY.ID AS VARCHAR2(50 CHAR)) AS ID, + CAST(OU_TMS_ACTIVITY.TYPE_ID AS VARCHAR2(50 CHAR)) AS TYPE_ID, + CAST(OU_TMS_ACTIVITY.DOMAIN_ID AS VARCHAR2(50 CHAR)) AS DOMAIN_ID, + CAST(OU_TMS_ACTIVITY.FLAGS AS VARCHAR2(50 CHAR)) AS FLAGS, + CAST(OU_TMS_ACTIVITY.NAME AS VARCHAR2(255 CHAR)) AS NAME, + CAST(OU_TMS_ACTIVITY.SERVICE_NAME AS VARCHAR2(50 CHAR)) AS SERVICE_NAME, + CAST(OU_TMS_ACTIVITY.SERVICE_GROUP AS VARCHAR2(50 CHAR)) AS SERVICE_GROUP, + OU_TMS_ACTIVITY.START_DATE, + CAST(OU_TMS_ACTIVITY.TIME_ZONE_ID AS VARCHAR2(50 CHAR)) AS TIME_ZONE_ID, + OU_TMS_ACTIVITY.END_DATE, + CAST(OU_TMS_ACTIVITY.PREREQUISITE_ID AS VARCHAR2(50 CHAR)) AS PREREQUISITE_ID, + CAST(OU_TMS_ACTIVITY.OWNER_ID AS VARCHAR2(50 CHAR)) AS OWNER_ID, + CAST(OU_TMS_ACTIVITY.TYPE_NAME AS VARCHAR2(50 CHAR)) AS TYPE_NAME, + OU_TMS_ACTIVITY.START_DATE_LOCAL, + OU_TMS_ACTIVITY.END_DATE_LOCAL +FROM + {{ source("ods","ACTIVITY")}} OU_TMS_ACTIVITY +INNER JOIN + {{ filter_workflow_history_max_key( + "w_ODS_TMS_ACTIVITY", + get_main_task_name(model.name) + ) }} wh + ON OU_TMS_ACTIVITY.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ.sql new file mode 100644 index 0000000..ecaa841 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH"], + alias = "T_BRANCH_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-07-22", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-07-22" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_BRANCH", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_BRANCH.A_KEY, + OU_TMS_BRANCH.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_BRANCH.ID AS VARCHAR2(50)) AS ID, + CAST(OU_TMS_BRANCH.NAME AS VARCHAR2(100)) AS NAME, + CASE + WHEN OU_TMS_BRANCH.NUMBER_ BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_BRANCH.NUMBER_ as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS NUMBER_, + CASE + WHEN OU_TMS_BRANCH.ORDER_NUMBER BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_BRANCH.ORDER_NUMBER as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS ORDER_NUMBER +FROM + {{ source("ods","BRANCH")}} OU_TMS_BRANCH +INNER JOIN + WH ON OU_TMS_BRANCH.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ.sql new file mode 100644 index 0000000..1fe3ab9 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR"], + alias = "T_CALENDAR_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-05", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-06-05" + } +) }} + +SELECT + OU_TMS_CALENDAR.A_KEY, + OU_TMS_CALENDAR.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_CALENDAR.ID as varchar2(50)) AS ID, + CAST(OU_TMS_CALENDAR.NAME as varchar2(255)) AS NAME, + OU_TMS_CALENDAR.HOLIDAY_DATE, + CAST(OU_TMS_CALENDAR.HOLIDAY_REASON as varchar2(255)) AS HOLIDAY_REASON, + CAST(OU_TMS_CALENDAR.DEFAULT_HOLIDAYS as varchar2(50)) AS DEFAULT_HOLIDAYS +FROM + {{ source("ods","CALENDAR")}} OU_TMS_CALENDAR +INNER JOIN + {{ filter_workflow_history_max_key( + "w_ODS_TMS_CALENDAR", + get_main_task_name(model.name) + ) }} wh + ON OU_TMS_CALENDAR.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ.sql new file mode 100644 index 0000000..503b25c --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT"], + alias = "T_CLIENTACCOUNT_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-05-27", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-05-27" + } +) }} + + + SELECT + A_KEY, + A_WORKFLOW_HISTORY_KEY, + CAST(ACCOUNT1 as varchar2(35)) AS ACCOUNT_1, + CAST(ACCOUNT2 as varchar2(35)) AS ACCOUNT_2, + CAST(ACCOUNT3 as varchar2(35)) AS ACCOUNT_3, + CAST(ACCOUNT4 as varchar2(35)) AS ACCOUNT_4, + ACTIVE_SINCE, + ACTIVE_UNTIL, + CAST(BANK1 as varchar2(20)) AS BANK1, + CAST(BANK2 as varchar2(20)) AS BANK2, + CAST(BANK3 as varchar2(20)) AS BANK3, + CAST(BANK4 as varchar2(20)) AS BANK4, + CAST(CLIENT_ID as varchar2(20)) AS CLIENT_ID, + CAST(COMMENT_ as varchar2(255)) AS COMMENT_, + CAST(CURRENCY_ID as varchar2(4)) AS CURRENCY_ID, + CAST(DELIVERY_VS_PAYMENT as NUMBER(10,0)) AS DELIVERY_VS_PAYMENT, + CAST(FULL_CHAIN as NUMBER(10,0)) AS FULL_CHAIN, + CASE + WHEN KIND BETWEEN -9999999999 AND 9999999999 THEN CAST(KIND as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS KIND, + CAST(LAST_LINK as NUMBER(10,0)) AS LAST_LINK, + CAST(NAME as varchar2(255)) AS NAME, + CAST(NOT_RULE_ID as varchar2(50)) AS NOT_RULE_ID, + CAST(OWN_ACCOUNT as NUMBER(10,0)) AS OWN_ACCOUNT, + CAST(PREFER_ACCOUNT as NUMBER(10,0)) AS PREFER_ACCOUNT, + CAST(PREFER_BANK as NUMBER(10,0)) AS PREFER_BANK, + CAST(PRIORITY as NUMBER(10,0)) AS PRIORITY, + CAST(RULE_ID as varchar2(50)) AS RULE_ID +FROM + {{ source("ods","CLIENT_ACCOUNT")}} OU_TMS_CLIENTACCOUNT +WHERE + OU_TMS_CLIENTACCOUNT.A_WORKFLOW_HISTORY_KEY + IN {{ filter_workflow_history_max_key( + "w_ODS_TMS_CLIENT_ACCOUNT", + get_main_task_name(model.name) + ) + }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ.sql new file mode 100644 index 0000000..4f3be40 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP"], + alias = "T_CLIENTGROUPMAP_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-03", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-08-21 - change in load logic -> Jira ticket https://europeancentralbank.atlassian.net/browse/UCDB-3573" + } +) }} + +SELECT + OU_TMS_CLIENTGROUPMAP.A_KEY, + OU_TMS_CLIENTGROUPMAP.A_WORKFLOW_HISTORY_KEY, + CAST(NULL as VARCHAR2(6 CHAR)) AS ECBOPERATION, + TRUNC(wh.workflow_start) AS ECBTS, + CAST(OU_TMS_CLIENTGROUPMAP.CLIENT_ID as VARCHAR2(20 CHAR)) AS CLIENT_ID, + CAST(OU_TMS_CLIENTGROUPMAP.MAIN_GROUP_ID as VARCHAR2(50 CHAR)) AS MAIN_GROUP_ID, + CAST(OU_TMS_CLIENTGROUPMAP.GROUP_ID as VARCHAR2(50 CHAR)) AS GROUP_ID, + CAST(NULL as NUMBER(28,0)) AS STAMP, + CAST(OU_TMS_CLIENTGROUPMAP.MAIN_GROUP_NAME as VARCHAR2(4000 CHAR)) AS MAIN_GROUP_NAME, + CAST(OU_TMS_CLIENTGROUPMAP.GROUP_NAME as VARCHAR2(4000 CHAR)) AS GROUP_NAME +FROM + {{ source("ods","CLIENTGROUPMAP")}} OU_TMS_CLIENTGROUPMAP +INNER JOIN + {{ filter_workflow_history_key( + "w_ODS_TMS_RAR_CLIENTGROUPMAP", + get_main_task_name(model.name), + ", WORKFLOW_START" + ) }} wh + ON OU_TMS_CLIENTGROUPMAP.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY + diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ.sql new file mode 100644 index 0000000..3d894bd --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ.sql @@ -0,0 +1,128 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION"], + alias = "T_CLIENTINSTRUCTION_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-06-30", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-06-30" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_CLIENT_INSTRUCTION", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_CLIENTINSTRUCTION.A_KEY, + OU_TMS_CLIENTINSTRUCTION.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_COMMENT AS VARCHAR2(255 CHAR)) AS ACCOUNT_1_COMMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_CURRENCY AS VARCHAR2(4 CHAR)) AS ACCOUNT_1_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_GROUP AS VARCHAR2(30 CHAR)) AS ACCOUNT_1_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNT_1_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_1_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_PARENT_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACCOUNT_1_PARENT_ACCOUNT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_SECONDARY_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_1_SECONDARY_NUMBER, + OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_TARGET_CLOSING_DATE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_1_TYPE AS VARCHAR2(50 CHAR)) AS ACCOUNT_1_TYPE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_COMMENT AS VARCHAR2(255 CHAR)) AS ACCOUNT_2_COMMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_CURRENCY AS VARCHAR2(4 CHAR)) AS ACCOUNT_2_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_GROUP AS VARCHAR2(30 CHAR)) AS ACCOUNT_2_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNT_2_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_2_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_PARENT_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACCOUNT_2_PARENT_ACCOUNT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_SECONDARY_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_2_SECONDARY_NUMBER, + OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_TARGET_CLOSING_DATE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_2_TYPE AS VARCHAR2(50 CHAR)) AS ACCOUNT_2_TYPE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_COMMENT AS VARCHAR2(255 CHAR)) AS ACCOUNT_3_COMMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_CURRENCY AS VARCHAR2(4 CHAR)) AS ACCOUNT_3_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_GROUP AS VARCHAR2(30 CHAR)) AS ACCOUNT_3_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNT_3_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_3_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_PARENT_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACCOUNT_3_PARENT_ACCOUNT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_SECONDARY_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_3_SECONDARY_NUMBER, + OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_TARGET_CLOSING_DATE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_3_TYPE AS VARCHAR2(50 CHAR)) AS ACCOUNT_3_TYPE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_COMMENT AS VARCHAR2(255 CHAR)) AS ACCOUNT_4_COMMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_CURRENCY AS VARCHAR2(4 CHAR)) AS ACCOUNT_4_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_GROUP AS VARCHAR2(30 CHAR)) AS ACCOUNT_4_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_NAME AS VARCHAR2(255 CHAR)) AS ACCOUNT_4_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_4_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_PARENT_ACCOUNT AS VARCHAR2(35 CHAR)) AS ACCOUNT_4_PARENT_ACCOUNT, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_SECONDARY_NUMBER AS VARCHAR2(35 CHAR)) AS ACCOUNT_4_SECONDARY_NUMBER, + OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_TARGET_CLOSING_DATE, + CAST(OU_TMS_CLIENTINSTRUCTION.ACCOUNT_4_TYPE AS VARCHAR2(50 CHAR)) AS ACCOUNT_4_TYPE, + OU_TMS_CLIENTINSTRUCTION.ACTIVE_SINCE, + OU_TMS_CLIENTINSTRUCTION.ACTIVE_UNTIL, + CAST(OU_TMS_CLIENTINSTRUCTION.BANK_1 AS VARCHAR2(20 CHAR)) AS BANK_1, + CAST(OU_TMS_CLIENTINSTRUCTION.BANK_2 AS VARCHAR2(20 CHAR)) AS BANK_2, + CAST(OU_TMS_CLIENTINSTRUCTION.BANK_3 AS VARCHAR2(20 CHAR)) AS BANK_3, + CAST(OU_TMS_CLIENTINSTRUCTION.BANK_4 AS VARCHAR2(20 CHAR)) AS BANK_4, + CAST(OU_TMS_CLIENTINSTRUCTION.CLIENT_ID AS VARCHAR2(20 CHAR)) AS CLIENT_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.CLIENT_NAME AS VARCHAR2(255 CHAR)) AS CLIENT_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.CLIENT_UNIT_ID AS VARCHAR2(50 CHAR)) AS CLIENT_UNIT_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.COMMENT_ AS VARCHAR2(255 CHAR)) AS COMMENT_, + CAST(OU_TMS_CLIENTINSTRUCTION.CURRENCY_ID AS VARCHAR2(4 CHAR)) AS CURRENCY_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.INSTRUCTION_NAME AS VARCHAR2(255 CHAR)) AS INSTRUCTION_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.KIND AS VARCHAR2(50 CHAR)) AS KIND, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CASHFLOW_CATEGORY AS NUMBER(3,0)) AS NOT_RULE_CF_CATEGORY_ID, + CASE + WHEN OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CF_SUBTYPE_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CF_SUBTYPE_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS NOT_RULE_CF_SUBTYPE_ID, + CASE + WHEN OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CF_TYPE_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CF_TYPE_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS NOT_RULE_CF_TYPE_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_COUNTERPARTY AS VARCHAR2(20 CHAR)) AS NOT_RULE_COUNTERPARTY, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_COUNTERPARTY_GROUP AS VARCHAR2(50 CHAR)) AS NOT_RULE_COUNTERPARTY_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CPTY_MAIN_GROUP AS VARCHAR2(50 CHAR)) AS NOT_RULE_CPTY_MAIN_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CURRENCY AS VARCHAR2(4 CHAR)) AS NOT_RULE_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_CURRENCY_2 AS VARCHAR2(4 CHAR)) AS NOT_RULE_CURRENCY_2, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_ID AS VARCHAR2(50 CHAR)) AS NOT_RULE_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_INSTRUMENT AS VARCHAR2(30 CHAR)) AS NOT_RULE_INSTRUMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_INSTRUMENT_GROUP AS NUMBER(10,0)) AS NOT_RULE_INSTRUMENT_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_NAME AS VARCHAR2(50 CHAR)) AS NOT_RULE_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_NUMBER AS NUMBER(12,0)) AS NOT_RULE_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_OWNER AS VARCHAR2(20 CHAR)) AS NOT_RULE_OWNER, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_PORTFOLIO AS VARCHAR2(50 CHAR)) AS NOT_RULE_PORTFOLIO, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_TRANSACTION_KIND AS VARCHAR2(50 CHAR)) AS NOT_RULE_TRANSACTION_KIND, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_TRANSACTION_SIGN AS VARCHAR2(10 CHAR)) AS NOT_RULE_TRANSACTION_SIGN, + CAST(OU_TMS_CLIENTINSTRUCTION.NOT_RULE_TRANSACTION_TYPE AS VARCHAR2(50 CHAR)) AS NOT_RULE_TRANSACTION_TYPE, + CASE + WHEN OU_TMS_CLIENTINSTRUCTION.PRIORITY BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_CLIENTINSTRUCTION.PRIORITY as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS PRIORITY, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CASHFLOW_CATEGORY AS varchar2(100 char)) AS RULE_CASHFLOW_CATEGORY, + CASE + WHEN OU_TMS_CLIENTINSTRUCTION.RULE_CF_SUBTYPE_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CF_SUBTYPE_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS RULE_CF_SUBTYPE_ID, + CASE + WHEN OU_TMS_CLIENTINSTRUCTION.RULE_CF_TYPE_ID BETWEEN -9999999999 AND 9999999999 THEN CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CF_TYPE_ID as NUMBER(10,0)) + ELSE CAST(NULL as NUMBER(10,0)) -- Handle out-of-range values + END AS RULE_CF_TYPE_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CF_TYPE_NAME AS VARCHAR2(200 CHAR)) AS RULE_CF_TYPE_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_COUNTERPARTY AS VARCHAR2(20 CHAR)) AS RULE_COUNTERPARTY, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_COUNTERPARTY_GROUP AS VARCHAR2(50 CHAR)) AS RULE_COUNTERPARTY_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CPTY_MAIN_GROUP AS VARCHAR2(50 CHAR)) AS RULE_CPTY_MAIN_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CURRENCY AS VARCHAR2(4 CHAR)) AS RULE_CURRENCY, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_CURRENCY_2 AS VARCHAR2(4 CHAR)) AS RULE_CURRENCY_2, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_ID AS VARCHAR2(50 CHAR)) AS RULE_ID, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_INSTRUMENT AS VARCHAR2(30 CHAR)) AS RULE_INSTRUMENT, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_INSTRUMENT_GROUP AS VARCHAR2(50 CHAR)) AS RULE_INSTRUMENT_GROUP, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_NAME AS VARCHAR2(50 CHAR)) AS RULE_NAME, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_NUMBER AS NUMBER(12,0)) AS RULE_NUMBER, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_OWNER AS VARCHAR2(20 CHAR)) AS RULE_OWNER, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_PORTFOLIO AS VARCHAR2(50 CHAR)) AS RULE_PORTFOLIO, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_TRANSACTION_KIND AS VARCHAR2(50 CHAR)) AS RULE_TRANSACTION_KIND, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_TRANSACTION_SIGN AS VARCHAR2(10 CHAR)) AS RULE_TRANSACTION_SIGN, + CAST(OU_TMS_CLIENTINSTRUCTION.RULE_TRANSACTION_TYPE AS VARCHAR2(50 CHAR)) AS RULE_TRANSACTION_TYPE +FROM + {{ source("ods","CLIENT_INSTRUCTION")}} OU_TMS_CLIENTINSTRUCTION +INNER JOIN + WH ON OU_TMS_CLIENTINSTRUCTION.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ.sql new file mode 100644 index 0000000..781acf1 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT"], + alias = "T_CLIENT_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-05-29", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-05-29" + } +) }} + +SELECT + OU_TMS_CLIENT.A_KEY, + OU_TMS_CLIENT.A_WORKFLOW_HISTORY_KEY, + CAST(NULL as VARCHAR2(6 CHAR)) AS ECBOPERATION, + TRUNC(WH_HISTORY.WORKFLOW_START) AS ECBTS, + CAST(OU_TMS_CLIENT.CLIENT as VARCHAR2(20 CHAR)) AS CLIENT_ID, + CAST(OU_TMS_CLIENT.DOMAIN as VARCHAR2(50 CHAR)) AS DOMAIN_ID, + CAST(OU_TMS_CLIENT.COUNTRY as VARCHAR2(2 CHAR)) AS COUNTRY_ID, + CAST(OU_TMS_CLIENT.ROLES as VARCHAR2(100 CHAR)) AS ROLES, + CAST(NULL as NUMBER(28,0)) AS GROUPS, + CAST(NULL as NUMBER(28,0)) AS FLAGS, + CAST(NULL as NUMBER(28,0)) AS STAMP, + CAST(NULL as VARCHAR2(50 CHAR)) AS TEMPLATE_ID, + CAST(OU_TMS_CLIENT.NAME as VARCHAR2(255 CHAR)) AS NAME, + CAST(OU_TMS_CLIENT.PARENT as VARCHAR2(20 CHAR)) AS PARENT_ID, + CAST(NULL as VARCHAR2(50 CHAR)) AS SHORT_NAME, + CAST(OU_TMS_CLIENT.POSTAL_ADDRESS as VARCHAR2(255 CHAR)) AS POSTAL_ADDRESS, + CAST(OU_TMS_CLIENT.STREET_ADDRESS as VARCHAR2(255 CHAR)) AS STREET_ADDRESS, + CAST(NULL as VARCHAR2(255 CHAR)) AS PAYMENT_ADDRESS, + CAST(OU_TMS_CLIENT.CITY as VARCHAR2(100 CHAR)) AS CITY, + CAST(OU_TMS_CLIENT.RISK_COUNTRY as VARCHAR2(2 CHAR)) AS RISK_COUNTRY_ID, + CAST(OU_TMS_CLIENT.SWIFT_CODE as VARCHAR2(20 CHAR)) AS SWIFT_CODE, + CAST(OU_TMS_CLIENT.FLAGS as VARCHAR2(100 CHAR)) AS CLIENT_FLAGS +FROM + {{ source("ods","CLIENT")}} OU_TMS_CLIENT +INNER JOIN + {{ filter_workflow_history_max_key( + "W_ODS_TMS_RAR_CLIENT", + get_main_task_name(model.name) + ) }} wh + ON OU_TMS_CLIENT.A_WORKFLOW_HISTORY_KEY = wh.A_WORKFLOW_HISTORY_KEY +INNER JOIN + {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} WH_HISTORY + ON wh.A_WORKFLOW_HISTORY_KEY = WH_HISTORY.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ.sql new file mode 100644 index 0000000..026279d --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE"], + alias = "T_CUSTODYBALANCE_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-07-03", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-07-03" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_CUSTODYBALANCE", + get_main_task_name(model.name) + ) }} +) + +SELECT + OU_TMS_CUSTODYBALANCE.A_KEY, + OU_TMS_CUSTODYBALANCE.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_CUSTODYBALANCE.ID AS NUMBER(12,0)) AS ID, + CAST(OU_TMS_CUSTODYBALANCE.CLIENT_ID AS VARCHAR2(20 CHAR)) AS CLIENT_ID, + CAST(OU_TMS_CUSTODYBALANCE.CLIENT_UNIT_ID AS VARCHAR2(50 CHAR)) AS CLIENT_UNIT_ID, + CAST(OU_TMS_CUSTODYBALANCE.CUSTODY_ID AS VARCHAR2(20 CHAR)) AS CUSTODY_ID, + CAST(OU_TMS_CUSTODYBALANCE.CUSTODY_UNIT_ID AS VARCHAR2(50 CHAR)) AS CUSTODY_UNIT_ID, + CAST(OU_TMS_CUSTODYBALANCE.ACCOUNT_ID AS VARCHAR2(35 CHAR)) AS ACCOUNT_ID, + OU_TMS_CUSTODYBALANCE.DATE_, + CAST(OU_TMS_CUSTODYBALANCE.INSTRUMENT_ID AS VARCHAR2(30 CHAR)) AS INSTRUMENT_ID, + CAST(OU_TMS_CUSTODYBALANCE.CURRENCY_ID AS VARCHAR2(4 CHAR)) AS CURRENCY_ID, + CAST(OU_TMS_CUSTODYBALANCE.ISSUER_ID AS VARCHAR2(20 CHAR)) AS ISSUER_ID, + CAST(OU_TMS_CUSTODYBALANCE.ISSUER_UNIT_ID AS VARCHAR2(50 CHAR)) AS ISSUER_UNIT_ID, + OU_TMS_CUSTODYBALANCE.MATURITY_DATE, + CAST(OU_TMS_CUSTODYBALANCE.BALANCE_AMOUNT AS NUMBER(19,4)) AS BALANCE_AMOUNT, + CAST(OU_TMS_CUSTODYBALANCE.SELLABLE_AMOUNT AS NUMBER(19,4)) AS SELLABLE_AMOUNT, + CAST(OU_TMS_CUSTODYBALANCE.BALANCE_FACE_AMOUNT AS NUMBER(19,4)) AS BALANCE_FACE_AMOUNT, + CAST(OU_TMS_CUSTODYBALANCE.SELLABLE_FACE_AMOUNT AS NUMBER(19,4)) AS SELLABLE_FACE_AMOUNT, + CAST(OU_TMS_CUSTODYBALANCE.BALANCE_UNITS AS NUMBER(38,8)) AS BALANCE_UNITS, + CAST(OU_TMS_CUSTODYBALANCE.SELLABLE_UNITS AS NUMBER(38,8)) AS SELLABLE_UNITS, + CAST(OU_TMS_CUSTODYBALANCE.COMMENT_ AS VARCHAR2(255 CHAR)) AS COMMENT_, + CAST(OU_TMS_CUSTODYBALANCE.FLAGS AS NUMBER(10,0)) AS FLAGS, + CAST(OU_TMS_CUSTODYBALANCE.INSTRUMENT_GROUP AS VARCHAR2(100 CHAR)) AS INSTRUMENT_GROUP, + CAST(OU_TMS_CUSTODYBALANCE.ISIN_CODE AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + CAST(OU_TMS_CUSTODYBALANCE.CUSIP_CODE AS VARCHAR2(50 CHAR)) AS CUSIP_CODE, + CAST(OU_TMS_CUSTODYBALANCE.SEDOL_CODE AS VARCHAR2(50 CHAR)) AS SEDOL_CODE + MAX(WH.A_WORKFLOW_HISTORY_KEY) OVER () AS MAX_KEY +FROM + {{ source("ods","CUSTODYBALANCE")}} OU_TMS_CUSTODYBALANCE +INNER JOIN + WH ON OU_TMS_CUSTODYBALANCE.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ.sql new file mode 100644 index 0000000..2c3ddd3 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS"], + alias = "T_EFFECTIVEPERMISSIONS_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-07-17", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-07-17" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_EFFECTIVEPERMISSIONS", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_EFFECTIVEPERMISSIONS.A_KEY, + OU_TMS_EFFECTIVEPERMISSIONS.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ID AS VARCHAR2(30 CHAR)) AS ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.NAME AS VARCHAR2(100 CHAR)) AS NAME, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ADMIN_CENTER_ID AS VARCHAR2(30 CHAR)) AS ADMIN_CENTER_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.TIME_ZONE_ID AS VARCHAR2(50 CHAR)) AS TIME_ZONE_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.MENU_NAME AS VARCHAR2(255 CHAR)) AS MENU_NAME, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.WEB_HOME_GROUP_ID AS VARCHAR2(50 CHAR)) AS WEB_HOME_GROUP_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.COMMENT_ AS VARCHAR2(255 CHAR)) AS COMMENT_, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.FLAGS as NUMBER(10,0)) AS FLAGS, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ACCESS_PROFILE_ID AS VARCHAR2(128 CHAR)) AS ACCESS_PROFILE_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ACCESS_PROFILE_NAME AS VARCHAR2(100 CHAR)) AS ACCESS_PROFILE_NAME, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ACCESS_PROFILE_TYPE_ID AS VARCHAR2(128 CHAR)) AS ACCESS_PROFILE_TYPE_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ACCESS_PROFILE_TYPE_NAME AS VARCHAR2(100 CHAR)) AS ACCESS_PROFILE_TYPE_NAME, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.ENTITY_ID AS VARCHAR2(4000 CHAR)) AS ENTITY_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.PERMISSION_ID AS VARCHAR2(255 CHAR)) AS PERMISSION_ID, + CAST(OU_TMS_EFFECTIVEPERMISSIONS.RESOURCE_RULE AS VARCHAR2(4000 CHAR)) AS RESOURCE_RULE, + OU_TMS_EFFECTIVEPERMISSIONS.ACTIVE_FROM, + OU_TMS_EFFECTIVEPERMISSIONS.ACTIVE_TO +FROM + {{ source("ods","EFFECTIVEPERMISSIONS")}} OU_TMS_EFFECTIVEPERMISSIONS +INNER JOIN + WH ON OU_TMS_EFFECTIVEPERMISSIONS.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ.sql new file mode 100644 index 0000000..55d171d --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE"], + alias = "T_EFFECTIVEROLEPROFILE_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-07-18", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-07-18" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_EFFECTIVEROLEPROFILE", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_EFFECTIVEROLEPROFILE.A_KEY, + OU_TMS_EFFECTIVEROLEPROFILE.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ID AS VARCHAR2(30 CHAR)) AS ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.NAME AS VARCHAR2(100 CHAR)) AS NAME, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ADMIN_CENTER_ID AS VARCHAR2(30 CHAR)) AS ADMIN_CENTER_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.TIME_ZONE_ID AS VARCHAR2(50 CHAR)) AS TIME_ZONE_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.MENU_NAME AS VARCHAR2(255 CHAR)) AS MENU_NAME, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.WEB_HOME_GROUP_ID AS VARCHAR2(50 CHAR)) AS WEB_HOME_GROUP_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.FLAGS as NUMBER(10,0)) AS FLAGS, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_ROLE_ID AS VARCHAR2(30 CHAR)) AS ACCESS_ROLE_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_ROLE_NAME AS VARCHAR2(100 CHAR)) AS ACCESS_ROLE_NAME, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.DIRECT_P AS VARCHAR2(10 CHAR)) AS DIRECT_P, + OU_TMS_EFFECTIVEROLEPROFILE.ACTIVE_FROM, + OU_TMS_EFFECTIVEROLEPROFILE.ACTIVE_TO, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_PROFILE_ID AS VARCHAR2(30 CHAR)) AS ACCESS_PROFILE_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_PROFILE_NAME AS VARCHAR2(100 CHAR)) AS ACCESS_PROFILE_NAME, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_PROFILE_TYPE_ID AS VARCHAR2(128 CHAR)) AS ACCESS_PROFILE_TYPE_ID, + CAST(OU_TMS_EFFECTIVEROLEPROFILE.ACCESS_PROFILE_TYPE_NAME AS VARCHAR2(100 CHAR)) AS ACCESS_PROFILE_TYPE_NAME +FROM + {{ source("ods","EFFECTIVEROLEPROFILE")}} OU_TMS_EFFECTIVEROLEPROFILE +INNER JOIN + WH ON OU_TMS_EFFECTIVEROLEPROFILE.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ.sql new file mode 100644 index 0000000..c371a63 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG"], + alias = "T_HISTORY_LOG_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-08-19", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-08-19" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_key( + "w_ODS_TMS_HISTORY_LOG", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_HISTORY_LOG.A_KEY, + OU_TMS_HISTORY_LOG.A_WORKFLOW_HISTORY_KEY, + CAST(OU_TMS_HISTORY_LOG.OBJECT_ID AS VARCHAR2(50)) AS OBJECT_ID, + CAST(OU_TMS_HISTORY_LOG.ACTION AS VARCHAR2(255)) AS ACTION, + CAST(OU_TMS_HISTORY_LOG.USER_ID AS VARCHAR2(30)) AS USER_ID, + OU_TMS_HISTORY_LOG.DATE_, + CAST(OU_TMS_HISTORY_LOG.KEY_1_STRING AS VARCHAR2(300)) AS KEY_1_STRING, + CAST(OU_TMS_HISTORY_LOG.FIELD_NAME AS VARCHAR2(255)) AS FIELD_NAME, + CAST(OU_TMS_HISTORY_LOG.KEY_2_STRING AS VARCHAR2(255)) AS KEY_2_STRING, + CAST(OU_TMS_HISTORY_LOG.KEY_3_STRING AS VARCHAR2(255)) AS KEY_3_STRING, + CAST(OU_TMS_HISTORY_LOG.KEY_4_STRING AS VARCHAR2(255)) AS KEY_4_STRING, + CAST(OU_TMS_HISTORY_LOG.FROM_VALUE AS VARCHAR2(300)) AS FROM_VALUE, + CAST(OU_TMS_HISTORY_LOG.TO_VALUE AS VARCHAR2(300)) AS TO_VALUE, + CAST(OU_TMS_HISTORY_LOG.DOMAIN_ID AS VARCHAR2(50)) AS DOMAIN_ID +FROM + {{ source("ods","HISTORY_LOG")}} OU_TMS_HISTORY_LOG +INNER JOIN + WH ON OU_TMS_HISTORY_LOG.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ.sql new file mode 100644 index 0000000..a6effc1 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ.sql @@ -0,0 +1,35 @@ +{{ + config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS"], + alias = "T_LIMIT_ACCESS_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-08-21", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-08-21" + } + ) +}} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_LIMIT_ACCESS", + get_main_task_name(model.name) + ) }} +) + +SELECT +OU_TMS_RARLIMITACCESS.A_KEY, +OU_TMS_RARLIMITACCESS.A_WORKFLOW_HISTORY_KEY, +CAST(NULL AS VARCHAR2(6)) AS ECBOPERATION, +CAST(TO_TIMESTAMP_TZ('{{ run_started_at }}','YYYY-MM-DD HH24:MI:SS.FF TZH:TZM') AS DATE) AS ECBTS, +--TO_DATE('{{ run_started_at }}','DD/MM/YYYY HH24:MI:SS') AS ECBTS, +CAST(OU_TMS_RARLIMITACCESS.LIMIT_ID AS VARCHAR2(50)) AS LIMIT_ID, +CAST(OU_TMS_RARLIMITACCESS.USER_ID AS VARCHAR2(30)) AS USER_ID, +OU_TMS_RARLIMITACCESS.ECBUSRBDTS +FROM + {{ source("ods","LIMIT_ACCESS")}} OU_TMS_RARLIMITACCESS +INNER JOIN + WH ON OU_TMS_RARLIMITACCESS.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ.sql new file mode 100644 index 0000000..403f8f3 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS"], + alias = "T_PORTFOLIO_ACCESS_SQ", + schema = "TMS", + meta={ + "author": "Adelina Borisova", + "created": "2025-10-23", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-10-23" + } +) }} +WITH WH AS ( + + {{ filter_workflow_history_max_key( + "w_ODS_TMS_PORTFOLIO_ACCESS", + get_main_task_name(model.name) + ) }} +) +SELECT + OU_TMS_RARPORTFOLIOACCESS.A_KEY, + OU_TMS_RARPORTFOLIOACCESS.A_WORKFLOW_HISTORY_KEY, + CAST(TO_TIMESTAMP_TZ('{{ run_started_at }}','YYYY-MM-DD HH24:MI:SS.FF TZH:TZM') AS DATE) AS ECBTS, + CAST(OU_TMS_RARPORTFOLIOACCESS.PORTFOLIO_ID AS VARCHAR2(50 CHAR)) AS PORTFOLIO_ID, + CAST(OU_TMS_RARPORTFOLIOACCESS.USER_ID AS VARCHAR2(30 CHAR)) AS USER_ID, + OU_TMS_RARPORTFOLIOACCESS.ECBUSRBDTS +FROM + {{ source("ods","PORTFOLIO_ACCESS")}} OU_TMS_RARPORTFOLIOACCESS +INNER JOIN + WH ON OU_TMS_RARPORTFOLIOACCESS.A_WORKFLOW_HISTORY_KEY = WH.A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ.sql new file mode 100644 index 0000000..57bfb7b --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ.sql @@ -0,0 +1,23 @@ +{{ + config( + materialized='table' , + schema='TMS', + alias='T_ROLEPORTFOLIOPROFILE_SQ', + tags=["m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE"] + ) +}} + +select +A_KEY , +A_WORKFLOW_HISTORY_KEY , +CAST(ROLE AS VARCHAR2(512 CHAR)) as ROLE, +CAST(INHERITED_ROLE AS VARCHAR2(512 CHAR)) as INHERITED_ROLE, +CAST(ACCESS_PROFILE_ID AS VARCHAR2(512 CHAR)) as ACCESS_PROFILE_ID, +CAST(ACCESS_PROFILE_TYPE_ID AS VARCHAR2(512 CHAR)) as ACCESS_PROFILE_TYPE_ID, +CAST(PERMISSION_ID AS VARCHAR2(512 CHAR)) as PERMISSION_ID, +CAST(ENTITY_ID AS VARCHAR2(512 CHAR)) as ENTITY_ID +from + {{source("ods","ROLE_PORTFOLIO_PROFILE")}} +where + A_WORKFLOW_HISTORY_KEY in + {{filter_workflow_history_max_key("w_ODS_TMS_ROLEPORTFOLIOPROFILE",get_main_task_name(model.name))}} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ.sql new file mode 100644 index 0000000..7f2a2a2 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ.sql @@ -0,0 +1,59 @@ +{{ + config( + materialized="table", + schema="TMS", + alias="T_SECURITYPOSITION_SQ" , + tags=["m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION"] + ) +}} + +Select +A_KEY , +A_WORKFLOW_HISTORY_KEY , +DATE_ , +CAST(PORTFOLIO_ID AS VARCHAR2(50 CHAR)) as PORTFOLIO_ID, +CAST(INSTRUMENT_ID AS VARCHAR2(30 CHAR)) as INSTRUMENT_ID, +CONTEXTS , +CAST(CURRENCY_ID AS VARCHAR2(4 CHAR)) as CURRENCY_ID, +CAST(PACKAGE_MAIN_TYPE_ID AS VARCHAR2(50 CHAR)) as PACKAGE_MAIN_TYPE_ID, +CAST(PACKAGE_TYPE_ID AS VARCHAR2(50 CHAR)) as PACKAGE_TYPE_ID, +PACKAGE_ID , +INFLOW_NOMINAL_AMOUNT , +OUTFLOW_NOMINAL_AMOUNT , +NOMINAL_AMOUNT , +INFLOW_UNITS , +OUTFLOW_UNITS , +UNITS , +INFLOW_BOOK_VALUE , +OUTFLOW_BOOK_VALUE , +BOOK_VALUE , +REDEMPTION_AMOUNT , +ACCRUED_PROFIT , +INDEXATION_PROFIT , +DISCOUNT_PREMIUM , +ACCRUED_BOOK_VALUE , +REALIZED_MTOM_PROFIT , +REALIZED_ACCRUED_PROFIT , +REALIZED_INDEXATION_PROFIT , +BVC_MTOM_PROFIT , +MTOM_PROFIT , +MARKET_VALUE , +ACCRUED_INTEREST , +ACCRUED_INTEREST_BOUGHT , +ACCRUED_INTEREST_SOLD , +YIELD , +PRICE , +MARKET_QUOTE , +INDEX_VALUE , +CAST(ISSUER_ID AS VARCHAR2(20 CHAR)) as ISSUER_ID, +ISSUE_DATE , +MATURITY_DATE , +CAST(ISIN_CODE AS VARCHAR2(50 CHAR)) as ISIN_CODE, +CAST(ACCRUAL_METHOD AS VARCHAR2(200 CHAR)) as ACCRUAL_METHOD, +CAST(ACCRUAL_INTEREST_TYPE AS VARCHAR2(200 CHAR)) as ACCRUAL_INTEREST_TYPE, +CAST(ACCRUAL_DATE_BASIS AS VARCHAR2(200 CHAR)) as ACCRUAL_DATE_BASIS, +INPUT_DATE , +FLAGS , +FACE_AMOUNT +from + {{source("ods","TMS_SECURITY_POSITION")}} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ.sql new file mode 100644 index 0000000..de42c02 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ.sql @@ -0,0 +1,59 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT"], + alias = "T_AGGREGATED_ALLOTMENT_SQ", + schema = "TOP", +) }} +SELECT MAX(A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_TOP_AGGREGATED_ALLOTMENT.A_KEY, +OU_TOP_AGGREGATED_ALLOTMENT.A_WORKFLOW_HISTORY_KEY, +OU_TOP_AGGREGATED_ALLOTMENT.REF_NO, +OU_TOP_AGGREGATED_ALLOTMENT.TIMESTAMP_SENDER, +OU_TOP_AGGREGATED_ALLOTMENT.ID, +OU_TOP_AGGREGATED_ALLOTMENT.CHECKSUM, +OU_TOP_AGGREGATED_ALLOTMENT.REVISION, +OU_TOP_AGGREGATED_ALLOTMENT.TRANSACTION_TYPE, +OU_TOP_AGGREGATED_ALLOTMENT.OPERATION_CURRENCY, +OU_TOP_AGGREGATED_ALLOTMENT.OPERATION_NAME, +OU_TOP_AGGREGATED_ALLOTMENT.OPERATION_CATEGORY, +OU_TOP_AGGREGATED_ALLOTMENT.OPERATION_TYPE, +OU_TOP_AGGREGATED_ALLOTMENT.PROCEDURE, +OU_TOP_AGGREGATED_ALLOTMENT.AUCTION_TYPE, +OU_TOP_AGGREGATED_ALLOTMENT.ALLOTMENT_METHOD, +OU_TOP_AGGREGATED_ALLOTMENT.QUOTATION, +OU_TOP_AGGREGATED_ALLOTMENT.ALLOTMENT_DATE, +OU_TOP_AGGREGATED_ALLOTMENT.VALUE_DATE, +OU_TOP_AGGREGATED_ALLOTMENT.MATURITY_DATE, +OU_TOP_AGGREGATED_ALLOTMENT.DURATION_OF_OPERATION, +OU_TOP_AGGREGATED_ALLOTMENT.REFERENCE_ENTITY, +OU_TOP_AGGREGATED_ALLOTMENT.SPOT_RATE, +OU_TOP_AGGREGATED_ALLOTMENT.REPAID_REF_NO, +OU_TOP_AGGREGATED_ALLOTMENT.BUSINESS_VALUE_TYPE, +OU_TOP_AGGREGATED_ALLOTMENT.BID_CURRENCY, +OU_TOP_AGGREGATED_ALLOTMENT.OTHER_CURRENCY, +OU_TOP_AGGREGATED_ALLOTMENT.TOTAL_NUMBER_BIDDERS, +OU_TOP_AGGREGATED_ALLOTMENT.TOTAL_BID_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.TOTAL_ALLOTTED_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.NCB_SENDER_REFERENCE, +OU_TOP_AGGREGATED_ALLOTMENT.NCB_NUMBER_BIDDERS, +OU_TOP_AGGREGATED_ALLOTMENT.NCB_BID_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.NCB_ALLOTTED_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.BIDDER_ID, +OU_TOP_AGGREGATED_ALLOTMENT.BIDDER_NAME, +OU_TOP_AGGREGATED_ALLOTMENT.BIDDER_BID_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.BIDDER_ALLOTTED_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.BIDDER_ALLOTTED_BUSINESS_VALUE, +OU_TOP_AGGREGATED_ALLOTMENT.BID_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.BID_BUSINESS_VALUE, +OU_TOP_AGGREGATED_ALLOTMENT.BID_ALLOTTED_AMOUNT, +OU_TOP_AGGREGATED_ALLOTMENT.BID_ALLOTTED_BUSINESS_VALUE, +OU_TOP_AGGREGATED_ALLOTMENT.BID_ALLOTTED_PERCENTAGE +FROM + {{ source("ou_top", "AGGREGATED_ALLOTMENT") }} OU_TOP_AGGREGATED_ALLOTMENT +WHERE + REF_NO IS NOT NULL AND + OU_TOP_AGGREGATED_ALLOTMENT.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_TOP_AGGREGATED_ALLOTMENT", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ.sql new file mode 100644 index 0000000..19cea4c --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = "table", + schema = "TOP", + alias = "T_ALLOTMENT_MODIFICATIONS_AGGREGATED_SQ", + tags = ["m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_LOAD"], + meta={ + "author": "Atanas Georgiev", + "created": "2025-11-14", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-11-14" + } +) }} + +SELECT +MAX(A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, +A_WORKFLOW_HISTORY_KEY, +A_KEY, +CHECKSUM , +CAST(ID AS NUMBER(38,0)) as ID, +CAST(OPERATION_CURRENCY AS VARCHAR2(3 CHAR)) as OPERATION_CURRENCY, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST(REVISION AS NUMBER(38,0)) as REVISION, + cast( to_timestamp_tz(TIMESTAMP_SENDER ,'YYYY-MM-DD"T"HH24:MI:SS.FFTZH:TZM') as date ) as TIMESTAMP_SENDER, +CAST(AMOUNT_ITEM AS NUMBER(30,8)) as AMOUNT_ITEM, +CAST(BIDDER_ID AS VARCHAR2(15 CHAR)) as BIDDER_ID, +CAST(FREE_TEXT AS VARCHAR2(4000 CHAR)) as FREE_TEXT, +CAST(SENDER_REFERENCE AS VARCHAR2(3 CHAR)) as SENDER_REFERENCE, +to_date(trim(VALUE_DATE) , 'YYYY-MM-DD' ) as VALUE_DATE, +CAST(NEW_BIDDER AS VARCHAR2(4000 CHAR)) as NEW_BIDDER +FROM + {{ source("ods","ALLOTMENT_MODIFICATION_AGGREGATED" )}} ALLOTMENT_MODIFICATION_AGGREGATED +WHERE + REF_NO IS NOT NULL AND A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_TOP_ALLOTMENT_MODIFICATIONS_AGGREGATED", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ.sql new file mode 100644 index 0000000..068c855 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = "table", + schema = "TOP", + alias = "T_ALLOTMENT_MODIFICATIONS_SQ", + tags = ["m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS"], +) }} + +SELECT +MAX(OU_TOP_ALLOTMENT_MOD_HEADER.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_TOP_ALLOTMENT_MOD_HEADER.REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_TOP_ALLOTMENT_MOD_HEADER.A_WORKFLOW_HISTORY_KEY, +OU_TOP_ALLOTMENT_MOD_HEADER.REF_NO, +OU_TOP_ALLOTMENT_MOD_HEADER.OPERATION_CURRENCY, +OU_TOP_ALLOTMENT_MOD_HEADER.TIMESTAMP_, +OU_TOP_ALLOTMENT_MOD_HEADER.REVISION, +OU_TOP_ALLOTMENT_MOD_HEADER.TIMESTAMP_SENDER, +OU_TOP_ALLOTMENT_MOD_HEADER.ID, +OU_TOP_ALLOTMENT_MOD_HEADER.CHECKSUM, +OU_TOP_ALLOTMENT_MOD_ITEM.SENDER_REFERENCE, +OU_TOP_ALLOTMENT_MOD_ITEM.BIDDER_ID, +OU_TOP_ALLOTMENT_MOD_ITEM.FLOW_TYPE, +OU_TOP_ALLOTMENT_MOD_ITEM.AMOUNT_ITEM, +OU_TOP_ALLOTMENT_MOD_ITEM.VALUE_DATE, +OU_TOP_ALLOTMENT_MOD_ITEM.FREE_TEXT +FROM + {{ source("ou_top", "allotment_modification_header") }} OU_TOP_ALLOTMENT_MOD_HEADER , + {{ source("ou_top", "allotment_modification_item") }} OU_TOP_ALLOTMENT_MOD_ITEM + +WHERE OU_TOP_ALLOTMENT_MOD_HEADER.A_KEY = OU_TOP_ALLOTMENT_MOD_ITEM.A_ALL_MOD_FK AND +OU_TOP_ALLOTMENT_MOD_HEADER.A_WORKFLOW_HISTORY_KEY = OU_TOP_ALLOTMENT_MOD_ITEM.A_WORKFLOW_HISTORY_KEY +AND + OU_TOP_ALLOTMENT_MOD_HEADER.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_TOP_ALLOTMENT_MODIFICATIONS", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ.sql new file mode 100644 index 0000000..dd64390 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT"], + alias = "T_ALLOTMENT_SQ", + schema = "TOP", +) }} + +SELECT +MAX(OU_TOP_ALLOTMENT.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_TOP_ALLOTMENT.REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, + OU_TOP_ALLOTMENT.A_KEY, + OU_TOP_ALLOTMENT.A_WORKFLOW_HISTORY_KEY, + OU_TOP_ALLOTMENT.REF_NO, + OU_TOP_ALLOTMENT.CURRENCY_INFO, + OU_TOP_ALLOTMENT.SENDER_REF, + OU_TOP_ALLOTMENT.USER_REF, + OU_TOP_ALLOTMENT.TIMESTAMP, + OU_TOP_ALLOTMENT.VERSION, + OU_TOP_ALLOTMENT.AMOUNT, + OU_TOP_ALLOTMENT.NUMBER_BIDDERS, + OU_TOP_ALLOTMENT.CUTOFF_LEVEL, + OU_TOP_ALLOTMENT.AMNT_ALLOTED, + OU_TOP_ALLOTMENT.MIN_ALLOTMENT, + OU_TOP_ALLOTMENT.PERC_ALLOTED, + OU_TOP_ALLOTMENT.MARGINAL_RATE_TYPE, + OU_TOP_ALLOTMENT.MARGINAL_RATE, + OU_TOP_ALLOTMENT.MIN_RATE_TYPE, + OU_TOP_ALLOTMENT.MIN_RATE, + OU_TOP_ALLOTMENT.MAX_RATE_TYPE, + OU_TOP_ALLOTMENT.MAX_RATE, + OU_TOP_ALLOTMENT.AVG_RATE_TYPE, + OU_TOP_ALLOTMENT.AVG_RATE, + OU_TOP_ALLOTMENT.SPOT_RATE, + OU_TOP_ALLOTMENT.FREE_TEXT, + OU_TOP_ALLOTMENT.CHECKSUM, + OU_TOP_ALLOTMENT.PRINT_DATE, + OU_TOP_ALLOTMENT.ISIN, + OU_TOP_ALLOTMENT.REF_ENTITY, + OU_TOP_ALLOTMENT.REF_ENTITY_VALUE, + OU_TOP_ALLOTMENT.SPOT_RATE_WITH_MARGIN, + OU_TOP_ALLOTMENT.TIMESTAMP_SENDER, + OU_TOP_ALLOTMENT.ID, + OU_TOP_ALLOTMENT.REPAID_REF_NO, + OU_TOP_ALLOTMENT.REVISION +FROM + {{ source("ou_top", "ALLOTMENT") }} OU_TOP_ALLOTMENT +WHERE + REF_NO IS NOT NULL AND + OU_TOP_ALLOTMENT.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_TOP_ALLOTMENT", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ.sql new file mode 100644 index 0000000..e6a39a8 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ.sql @@ -0,0 +1,66 @@ + + +{{ config( + materialized = "table", + tags = ["m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT"], + alias = "T_ANNOUNCEMENT_SQ", + schema = "TOP", +) }} + +SELECT +MAX(OU_TOP_ANNOUNCEMENT.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_TOP_ANNOUNCEMENT.REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, +OU_TOP_ANNOUNCEMENT.A_WORKFLOW_HISTORY_KEY, +OU_TOP_ANNOUNCEMENT.REF_NO, +OU_TOP_ANNOUNCEMENT.CURRENCYINFO, +OU_TOP_ANNOUNCEMENT.SENDER_REF, +OU_TOP_ANNOUNCEMENT.USER_REF, +OU_TOP_ANNOUNCEMENT.TIMESTAMP, +OU_TOP_ANNOUNCEMENT.VERSION, +OU_TOP_ANNOUNCEMENT.OPERATION_TYPE, +OU_TOP_ANNOUNCEMENT.PROCEDURE, +OU_TOP_ANNOUNCEMENT.TRANSACTION_TYPE, +OU_TOP_ANNOUNCEMENT.OPERATION_NAME, +OU_TOP_ANNOUNCEMENT.AUCTION_TYPE, +OU_TOP_ANNOUNCEMENT.ALL_METHOD, +OU_TOP_ANNOUNCEMENT.QUOTATION, +OU_TOP_ANNOUNCEMENT.BUSINESS_VALUE_TYPE, +OU_TOP_ANNOUNCEMENT.BUSINESS_VALUE, +OU_TOP_ANNOUNCEMENT.ALLOTMENT_DATE, +OU_TOP_ANNOUNCEMENT.VALUE_DATE, +OU_TOP_ANNOUNCEMENT.MATURITY_DATE, +OU_TOP_ANNOUNCEMENT.DURATION_OF_OPERATION, +OU_TOP_ANNOUNCEMENT.SPOT_RATE, +OU_TOP_ANNOUNCEMENT.OTHER_CURRENCY, +OU_TOP_ANNOUNCEMENT.OPERATION_CURRENCY_LEG, +OU_TOP_ANNOUNCEMENT.MIN_BID_AMT, +OU_TOP_ANNOUNCEMENT.MAX_BID_AMT, +OU_TOP_ANNOUNCEMENT.INTENDED_VOLUME, +OU_TOP_ANNOUNCEMENT.MIN_ALLOTMENT, +OU_TOP_ANNOUNCEMENT.MIN_ALLOTMENT_RATIO, +OU_TOP_ANNOUNCEMENT.DEADLINE_CP, +OU_TOP_ANNOUNCEMENT.DEADLINE_NCB, +OU_TOP_ANNOUNCEMENT.PUBLIC_ANNOUNCEMENT, +OU_TOP_ANNOUNCEMENT.FREE_TEXT, +OU_TOP_ANNOUNCEMENT.CHECKSUM, +OU_TOP_ANNOUNCEMENT.PRINT_DATE, +OU_TOP_ANNOUNCEMENT.OPER_CURRENCY, +OU_TOP_ANNOUNCEMENT.ISIN, +OU_TOP_ANNOUNCEMENT.REF_ENTITY, +OU_TOP_ANNOUNCEMENT.SPOT_RATE_WITH_MARGIN, +OU_TOP_ANNOUNCEMENT.MAX_BIDS_COUNTER_PARTY, +OU_TOP_ANNOUNCEMENT.EXACT_DENOMINATION, +OU_TOP_ANNOUNCEMENT.BID_CURRENCY, +OU_TOP_ANNOUNCEMENT.TIMESTAMP_SENDER, +OU_TOP_ANNOUNCEMENT.ID, +OU_TOP_ANNOUNCEMENT.REPAID_REF_NO, +OU_TOP_ANNOUNCEMENT.REVISION, +OU_TOP_ANNOUNCEMENT.OPERATION_CATEGORY +FROM + {{ source("ou_top", "ANNOUNCEMENT") }} OU_TOP_ANNOUNCEMENT +WHERE + REF_NO IS NOT NULL AND + OU_TOP_ANNOUNCEMENT.A_WORKFLOW_HISTORY_KEY IN + {{ filter_workflow_history_key( + "w_ODS_TOP_ANNOUNCEMENT", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ.sql new file mode 100644 index 0000000..89680ff --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_LOAD"], + alias = "T_FBL_ITEM_SQ", + schema = "TOP", +) }} +SELECT /*+ PARALLEL(OU_TOP_FBL_ITEM,4) */ + MAX(OU_TOP_FBL_ITEM.A_WORKFLOW_HISTORY_KEY) OVER ( PARTITION BY OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO ) as MAX_A_WORKFLOW_HISTORY_KEY, + OU_TOP_FBL_ITEM.A_WORKFLOW_HISTORY_KEY, + OU_TOP_FBL_ITEM.SEND_REF, + OU_TOP_FBL_ITEM.LARGE_BIDDER_NAME, + OU_TOP_FBL_ITEM.LARGE_BIDDER_ID, + OU_TOP_FBL_ITEM.AMOUNT, + OU_TOP_FBL_ITEM.AMOUNT_BUS_VALUE_TYPE, + OU_TOP_FBL_ITEM.AMOUNT_BUS_VALUE, + OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO, + OU_TOP_FULLBID_ARRAY_COMPILED.REVISION +FROM + {{ source("ou_top", "FBL_ITEM") }} OU_TOP_FBL_ITEM, + {{ source("ou_top", "FULLBID_ARRAY_COMPILED") }} OU_TOP_FULLBID_ARRAY_COMPILED +WHERE OU_TOP_FULLBID_ARRAY_COMPILED.A_KEY = OU_TOP_FBL_ITEM.A_FBACOMPILED_FK AND +OU_TOP_FULLBID_ARRAY_COMPILED.A_WORKFLOW_HISTORY_KEY = OU_TOP_FBL_ITEM.A_WORKFLOW_HISTORY_KEY AND +OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO IS NOT NULL AND +OU_TOP_FBL_ITEM.A_WORKFLOW_HISTORY_KEY IN +{{ filter_workflow_history_key( + "w_ODS_TOP_FULLBIDARRAY_COMPILED", + get_main_task_name(model.name) + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ.sql b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ.sql new file mode 100644 index 0000000..4489f73 --- /dev/null +++ b/dbt/models/mopdb/source_qualifiers/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = "table", + tags = ["m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_LOAD"], + alias = "T_FULLBID_ARRAY_COMPILED_SQ", + schema = "TOP", +) }} + +SELECT /*+ PARALLEL(OU_TOP_FULLBID_ARRAY_COMPILED,4) */ +MAX(OU_TOP_FULLBID_ARRAY_COMPILED.a_workflow_history_key) OVER ( PARTITION BY OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO ) as MAX_a_workflow_history_key, + ou_top_fullbid_array_compiled.a_workflow_history_key, +OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO, +OU_TOP_FULLBID_ARRAY_COMPILED.OPER_CURR, +OU_TOP_FULLBID_ARRAY_COMPILED.CURR_INFO, +OU_TOP_FULLBID_ARRAY_COMPILED.NUM_BIDDERS, +OU_TOP_FULLBID_ARRAY_COMPILED.NUM_BIDS, +OU_TOP_FULLBID_ARRAY_COMPILED.MIN_RATE_TYPE, +OU_TOP_FULLBID_ARRAY_COMPILED.MIN_RATE, +OU_TOP_FULLBID_ARRAY_COMPILED.MAX_RATE_TYPE, +OU_TOP_FULLBID_ARRAY_COMPILED.MAX_RATE, +OU_TOP_FULLBID_ARRAY_COMPILED.IS_ASCENDING, +OU_TOP_FULLBID_ARRAY_COMPILED.ID, +OU_TOP_FULLBID_ARRAY_COMPILED.BUSINESS_VALUE, +OU_TOP_FULLBID_ARRAY_COMPILED.BUSINESS_VALUE_TYPE, +OU_TOP_FULLBID_ARRAY_COMPILED.REPAID_REF_NO, +OU_TOP_FULLBID_ARRAY_COMPILED.TIMESTAMP_SENDER, +OU_TOP_FULLBID_ARRAY_COMPILED.REVISION , + OU_TOP_FULLBID_ARRAY_COMPILED.CHECKSUM +FROM +{{ source("ods", "FULLBID_ARRAY_COMPILED") }} OU_TOP_FULLBID_ARRAY_COMPILED +WHERE +OU_TOP_FULLBID_ARRAY_COMPILED.REF_NO IS NOT NULL +AND OU_TOP_FULLBID_ARRAY_COMPILED.a_workflow_history_key +in +{{ filter_workflow_history_key( + "w_ODS_TOP_FULLBIDARRAY_COMPILED", + get_main_task_name(model.name) + ) }} diff --git a/dbt/models/mopdb/targets/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING.sql b/dbt/models/mopdb/targets/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING.sql new file mode 100644 index 0000000..c583df6 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING.sql @@ -0,0 +1,40 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "C2D", + alias = "T_CEPH_MARS", + tags = ["m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING"], + meta={ + "author": "Shane Dalisay", + "created": "2026-01-13", + "last_updated_by": "Shane Dalisay", + "last_updated": "2026-01-13 - ticket ticket MARS-193" + } + ) +}} + +WITH c2d_v_ea_snapshot_dates as ( + SELECT + SNAPSHOT_DATE, + CREATION_TIME + FROM {{ source("c2d","V_EA_SNAPSHOT_DATES")}} +) + +SELECT DISTINCT + sq.ISIN_CODE, + sq.WARNING_CODE, + sq.PRICE_NATURE, + sq.PRICE, + sq.ACCRUED_INTEREST, + sq.POOL_FACTOR, + sq.PRICE_DATE, + c2d_v_ea_snapshot_dates.SNAPSHOT_DATE AS SNAPSHOT_DATE, + sq.WAL, + sq.CLEAN_PRICE_WO_MARKDOWN, + sq.ACCRUED_INTEREST_WO_MARKDOWN, + sq.THEORETICAL_PRICE, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY +FROM {{ref("m_MOPDB_C2D_T_CEPH_OU_CEPH_PRICING_SQ")}} sq +LEFT JOIN c2d_v_ea_snapshot_dates + ON TRUNC(sq.PRICE_DATE) = c2d_v_ea_snapshot_dates.CREATION_TIME \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_AGGREGATED.sql b/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_AGGREGATED.sql new file mode 100644 index 0000000..a3f371e --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_AGGREGATED.sql @@ -0,0 +1,198 @@ +{{ config(materialized='table', + schema='C2D', + alias='T_UC_AGGREGATED', + tags=["m_MOPDB_C2D_UC_AGGREGATED"] +) }} + +select 'MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + ISSUER_CAS AS ISSUER_CRED_ASSESS_SOURCE, + ISSUER_RATING_ENUM_VALUE, + ISSUER_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(18 CHAR)) AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + ISSUER_CRED_PROVIDER AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +from {{ ref('m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_MA_DISSEM_SQ') }} + +UNION + +select 'NON_MARKET_DECC' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + ISIN_CODE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + NOM_AMT_SUBMITTED AS COLL_VALUE_BEFORE_HAIRCUTS, + NOM_AMT_SUBMITTED AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_DEBTORS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(200 CHAR)) AS DEBTOR_NAME, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_ID, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_GROUP, + CAST(NULL AS VARCHAR2(17 CHAR)) AS DEBTOR_CLASS, + CAST(NULL AS VARCHAR2(50 CHAR)) AS DEBTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS DEBTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS DEBTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS DEBTOR_RATING_NUMBER_VALUE, + CAST(NULL AS VARCHAR2(255 CHAR)) AS GUARANTOR_NAME, + CAST(NULL AS VARCHAR2(256 CHAR)) AS GUARANTOR_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_RESIDENCE, + CAST(NULL AS VARCHAR2(4 CHAR)) AS GUARANTOR_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(20 CHAR)) AS GUARANTOR_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS GUARANTOR_RATING_NUMBER_VALUE, + CAST(NULL AS NUMBER(10,0)) AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + CAST(NULL AS DATE) AS MATURITY_DATE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(5 CHAR)) AS RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(22 CHAR)) AS BUCKET_SIZE, + CAST(NULL AS VARCHAR2(100 CHAR)) AS DEBTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(1 CHAR)) AS SECURED, + CAST(NULL AS VARCHAR2(10 CHAR)) AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + TRIPARTY_AGENT, + CAST(NULL AS VARCHAR2(5 CHAR)) AS DEBTOR_IDTYPE, + CAST(NULL AS VARCHAR2(1 CHAR)) AS CAP, + CAST(NULL AS VARCHAR2(9 CHAR)) AS REFERENCE_RATE, + CAST(NULL AS VARCHAR2(200 CHAR)) AS REFERENCE_RATE_COMMENT, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RESET_PERIOD_MORE_ONE_YEAR, + CAST(NULL AS VARCHAR2(50 CHAR)) AS OA_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS CONTRACT_ID, + CAST(NULL AS VARCHAR2(60 CHAR)) AS INSTRMNT_ID, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +from {{ ref('m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DECC_DISSEM_SQ') }} + +UNION + +select 'NON_MARKET_ASSET' AS DATASET, + FILE_CREATION_DATE AS DATABASE_ENTRY_TIMESTAMP, + SNAPSHOT_DATE, + REPORTING_NCB AS NCB, + MFI_ID, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISIN_CODE, + OTHER_REG_NO AS OTHER_REGISTRATION_NUMBER, + NOM_AMT_SUBMITTED AS NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS AS COLL_VALUE_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS AS COLL_VALUE_AFTER_HAIRCUTS, + TYPE_OF_SYSTEM AS SYSTEM_TYPE, + TYPE_OF_OPERATION, + DOM_OR_XBORDER AS DOM_OR_CB, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_CRED_ASSESS_SOURCE, + CAST(NULL AS VARCHAR2(50 CHAR)) AS ISSUER_RATING_ENUM_VALUE, + CAST(NULL AS NUMBER(9,8)) AS ISSUER_RATING_NUMBER_VALUE, + NON_MKT_ASSET_TYPE AS NONMKT_ASSET_TYPE, + NO_AGGR_DEBTORS AS NO_OF_AGGREGATED_DEBTORS, + ELIGIBLE_VIA_GUAR AS ELIGIBLE_VIA_GUARANTOR, + DEBTOR_NAME, + DEBTOR_ID, + DEBTOR_TYPE AS DEBTOR_GROUP, + DEBTOR_CLASS, + DEBTOR_RESIDENCE, + DEBTOR_CAS AS DEBTOR_CRED_ASSESS_SOURCE, + DEBTOR_RATING_ENUM_VALUE, + DEBTOR_RATING_NUMBER_VALUE, + GUAR_NAME AS GUARANTOR_NAME, + GUAR_ID AS GUARANTOR_ID, + GUAR_RESIDENCE AS GUARANTOR_RESIDENCE, + GUAR_CRED_CAS AS GUARANTOR_CRED_ASSESS_SOURCE, + GUAR_RATING_ENUM_VALUE AS GUARANTOR_RATING_ENUM_VALUE, + GUAR_RATING_NUMBER_VALUE AS GUARANTOR_RATING_NUMBER_VALUE, + NO_AGGR_ASSETS AS NO_OF_AGGREGATED_ASSETS, + NCB_COMMENT AS NCB_COMMENTS, + MATURITY_DATE, + GUAR_CLASS AS GUARANTOR_CLASS, + CAST(NULL AS VARCHAR2(8 CHAR)) AS ISSUER_CLASS, + INTEREST_PAYMENT_TYPE, + RESIDUAL_MATURITY, + BUCKET_SIZE, + DEBTOR_CRED_PROV AS DEBTOR_CAS_PROVIDER, + GUAR_CRED_PROV AS GUARANTOR_CAS_PROVIDER, + CAST(NULL AS VARCHAR2(100 CHAR)) AS ISSUER_CAS_PROVIDER, + SECURED_FLAG AS SECURED, + DENOMINATION AS NON_MARKET_DENOMINATION, + MOBILISATION_CHANNEL, + CCB, + INVESTOR_SSS, + INTERMEDIARY_SSS, + ISSUER_SSS, + CAST(NULL AS VARCHAR2(6 CHAR)) AS TRIPARTY_AGENT, + DEBTOR_ID_TYPE AS DEBTOR_IDTYPE, + CAP, + REFERENCE_RATE, + REFERENCE_RATE_COMMENT, + RESET_PERIOD_MORE_ONE_YEAR, + OA_ID, + CONTRACT_ID, + INSTRMNT_ID, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +from {{ ref('m_MOPDB_C2D_T_UC_AGGREGATED_OU_C2D_UC_NMA_DISSEM_SQ') }} diff --git a/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_ALL_SOURCES.sql b/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_ALL_SOURCES.sql new file mode 100644 index 0000000..378de31 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_C2D_UC_ALL_SOURCES.sql @@ -0,0 +1,30 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + schema='C2D', + alias='T_COLLATERAL_ALL_SOURCES', + tags=["m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM", "m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC", "m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM", "m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM"], + tag_to_run='default_tag_lineage', + pre_hook=get_sql_t_collateral_prehook('{{ this }}', '{{ ref(var("tag_to_run", None) ~ "_SQ") }}') +) }} + +{% if has_tag('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_MA_DISSEM', var('tag_to_run', None)) %} + {{ elog("Run macro get_sql_t_collateral_ma_dissem", "INFO", 1)}} + {{ get_sql_t_collateral_ma_dissem() }} + +{% elif has_tag('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_ACC', var('tag_to_run', None)) %} + {{ elog("Run macro get_sql_t_collateral_nma_acc", "INFO", 1)}} + {{ get_sql_t_collateral_nma_acc() }} + +{% elif has_tag('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DECC_DISSEM', var('tag_to_run', None)) %} + {{ elog("Run macro get_sql_t_collateral_nma_decc_dissem", "INFO", 1)}} + {{ get_sql_t_collateral_nma_decc_dissem() }} + +{% elif has_tag('m_MOPDB_C2D_T_COLLATERAL_ALL_SOURCES_OU_C2D_UC_NMA_DISSEM', var('tag_to_run', None)) %} + {{ elog("Run macro get_sql_t_collateral_nma_dissem", "INFO", 1)}} + {{ get_sql_t_collateral_nma_dissem() }} + +{% elif has_tag('default_tag_lineage', var('tag_to_run', None)) %} + {{ elog("Run macro get_sql_t_collateral_default", "INFO", 1)}} + {{ get_sql_t_collateral_default() }} + +{% endif %} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY.sql new file mode 100644 index 0000000..9e09737 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY.sql @@ -0,0 +1,160 @@ + +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "CSDB", + alias = "DEBT_DAILY_MARS", + tags = ["m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-22", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-10-03" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE SNAPSHOT_DATE IN ( + SELECT DISTINCT WORKFLOW_START + FROM {{ ref('m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ')}} + WHERE A_WORKFLOW_HISTORY_KEY = MAX_KEY + )" + ) +}} + + +SELECT +CAST(A_KEY AS NUMBER(38,0)) as A_KEY, +CAST(NEWUPDATED AS DATE) as NEWUPDATED, +CAST(IDLOADDATE_DIM AS DATE) as IDLOADDATE_DIM, +CAST(EXTERNALCODE_ISIN AS VARCHAR2(32 CHAR)) as EXTERNALCODE_ISIN, +CAST(EXTERNALCODETYPE_NC AS VARCHAR2(124 CHAR)) as EXTERNALCODETYPE_NC, +CAST(EXTERNALCODE_NATIONAL AS VARCHAR2(32 CHAR)) as EXTERNALCODE_NATIONAL, +CAST(IDIRINSTRUMENT AS NUMBER(28,0)) as IDIRINSTRUMENT, +CAST(SHORTNAME AS VARCHAR2(255 CHAR)) as SHORTNAME, +CAST(IDIRDEPOSITORY AS VARCHAR2(1 CHAR)) as IDIRDEPOSITORY, +CAST(IDIRDEBTTYPE AS VARCHAR2(32 CHAR)) as IDIRDEBTTYPE, +CAST(IDIRASSETSECTYPE AS VARCHAR2(32 CHAR)) as IDIRASSETSECTYPE, +CAST(IDIRCLASSIFICATIONCODE_CFI AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_CFI, +CAST(IDIRCLASSIFICATIONCODE_ESAI AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI, +CAST(IDIRCLASSIFICATIONCODE_ESAI_DM AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI_DM, +CAST(IDIRCURRENCY_NOMINAL AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_NOMINAL, +CAST(AMOUNTISSUED AS NUMBER(28,10)) as AMOUNTISSUED, +CAST(AMOUNTOUTSTANDING AS NUMBER(28,10)) as AMOUNTOUTSTANDING, +CAST(AMOUNTOUTSTANDING_EUR AS NUMBER(28,10)) as AMOUNTOUTSTANDING_EUR, +CAST(POOLFACTOR AS NUMBER(28,10)) as POOLFACTOR, +CAST(ISSUEPRICE AS NUMBER(28,10)) as ISSUEPRICE, +CAST(IDISSUEDATE AS DATE) as IDISSUEDATE, +CAST(IDIRCOUPONTYPE AS VARCHAR2(32 CHAR)) as IDIRCOUPONTYPE, +CAST(IDIRCOUPONFREQUENCY AS VARCHAR2(32 CHAR)) as IDIRCOUPONFREQUENCY, +CAST(IDIRCURRENCY_COUPON AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_COUPON, +CAST(COUPONRATE AS NUMBER(28,10)) as COUPONRATE, +CAST(COUPONDATE AS DATE) as COUPONDATE, +CAST(IDIRREDEMPTIONTYPE AS VARCHAR2(32 CHAR)) as IDIRREDEMPTIONTYPE, +CAST(IDIRREDEMPTIONFREQUENCY AS VARCHAR2(32 CHAR)) as IDIRREDEMPTIONFREQUENCY, +CAST(IDIRCURRENCY_REDEMPTION AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_REDEMPTION, +CAST(REDEMPTIONPRICE AS NUMBER(28,10)) as REDEMPTIONPRICE, +CAST(IDMATURITYDATE AS DATE) as IDMATURITYDATE, +CAST(IDIRORGANISATIONALIASTYPE_IS AS VARCHAR2(32 CHAR)) as IDIRORGANISATIONALIASTYPE_IS, +CAST(ISSUERSOURCECODE AS VARCHAR2(32 CHAR)) as ISSUERSOURCECODE, +CAST(ISSUEREXTERNALCODE_MFI AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_MFI, +CAST(ISSUEREXTERNALCODE_BIC AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_BIC, +CAST(ISSUEREXTERNALCODE_BEI AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_BEI, +CAST(IDIRORGANISATION_ISSUER AS NUMBER(28,0)) as IDIRORGANISATION_ISSUER, +CAST(ISSUERNAME AS VARCHAR2(255 CHAR)) as ISSUERNAME, +CAST(IDIRCOUNTRY AS VARCHAR2(32 CHAR)) as IDIRCOUNTRY, +CAST(IDIRCOUNTRY_DM AS VARCHAR2(32 CHAR)) as IDIRCOUNTRY_DM, +CAST(IDIRCLASSIFICATIONCODE_ESAO AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO, +CAST(IDIRCLASSIFICATIONCODE_ESAO_DM AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO_DM, +CAST(IDIRCLASSIFICATIONCODE_NACE AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_NACE, +CAST(PUBLICATIONPRICEDATE AS DATE) as PUBLICATIONPRICEDATE, +CAST(PUBLICATIONPRICE AS NUMBER(28,10)) as PUBLICATIONPRICE, +CAST(PUBLICATIONPRICETYPE AS VARCHAR2(32 CHAR)) as PUBLICATIONPRICETYPE, +CAST(PUBLICATIONPRICEQUOTATIONBASIS AS VARCHAR2(32 CHAR)) as PUBLICATIONPRICEQUOTATIONBASIS, +CAST(MONTHLYAVERAGEPRICE AS NUMBER(28,10)) as MONTHLYAVERAGEPRICE, +CAST(ACCRUALSTARTDATE AS DATE) as ACCRUALSTARTDATE, +CAST(DEBTACCRUALDEBTOR AS NUMBER(28,10)) as DEBTACCRUALDEBTOR, +CAST(DEBTACCRUALDEBTOR_DM AS VARCHAR2(32 CHAR)) as DEBTACCRUALDEBTOR_DM, +CAST(DEBTACCRUALCREDITOR AS NUMBER(28,10)) as DEBTACCRUALCREDITOR, +CAST(DEBTACCRUALCREDITOR_TYP AS VARCHAR2(32 CHAR)) as DEBTACCRUALCREDITOR_TYP, +CAST(ACCRUEDINTEREST AS NUMBER(28,10)) as ACCRUEDINTEREST, +CAST(YTMNONOPTIONADJUSTED AS NUMBER(28,10)) as YTMNONOPTIONADJUSTED, +CAST(ESCB_ISSUER_IDENT AS VARCHAR2(32 CHAR)) as ESCB_ISSUER_IDENT, +CAST(ESCB_ISSUER_IDENT_TYP AS VARCHAR2(32 CHAR)) as ESCB_ISSUER_IDENT_TYP, +CAST(IDUDCMPPARTY AS NUMBER(28,0)) as IDUDCMPPARTY, +CAST(AMOUNTOUTSTANDINGTYPE AS VARCHAR2(32 CHAR)) as AMOUNTOUTSTANDINGTYPE, +CAST(MARKETCAPITALISATION AS NUMBER(28,10)) as MARKETCAPITALISATION, +CAST(MARKETCAPITALISATION_EUR AS NUMBER(28,10)) as MARKETCAPITALISATION_EUR, +CAST(VA_SECURITYSTATUS AS VARCHAR2(32 CHAR)) as VA_SECURITYSTATUS, +CAST(VA_INSTRSUPPLEMENTARYCLASS AS VARCHAR2(32 CHAR)) as VA_INSTRSUPPLEMENTARYCLASS, +CAST(VA_RESIDUALMATURITYCLASS AS VARCHAR2(32 CHAR)) as VA_RESIDUALMATURITYCLASS, +CAST(VA_ISINSEC AS VARCHAR2(32 CHAR)) as VA_ISINSEC, +CAST(VA_ISELIGIBLEFOREADB AS VARCHAR2(32 CHAR)) as VA_ISELIGIBLEFOREADB, +CAST(IDIRCLASSIFICATIONCODE_ESAI10 AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI10, +CAST(IDIRCLASSIFICATIONCODE_ESAO10 AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO10, +CAST(IDIRDEBTTYPE_N AS VARCHAR2(32 CHAR)) as IDIRDEBTTYPE_N, +CAST(SENIORITY AS VARCHAR2(32 CHAR)) as SENIORITY, +CAST(ISSUEREXTERNALCODE_LEI AS VARCHAR2(4000 CHAR)) as ISSUEREXTERNALCODE_LEI, +CAST(A_WORKFLOW_HISTORY_KEY AS NUMBER(38,0)) as A_WORKFLOW_HISTORY_SOURCE_KEY, +CAST(INSTR_ESA2010_CLASS_VALUETYPE AS VARCHAR2(4000 CHAR)) as INSTR_ESA2010_CLASS_VALUETYPE, +CAST(ISS_ESA2010_CLASS_VALUETYPE AS VARCHAR2(4000 CHAR)) as ISS_ESA2010_CLASS_VALUETYPE, +CAST(SEC_STATUS_DATE AS DATE) as SEC_STATUS_DATE, +CAST(GROUP_TYPE AS VARCHAR2(4000 CHAR)) as GROUP_TYPE, +CAST(HAS_EMBEDDED_OPTION AS VARCHAR2(4000 CHAR)) as HAS_EMBEDDED_OPTION, +CAST(VOLUME_TRADED AS VARCHAR2(4000 CHAR)) as VOLUME_TRADED, +CAST(PRIMARY_LISTING_NAME AS VARCHAR2(4000 CHAR)) as PRIMARY_LISTING_NAME, +CAST(PRIM_LISTING_RESIDENCY_COUNTRY AS VARCHAR2(4000 CHAR)) as PRIM_LISTING_RESIDENCY_COUNTRY, +CAST(INSTR_PORTFOLIO_FLAGS AS VARCHAR2(4000 CHAR)) as INSTR_PORTFOLIO_FLAGS, +CAST(BOND_DURATION AS NUMBER(28,10)) as BOND_DURATION, +CAST(RESIDUAL_MATURITY AS VARCHAR2(4000 CHAR)) as RESIDUAL_MATURITY, +CAST(ORIGINAL_MATURITY AS VARCHAR2(4000 CHAR)) as ORIGINAL_MATURITY, +CAST(CFIN_CLASSIFICATION AS VARCHAR2(4000 CHAR)) as CFIN_CLASSIFICATION, +CAST(COUPONFIRSTPAYMENTDATE AS VARCHAR2(4000 CHAR)) as COUPONFIRSTPAYMENTDATE, +CAST(COUPONLASTPAYMENTDATE AS VARCHAR2(4000 CHAR)) as COUPONLASTPAYMENTDATE, +CAST(COUPONRATEUNDERLYINGCODE_ISIN AS VARCHAR2(4000 CHAR)) as COUPONRATEUNDERLYINGCODE_ISIN, +CAST(COUPONRATESPREAD AS VARCHAR2(4000 CHAR)) as COUPONRATESPREAD, +CAST(COUPONRATEMULTIPLIER AS VARCHAR2(4000 CHAR)) as COUPONRATEMULTIPLIER, +CAST(COUPONRATECAP AS VARCHAR2(4000 CHAR)) as COUPONRATECAP, +CAST(COUPONRATEFLOOR AS VARCHAR2(4000 CHAR)) as COUPONRATEFLOOR, +CAST(IDISSUEDATE_TRANCHE AS VARCHAR2(4000 CHAR)) as IDISSUEDATE_TRANCHE, +CAST(ISSUEPRICE_TRANCHE AS VARCHAR2(4000 CHAR)) as ISSUEPRICE_TRANCHE, +CAST(VA_ISPRIVATEPLACEMENT AS VARCHAR2(4000 CHAR)) as VA_ISPRIVATEPLACEMENT, +CAST(RIAD_CODE AS VARCHAR2(4000 CHAR)) as RIAD_CODE, +CAST(RIAD_OUID AS NUMBER(38,0)) as RIAD_OUID, +CAST(ESG1 AS VARCHAR2(4000 CHAR)) as ESG1, +CAST(ESG2 AS VARCHAR2(4000 CHAR)) as ESG2, +CAST(ESG3 AS VARCHAR2(4000 CHAR)) as ESG3, +CAST(STRIP AS VARCHAR2(1 CHAR)) as STRIP, +CAST(DEPOSITORY_RECEIPT AS VARCHAR2(1 CHAR)) as DEPOSITORY_RECEIPT, +CAST(RULE_144A AS VARCHAR2(1 CHAR)) as RULE_144A, +CAST(REG_S AS VARCHAR2(1 CHAR)) as REG_S, +CAST(WARRANT AS VARCHAR2(1 CHAR)) as WARRANT, +CAST(CSEC_RELEVANCE_STOCK AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_STOCK, +CAST(CSEC_RELEVANCE_GROSS_ISSUANCE AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_GROSS_ISSUANCE, +CAST(CSEC_RELEVANCE_REDEMPTION AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_REDEMPTION, +CAST(ACCRUING_COUPON AS NUMBER(28,10)) as ACCRUING_COUPON, +CAST(ACCRUING_DISCOUNT AS NUMBER(28,10)) as ACCRUING_DISCOUNT, +CAST(PLACEHOLDER32 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER32, +CAST(PLACEHOLDER33 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER33, +CAST(PLACEHOLDER34 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER34, +CAST(PLACEHOLDER35 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER35, +CAST(PLACEHOLDER36 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER36, +CAST(PLACEHOLDER37 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER37, +CAST(PLACEHOLDER38 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER38, +CAST(PLACEHOLDER39 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER39, +CAST(PLACEHOLDER40 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER40, +CAST(PLACEHOLDER41 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER41, +CAST(PLACEHOLDER42 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER42, +CAST(PLACEHOLDER43 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER43, +CAST(PLACEHOLDER44 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER44, +CAST(PLACEHOLDER45 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER45, +CAST(PLACEHOLDER46 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER46, +CAST(PLACEHOLDER47 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER47, +CAST(PLACEHOLDER48 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER48, +CAST(PLACEHOLDER49 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER49, +CAST(PLACEHOLDER50 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER50, +CAST(WORKFLOW_START AS DATE) as SNAPSHOT_DATE, +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_CSDB_DEBT_DAILY_OU_CSDB_DEBT_DAILY_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT.sql new file mode 100644 index 0000000..8adb6d6 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT.sql @@ -0,0 +1,140 @@ +{{ config(materialized="incremental", + incremental_strategy="append", + schema="CSDB", + alias="DEBT_MARS", + tags=["m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT"] +) }} + +SELECT + /*+ PARALLEL(DEBT_MARS,4) */ +CAST(A_KEY AS NUMBER(38,0) ) as A_KEY, +CAST(NEWUPDATED AS DATE) as NEWUPDATED, +CAST(IDLoadDATE_DIM AS DATE) as IDLOADDATE_DIM, +CAST(EXTERNALCODE_ISIN AS VARCHAR2(32 CHAR)) as EXTERNALCODE_ISIN, +CAST(EXTERNALCODETYPE_NC AS VARCHAR2(124 CHAR)) as EXTERNALCODETYPE_NC, +CAST(EXTERNALCODE_NATIONAL AS VARCHAR2(32 CHAR)) as EXTERNALCODE_NATIONAL, +CAST(IDIRINSTRUMENT AS NUMBER(28,0)) as IDIRINSTRUMENT, +CAST(SHORTNAME AS VARCHAR2(255 CHAR)) as SHORTNAME, +CAST(IDIRDEPOSITORY AS VARCHAR2(1 CHAR)) as IDIRDEPOSITORY, +CAST(IDIRDEBTTYPE AS VARCHAR2(32 CHAR)) as IDIRDEBTTYPE, +CAST(IDIRASSETSECTYPE AS VARCHAR2(32 CHAR)) as IDIRASSETSECTYPE, +CAST(IDIRCLASSIFICATIONCODE_CFI AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_CFI, +CAST(IDIRCLASSIFICATIONCODE_ESAI AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI, +CAST(IDIRCLASSIFICATIONCODE_ESAI_DM AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI_DM, +CAST(IDIRCURRENCY_NOMINAL AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_NOMINAL, +CAST(AMOUNTISSUED AS NUMBER(28,10)) as AMOUNTISSUED, +CAST(AMOUNTOUTSTANDING AS NUMBER(28,10)) as AMOUNTOUTSTANDING, +CAST(AMOUNTOUTSTANDING_EUR AS NUMBER(28,10)) as AMOUNTOUTSTANDING_EUR, +CAST(POOLFACTOR AS NUMBER(28,10)) as POOLFACTOR, +CAST(ISSUEPRICE AS NUMBER(28,10)) as ISSUEPRICE, +CAST(IDISSUEDATE AS DATE) as IDISSUEDATE, +CAST(IDIRCOUPONTYPE AS VARCHAR2(32 CHAR)) as IDIRCOUPONTYPE, +CAST(IDIRCOUPONFREQUENCY AS VARCHAR2(32 CHAR)) as IDIRCOUPONFREQUENCY, +CAST(IDIRCURRENCY_COUPON AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_COUPON, +CAST(COUPONRATE AS NUMBER(28,10)) as COUPONRATE, +CAST(COUPONDATE AS DATE) as COUPONDATE, +CAST(IDIRREDEMPTIONTYPE AS VARCHAR2(32 CHAR)) as IDIRREDEMPTIONTYPE, +CAST(IDIRREDEMPTIONFREQUENCY AS VARCHAR2(32 CHAR)) as IDIRREDEMPTIONFREQUENCY, +CAST(IDIRCURRENCY_REDEMPTION AS VARCHAR2(32 CHAR)) as IDIRCURRENCY_REDEMPTION, +CAST(REDEMPTIONPRICE AS NUMBER(28,10)) as REDEMPTIONPRICE, +CAST(IDMATURITYDATE AS DATE) as IDMATURITYDATE, +CAST(IDIRORGANISATIONALIASTYPE_IS AS VARCHAR2(32 CHAR)) as IDIRORGANISATIONALIASTYPE_IS, +CAST(ISSUERSOURCECODE AS VARCHAR2(32 CHAR)) as ISSUERSOURCECODE, +CAST(ISSUEREXTERNALCODE_MFI AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_MFI, +CAST(ISSUEREXTERNALCODE_BIC AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_BIC, +CAST(ISSUEREXTERNALCODE_BEI AS VARCHAR2(32 CHAR)) as ISSUEREXTERNALCODE_BEI, +CAST(IDIRORGANISATION_ISSUER AS NUMBER(28,0)) as IDIRORGANISATION_ISSUER, +CAST(ISSUERNAME AS VARCHAR2(255 CHAR)) as ISSUERNAME, +CAST(IDIRCOUNTRY AS VARCHAR2(32 CHAR)) as IDIRCOUNTRY, +CAST(IDIRCOUNTRY_DM AS VARCHAR2(32 CHAR)) as IDIRCOUNTRY_DM, +CAST(IDIRCLASSIFICATIONCODE_ESAO AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO, +CAST(IDIRCLASSIFICATIONCODE_ESAO_DM AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO_DM, +CAST(IDIRCLASSIFICATIONCODE_NACE AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_NACE, +CAST(PUBLICATIONPRICEDATE AS DATE) as PUBLICATIONPRICEDATE, +CAST(PUBLICATIONPRICE AS NUMBER(28,10)) as PUBLICATIONPRICE, +CAST(PUBLICATIONPRICETYPE AS VARCHAR2(32 CHAR)) as PUBLICATIONPRICETYPE, +CAST(PUBLICATIONPRICEQUOTATIONBASIS AS VARCHAR2(32 CHAR)) as PUBLICATIONPRICEQUOTATIONBASIS, +CAST(MONTHLYAVERAGEPRICE AS NUMBER(28,10)) as MONTHLYAVERAGEPRICE, +CAST(ACCRUALSTARTDATE AS DATE) as ACCRUALSTARTDATE, +CAST(DEBTACCRUALDEBTOR AS NUMBER(28,10)) as DEBTACCRUALDEBTOR, +CAST(DEBTACCRUALDEBTOR_DM AS VARCHAR2(32 CHAR)) as DEBTACCRUALDEBTOR_DM, +CAST(DEBTACCRUALCREDITOR AS NUMBER(28,10)) as DEBTACCRUALCREDITOR, +CAST(DEBTACCRUALCREDITOR_TYP AS VARCHAR2(32 CHAR)) as DEBTACCRUALCREDITOR_TYP, +CAST(ACCRUEDINTEREST AS NUMBER(28,10)) as ACCRUEDINTEREST, +CAST(YTMNONOPTIONADJUSTED AS NUMBER(28,10)) as YTMNONOPTIONADJUSTED, +CAST(ESCB_ISSUER_IDENT AS VARCHAR2(32 CHAR)) as ESCB_ISSUER_IDENT, +CAST(VA_ESCBCODETYPE AS VARCHAR2(32 CHAR)) as VA_ESCBCODETYPE, +CAST(IDUDCMPPARTY AS NUMBER(28,0)) as IDUDCMPPARTY, +CAST(AMOUNTOUTSTANDINGTYPE AS VARCHAR2(32 CHAR)) as AMOUNTOUTSTANDINGTYPE, +CAST(MARKETCAPITALISATION AS NUMBER(28,10)) as MARKETCAPITALISATION, +CAST(MARKETCAPITALISATION_EUR AS NUMBER(28,10)) as MARKETCAPITALISATION_EUR, +CAST(VA_SECURITYSTATUS AS VARCHAR2(32 CHAR)) as VA_SECURITYSTATUS, +CAST(VA_INSTRSUPPLEMENTARYCLASS AS VARCHAR2(32 CHAR)) as VA_INSTRSUPPLEMENTARYCLASS, +CAST(VA_RESIDUALMATURITYCLASS AS VARCHAR2(32 CHAR)) as VA_RESIDUALMATURITYCLASS, +CAST(VA_ISINSEC AS VARCHAR2(32 CHAR)) as VA_ISINSEC, +CAST(VA_ISELIGIBLEFOREADB AS VARCHAR2(32 CHAR)) as VA_ISELIGIBLEFOREADB, +CAST(IDIRCLASSIFICATIONCODE_ESAI10 AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAI10, +CAST(IDIRCLASSIFICATIONCODE_ESAO10 AS VARCHAR2(32 CHAR)) as IDIRCLASSIFICATIONCODE_ESAO10, +CAST(IDIRDEBTTYPE_N AS VARCHAR2(32 CHAR)) as IDIRDEBTTYPE_N, +CAST(SENIORITY AS VARCHAR2(32 CHAR)) as SENIORITY, +CAST(ISSUEREXTERNALCODE_LEI AS VARCHAR2(4000 CHAR)) as ISSUEREXTERNALCODE_LEI, +CAST(INSTR_ESA2010_CLASS_VALUETYPE AS VARCHAR2(4000 CHAR)) as INSTR_ESA2010_CLASS_VALUETYPE, +CAST(ISS_ESA2010_CLASS_VALUETYPE AS VARCHAR2(4000 CHAR)) as ISS_ESA2010_CLASS_VALUETYPE, +CAST(VA_SECURITYSTATUSDATE AS DATE) as VA_SECURITYSTATUSDATE, +CAST(GROUP_TYPE AS VARCHAR2(4000 CHAR)) as GROUP_TYPE, +CAST(HASEMBEDDEDOPTION AS VARCHAR2(4000 CHAR)) as HASEMBEDDEDOPTION, +CAST(VOLUMETRADED AS VARCHAR2(4000 CHAR)) as VOLUMETRADED, +CAST(PRIMARYLISTINGNAME AS VARCHAR2(4000 CHAR)) as PRIMARYLISTINGNAME, +CAST(PRIMARYLISTINGCOUNTRY AS VARCHAR2(4000 CHAR)) as PRIMARYLISTINGCOUNTRY, +CAST(VA_INSTRPORTFLAGS AS VARCHAR2(4000 CHAR)) as VA_INSTRPORTFLAGS, +CAST(VA_BONDDURATION AS NUMBER(28,10)) as VA_BONDDURATION, +CAST(RESIDUALMATURITY AS VARCHAR2(4000 CHAR)) as RESIDUALMATURITY, +CAST(ORIGINAL_MATURITY AS VARCHAR2(4000 CHAR)) as ORIGINAL_MATURITY, +CAST(IDIRCLASSIFICATIONCODE_CFIN AS VARCHAR2(4000 CHAR)) as IDIRCLASSIFICATIONCODE_CFIN, +CAST(COUPONFIRSTPAYMENTDATE AS VARCHAR2(4000 CHAR)) as COUPONFIRSTPAYMENTDATE, +CAST(COUPONLASTPAYMENTDATE AS VARCHAR2(4000 CHAR)) as COUPONLASTPAYMENTDATE, +CAST(COUPONRATEUNDERLYINGCODE_ISIN AS VARCHAR2(4000 CHAR)) as COUPONRATEUNDERLYINGCODE_ISIN, +CAST(COUPONRATESPREAD AS VARCHAR2(4000 CHAR)) as COUPONRATESPREAD, +CAST(COUPONRATEMULTIPLIER AS VARCHAR2(4000 CHAR)) as COUPONRATEMULTIPLIER, +CAST(COUPONRATECAP AS VARCHAR2(4000 CHAR)) as COUPONRATECAP, +CAST(COUPONRATEFLOOR AS VARCHAR2(4000 CHAR)) as COUPONRATEFLOOR, +CAST(IDISSUEDATE_TRANCHE AS VARCHAR2(4000 CHAR)) as IDISSUEDATE_TRANCHE, +CAST(ISSUEPRICE_TRANCHE AS VARCHAR2(4000 CHAR)) as ISSUEPRICE_TRANCHE, +CAST(VA_ISPRIVATEPLACEMENT AS VARCHAR2(4000 CHAR)) as VA_ISPRIVATEPLACEMENT, +CAST(RIAD_CODE AS VARCHAR2(4000 CHAR)) as RIAD_CODE, +CAST(RIAD_OUID AS NUMBER(38,0)) as RIAD_OUID, +CAST(ESG1 AS VARCHAR2(4000 CHAR)) as ESG1, +CAST(ESG2 AS VARCHAR2(4000 CHAR)) as ESG2, +CAST(ESG3 AS VARCHAR2(4000 CHAR)) as ESG3, +CAST(STRIP AS VARCHAR2(1 CHAR)) as STRIP, +CAST(DEPOSITORY_RECEIPT AS VARCHAR2(1 CHAR)) as DEPOSITORY_RECEIPT, +CAST(RULE_144A AS VARCHAR2(1 CHAR)) as RULE_144A, +CAST(REG_S AS VARCHAR2(1 CHAR)) as REG_S, +CAST(WARRANT AS VARCHAR2(1 CHAR)) as WARRANT, +CAST(CSEC_RELEVANCE_STOCK AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_STOCK, +CAST(CSEC_RELEVANCE_GROSS_ISSUANCE AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_GROSS_ISSUANCE, +CAST(CSEC_RELEVANCE_REDEMPTION AS VARCHAR2(1 CHAR)) as CSEC_RELEVANCE_REDEMPTION, +CAST(ACCRUING_COUPON AS NUMBER(28,10)) as ACCRUING_COUPON, +CAST(ACCRUING_DISCOUNT AS NUMBER(28,10)) as ACCRUING_DISCOUNT, +CAST(NULL as varchar2(255)) AS PLACEHOLDER32, +CAST(NULL as varchar2(255)) AS PLACEHOLDER33, +CAST(NULL as varchar2(255)) AS PLACEHOLDER34, +CAST(NULL as varchar2(255)) AS PLACEHOLDER35, +CAST(NULL as varchar2(255)) AS PLACEHOLDER36, +CAST(NULL as varchar2(255)) AS PLACEHOLDER37, +CAST(NULL as varchar2(255)) AS PLACEHOLDER38, +CAST(NULL as varchar2(255)) AS PLACEHOLDER39, +CAST(NULL as varchar2(255)) AS PLACEHOLDER40, +CAST(NULL as varchar2(255)) AS PLACEHOLDER41, +CAST(NULL as varchar2(255)) AS PLACEHOLDER42, +CAST(NULL as varchar2(255)) AS PLACEHOLDER43, +CAST(NULL as varchar2(255)) AS PLACEHOLDER44, +CAST(NULL as varchar2(255)) AS PLACEHOLDER45, +CAST(NULL as varchar2(255)) AS PLACEHOLDER46, +CAST(NULL as varchar2(255)) AS PLACEHOLDER47, +CAST(NULL as varchar2(255)) AS PLACEHOLDER48, +CAST(NULL as varchar2(255)) AS PLACEHOLDER49, +CAST(NULL as varchar2(255)) AS PLACEHOLDER50, +CAST(WORKFLOW_START as DATE) AS SNAPSHOT_DATE, +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0) ) as A_WORKFLOW_HISTORY_KEY +FROM {{ ref("m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_OLD.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_OLD.sql new file mode 100644 index 0000000..7cc13f5 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_OLD.sql @@ -0,0 +1,142 @@ +{{ config(enabled = false) }} + +{{ config(materialized="incremental", + incremental_strategy="append", + schema="CSDB", + alias="OU_CSDB_DEBT", + tags=["m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT"] +) }} + +SELECT +A_KEY, +NEWUPDATED, +IDLOADDATE_DIM, +EXTERNALCODE_ISIN, +EXTERNALCODETYPE_NC, +EXTERNALCODE_NATIONAL, +IDIRINSTRUMENT, +SHORTNAME, +IDIRDEPOSITORY, +IDIRDEBTTYPE, +IDIRASSETSECTYPE, +IDIRCLASSIFICATIONCODE_CFI, +IDIRCLASSIFICATIONCODE_ESAI, +IDIRCLASSIFICATIONCODE_ESAI_DM, +IDIRCURRENCY_NOMINAL, +AMOUNTISSUED, +AMOUNTOUTSTANDING, +AMOUNTOUTSTANDING_EUR, +POOLFACTOR, +ISSUEPRICE, +IDISSUEDATE, +IDIRCOUPONTYPE, +IDIRCOUPONFREQUENCY, +IDIRCURRENCY_COUPON, +COUPONRATE, +COUPONDATE, +IDIRREDEMPTIONTYPE, +IDIRREDEMPTIONFREQUENCY, +IDIRCURRENCY_REDEMPTION, +REDEMPTIONPRICE, +IDMATURITYDATE, +IDIRORGANISATIONALIASTYPE_IS, +ISSUERSOURCECODE, +ISSUEREXTERNALCODE_MFI, +ISSUEREXTERNALCODE_BIC, +ISSUEREXTERNALCODE_BEI, +IDIRORGANISATION_ISSUER, +ISSUERNAME, +IDIRCOUNTRY, +IDIRCOUNTRY_DM, +IDIRCLASSIFICATIONCODE_ESAO, +IDIRCLASSIFICATIONCODE_ESAO_DM, +IDIRCLASSIFICATIONCODE_NACE, +PUBLICATIONPRICEDATE, +PUBLICATIONPRICE, +PUBLICATIONPRICETYPE, +PUBLICATIONPRICEQUOTATIONBASIS, +MONTHLYAVERAGEPRICE, +ACCRUALSTARTDATE, +DEBTACCRUALDEBTOR, +DEBTACCRUALDEBTOR_DM, +DEBTACCRUALCREDITOR, +DEBTACCRUALCREDITOR_TYP, +ACCRUEDINTEREST, +YTMNONOPTIONADJUSTED, +ESCB_ISSUER_IDENT, +VA_ESCBCODETYPE, +IDUDCMPPARTY, +AMOUNTOUTSTANDINGTYPE, +MARKETCAPITALISATION, +MARKETCAPITALISATION_EUR, +VA_SECURITYSTATUS, +VA_INSTRSUPPLEMENTARYCLASS, +VA_RESIDUALMATURITYCLASS, +VA_ISINSEC, +VA_ISELIGIBLEFOREADB, +IDIRCLASSIFICATIONCODE_ESAI10, +IDIRCLASSIFICATIONCODE_ESAO10, +IDIRDEBTTYPE_N, +SENIORITY, +ISSUEREXTERNALCODE_LEI, +A_WORKFLOW_HISTORY_KEY AS a_etl_load_set_fk, +INSTR_ESA2010_CLASS_VALUETYPE, +ISS_ESA2010_CLASS_VALUETYPE, +VA_SECURITYSTATUSDATE, +GROUP_TYPE, +HASEMBEDDEDOPTION, +VOLUMETRADED, +PRIMARYLISTINGNAME, +PRIMARYLISTINGCOUNTRY, +VA_INSTRPORTFLAGS, +VA_BONDDURATION, +RESIDUALMATURITY, +ORIGINAL_MATURITY, +IDIRCLASSIFICATIONCODE_CFIN, +COUPONFIRSTPAYMENTDATE, +COUPONLASTPAYMENTDATE, +COUPONRATEUNDERLYINGCODE_ISIN, +COUPONRATESPREAD, +COUPONRATEMULTIPLIER, +COUPONRATECAP, +COUPONRATEFLOOR, +IDISSUEDATE_TRANCHE, +ISSUEPRICE_TRANCHE, +VA_ISPRIVATEPLACEMENT, +RIAD_CODE, +RIAD_OUID, +ESG1, +ESG2, +ESG3, +STRIP, +DEPOSITORY_RECEIPT, +RULE_144A, +REG_S, +WARRANT, +CSEC_RELEVANCE_STOCK, +CSEC_RELEVANCE_GROSS_ISSUANCE, +CSEC_RELEVANCE_REDEMPTION, +ACCRUING_COUPON, +ACCRUING_DISCOUNT, +CAST(NULL as varchar2(255)) AS PLACEHOLDER32, +CAST(NULL as varchar2(255)) AS PLACEHOLDER33, +CAST(NULL as varchar2(255)) AS PLACEHOLDER34, +CAST(NULL as varchar2(255)) AS PLACEHOLDER35, +CAST(NULL as varchar2(255)) AS PLACEHOLDER36, +CAST(NULL as varchar2(255)) AS PLACEHOLDER37, +CAST(NULL as varchar2(255)) AS PLACEHOLDER38, +CAST(NULL as varchar2(255)) AS PLACEHOLDER39, +CAST(NULL as varchar2(255)) AS PLACEHOLDER40, +CAST(NULL as varchar2(255)) AS PLACEHOLDER41, +CAST(NULL as varchar2(255)) AS PLACEHOLDER42, +CAST(NULL as varchar2(255)) AS PLACEHOLDER43, +CAST(NULL as varchar2(255)) AS PLACEHOLDER44, +CAST(NULL as varchar2(255)) AS PLACEHOLDER45, +CAST(NULL as varchar2(255)) AS PLACEHOLDER46, +CAST(NULL as varchar2(255)) AS PLACEHOLDER47, +CAST(NULL as varchar2(255)) AS PLACEHOLDER48, +CAST(NULL as varchar2(255)) AS PLACEHOLDER49, +CAST(NULL as varchar2(255)) AS PLACEHOLDER50, +WORKFLOW_START AS SNAPSHOT_DATE, +{{ get_workflow_history_key() }} as A_WORKFLOW_HISTORY_KEY +FROM {{ ref("m_MOPDB_CSDB_DEBT_OU_CSDB_DEBT_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL.sql new file mode 100644 index 0000000..cf96d1c --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL.sql @@ -0,0 +1,63 @@ + +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "CSDB", + alias = "INSTR_DESC_FULL_MARS", + tags = ["m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-11", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-09-29" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE SNAPSHOT_DATE IN ( + SELECT DISTINCT ODS_WORKFLOW_START + FROM ( + {{ get_ODS_wf_start_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name ~ '_SQ')) + }} + ) + WHERE A_WORKFLOW_HISTORY_KEY = MAX_ODS_WORKFLOW_KEY_DAY + )" + ) +}} + +SELECT +CAST( ODS_WORKFLOW_START as DATE ) AS SNAPSHOT_DATE, +CAST(IDIRINSTRUMENT AS DECIMAL) AS IDIRINSTRUMENT, +CAST(ISIN AS VARCHAR2(4000 CHAR)) as ISIN, +CAST(MOO_INSTR_ID AS VARCHAR2(4000 CHAR)) as MOO_INSTR_ID, +CAST(SNP_INSTR_ID AS VARCHAR2(4000 CHAR)) as SNP_INSTR_ID, +CAST(FITCH_IDENTIFIER AS VARCHAR2(4000 CHAR)) as FITCH_IDENTIFIER, +CAST(DBRS_IDENTIFIER AS VARCHAR2(4000 CHAR)) as DBRS_IDENTIFIER, +CAST(EA_STATUS AS VARCHAR2(4000 CHAR)) as EA_STATUS, +CAST(IS_TMS AS VARCHAR2(4000 CHAR)) as IS_TMS, +CAST(DBRS_COVERED_BOND_PROGRAM AS VARCHAR2(4000 CHAR)) as DBRS_COVERED_BOND_PROGRAM, +CAST(FITCH_PRG_IDENTIFIER AS VARCHAR2(4000 CHAR)) as FITCH_PRG_IDENTIFIER, +CAST(MOO_DEAL_NUMBER AS VARCHAR2(4000 CHAR)) as MOO_DEAL_NUMBER, +CAST(SNP_PROGRAM_ID AS VARCHAR2(4000 CHAR)) as SNP_PROGRAM_ID, +CAST(IDIRDEBTTYPE AS VARCHAR2(4000 CHAR)) as IDIRDEBTTYPE, +CAST(SNP_DEBT_TYPE AS VARCHAR2(4000 CHAR)) as SNP_DEBT_TYPE, +CAST(MOODY_SENIORITY AS VARCHAR2(4000 CHAR)) as MOODY_SENIORITY, +CAST(FITCH_DEBT_LEVEL_CODE AS VARCHAR2(4000 CHAR)) as FITCH_DEBT_LEVEL_CODE, +CAST(DBRS_RANK_TYPE AS VARCHAR2(4000 CHAR)) as DBRS_RANK_TYPE, +CAST(DBRS_SECURITY_TYPE AS VARCHAR2(4000 CHAR)) as DBRS_SECURITY_TYPE, +CAST(SCO_DEBT_TYPE AS VARCHAR2(4000 CHAR)) as SCO_DEBT_TYPE, +CAST(SCO_INSTR_ID AS VARCHAR2(4000 CHAR)) as SCO_INSTR_ID, +CAST(SCO_COVERED_BOND_PROGRAM AS VARCHAR2(4000 CHAR)) as SCO_COVERED_BOND_PROGRAM, +CAST(SCO_CATEGORY AS VARCHAR2(4000 CHAR)) as SCO_CATEGORY, +CAST(PLACEHOLDER15 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER15, +CAST(PLACEHOLDER16 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER16, +CAST(PLACEHOLDER17 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER17, +CAST(PLACEHOLDER18 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER18, +CAST(PLACEHOLDER19 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER19, +CAST(PLACEHOLDER20 AS VARCHAR2(4000 CHAR)) as PLACEHOLDER20, +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_CSDB_INSTR_DESC_FULL_OU_CSDB_INSTR_DESC_FULL_SQ")}} +WHERE + MAX_ODS_WORKFLOW_KEY_DAY = A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL.sql new file mode 100644 index 0000000..ec707dd --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL.sql @@ -0,0 +1,58 @@ +{{ + config( + materialized = "table_if_rows_exist", + schema = "CSDB", + alias = "INSTR_RAT_FULL_MARS", + tags = ["m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-15", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-09-29" + } + ) +}} + +SELECT + A_KEY, + RDB_INSTR_ID, + SOURCE, + RATING_SCHEME, + RATING, + RATING_DATE, + TIME_HORIZON, + CURRENCY_TYPE, + NOTES, + VALID_FROM, + VALID_UNTIL, + RDB_RATINGS_ID, + WATCHLIST, + WATCHLIST_DATE, + OUTLOOK, + OUTLOOK_DATE, + RATING_ACTION, + RATING_ACTION_DATE, + IS_PRELIMINARY, + RATING_RAW, + RATING_TYPE, + ENDORSEMENT_INDICATOR, + LAST_REVIEW_DATE, + PLACEHOLDER6, + PLACEHOLDER7, + PLACEHOLDER8, + PLACEHOLDER9, + PLACEHOLDER10, + PLACEHOLDER11, + PLACEHOLDER12, + PLACEHOLDER13, + PLACEHOLDER14, + PLACEHOLDER15, + PLACEHOLDER16, + PLACEHOLDER17, + PLACEHOLDER18, + PLACEHOLDER19, + PLACEHOLDER20, + A_WORKFLOW_HISTORY_KEY AS A_WORKFLOW_HISTORY_SOURCE_KEY, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_CSDB_INSTR_RAT_FULL_OU_CSDB_INSTR_RAT_FULL_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL.sql new file mode 100644 index 0000000..16fa8a0 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL.sql @@ -0,0 +1,63 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "CSDB", + alias = "ISSUER_DESC_FULL_MARS", + tags = ["m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-15", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-04-15" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE SNAPSHOT_DATE IN ( + SELECT DISTINCT ODS_WORKFLOW_START + FROM ( + {{ get_ODS_wf_start_max_key( + 'w_ODS_CSDB_RATINGS_FULL', + get_main_task_name(model.name ~ '_SQ')) + }} + ) + WHERE A_WORKFLOW_HISTORY_KEY = MAX_ODS_WORKFLOW_KEY_DAY + )" + ) +}} + +SELECT + ODS_WORKFLOW_START AS SNAPSHOT_DATE, + RDB_ISSUER_ID, + ISSUERNAME, + COUNTRY_DOMICILE, + IS_SOVEREIGN, + MOODY_IDENTIFIER, + SNP_ISSUER_ID, + FITCH_IDENTIFIER, + DBRS_IDENTIFIER, + LEI_ISSUER_ID, + RIAD_CODE, + RIAD_OUID, + CLASH_GROUP_STATUS, + SCO_ISSUER_ID, + PLACEHOLDER5, + PLACEHOLDER6, + PLACEHOLDER7, + PLACEHOLDER8, + PLACEHOLDER9, + PLACEHOLDER10, + PLACEHOLDER11, + PLACEHOLDER12, + PLACEHOLDER13, + PLACEHOLDER14, + PLACEHOLDER15, + PLACEHOLDER16, + PLACEHOLDER17, + PLACEHOLDER18, + PLACEHOLDER19, + PLACEHOLDER20, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_CSDB_ISSUER_DESC_FULL_OU_CSDB_ISSUER_DESC_FULL_SQ")}} +WHERE + MAX_ODS_WORKFLOW_KEY_DAY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL.sql new file mode 100644 index 0000000..00f56b6 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL.sql @@ -0,0 +1,58 @@ +{{ + config( + materialized = "table_if_rows_exist", + schema = "CSDB", + alias = "ISSUER_RAT_FULL_MARS", + tags = ["m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-17", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-09-29" + } + ) +}} + +SELECT + A_KEY, + RDB_ISSUER_ID, + SOURCE, + RATING_SCHEME, + RATING, + RATING_DATE, + TIME_HORIZON, + CURRENCY_TYPE, + NOTES, + VALID_FROM, + VALID_UNTIL, + RDB_RATINGS_ID, + OUTLOOK, + OUTLOOK_DATE, + WATCHLIST, + WATCHLIST_DATE, + RATING_ACTION, + RATING_ACTION_DATE, + IS_PRELIMINARY, + RATING_RAW, + RATING_TYPE, + ENDORSEMENT_INDICATOR, + LAST_REVIEW_DATE, + PLACEHOLDER6, + PLACEHOLDER7, + PLACEHOLDER8, + PLACEHOLDER9, + PLACEHOLDER10, + PLACEHOLDER11, + PLACEHOLDER12, + PLACEHOLDER13, + PLACEHOLDER14, + PLACEHOLDER15 + PLACEHOLDER16, + PLACEHOLDER17, + PLACEHOLDER18, + PLACEHOLDER19, + PLACEHOLDER20, + A_WORKFLOW_HISTORY_KEY AS A_WORKFLOW_HISTORY_SOURCE_KEY, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_CSDB_ISSUER_RAT_FULL_OU_CSDB_ISSUER_RAT_FULL_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT.sql new file mode 100644 index 0000000..7c241a1 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT.sql @@ -0,0 +1,62 @@ +{{ config(materialized='table_if_rows_exist', + schema='CSDB', + alias='RATINGS_RT_INSTRUMENT_RATING', + tags=["m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT"], +) }} + +SELECT +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY, +IDRT_INSTRUMENT_RATING, +IDRT_INSTRUMENT, +IDRT_POPULATION, +IDIRINSTRRATINGCLASS, +IDIRINSTRUMENT, +SOURCE, +RATING_SCHEME, +TIME_HORIZON, +CURRENCY_TYPE, +RATING_REGION_TYPE, +RATING, +RATING_RAW, +RATING_DATE, +RATING_ACTION, +RATING_ACTION_DATE, +OUTLOOK, +OUTLOOK_DATE, +WATCHLIST, +WATCHLIST_DATE, +VALIDFROM, +VALIDUNTIL, +IS_VALID, +IS_CURRENT, +EADB_REFERENCE, +IS_PRELIMINARY, +NOTES, +SUFFIX, +ENDORSEMENT_INDICATOR, +ATTR3, +ATTR4, +ATTR5, +KEYATTR1, +LAST_REVIEW_DATE, +KEYATTR3, +KEYATTR4, +KEYATTR5, +CORRECTFROM, +CORRECTUNTIL, +BATCH_FIRST_CREATED_DATE, +BATCH_LAST_UPDATED_DATE, +IS_SOLICITED, +TEC_SOURCE_SYSTEM, +TEC_DATASET, +TEC_SURROGATE_KEY, +TEC_CRC, +TEC_INGESTION_DATE, +TEC_EXECUTION_DATE, +TEC_RUN_ID, +TEC_BUSINESS_DATE +FROM +{{ ref('m_MOPDB_CSDB_RATINGS_RT_INSTRUMENT_RATING_OU_CSDB_RATINGS_RT_INSTRUMENT_SQ') }} + + + diff --git a/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING.sql b/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING.sql new file mode 100644 index 0000000..f10c214 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING.sql @@ -0,0 +1,61 @@ +{{ config(materialized='table_if_rows_exist', + schema='CSDB', + alias='RATINGS_RT_PARTY_RATING', + tags=["m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING"], +) }} + +SELECT +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY, +IDRT_PARTY_RATING, +IDRT_PARTY, +IDRT_POPULATION, +IDIRPARTYRATINGCLASS, +SOURCE, +RATING_SCHEME, +TIME_HORIZON, +CURRENCY_TYPE, +RATING_REGION_TYPE, +RATING, +RATING_RAW, +RATING_DATE, +RATING_ACTION, +RATING_ACTION_DATE, +OUTLOOK, +OUTLOOK_DATE, +WATCHLIST, +WATCHLIST_DATE, +VALIDFROM, +VALIDUNTIL, +IS_VALID, +IS_CURRENT, +EADB_REFERENCE, +IS_PRELIMINARY, +NOTES, +SUFFIX, +ENDORSEMENT_INDICATOR, +ATTR3, +ATTR4, +ATTR5, +KEYATTR1, +LAST_REVIEW_DATE, +KEYATTR3, +KEYATTR4, +KEYATTR5, +CORRECTFROM, +CORRECTUNTIL, +BATCH_FIRST_CREATED_DATE, +BATCH_LAST_UPDATED_DATE, +IS_SOLICITED, +TEC_SOURCE_SYSTEM, +TEC_DATASET, +TEC_SURROGATE_KEY, +TEC_CRC, +TEC_INGESTION_DATE, +TEC_EXECUTION_DATE, +TEC_RUN_ID, +TEC_BUSINESS_DATE +FROM +{{ ref('m_MOPDB_CSDB_RATINGS_RT_PARTY_RATING_OU_CSDB_RATINGS_RT_PARTY_RATING_SQ') }} + + + diff --git a/dbt/models/mopdb/targets/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION.sql b/dbt/models/mopdb/targets/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION.sql new file mode 100644 index 0000000..0e4c7b8 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION.sql @@ -0,0 +1,62 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "ELA", + alias = "T_ELA_INFO_MARS", + tags = ["m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION"], + meta={ + "author": "Adelina Borisova", + "created": "2025-02-23", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-02-23" + } + ) +}} +SELECT + RIAD_CODE, + MFI_NAME, + ELA_MATURITY_DATE, + ELA_VALUE_DATE, + ELA_BASE, + ELA_DENOMINATION, + ELA, + INTEREST_RATE_APPLIED, + ISIN_CODE, + NOMINAL_AMOUNT_SUBMITTED, + COLL_BEFORE_HAIRCUTS, + COLL_AFTER_HAIRCUTS, + HAIRCUT, + ELA_ASSET_GROUP, + DENOMINATION, + ASSET_TYPE, + DOM_OR_CROSS, + ABS_TYPE, + NUMBER_OF_AGGREGATED_ASSETS, + NUMBER_OF_AGGREGATED_DEBTORS, + GUARANTEE, + ISSUER_CODE, + ISSUER_NAME, + ISSUER_RESIDENCE, + ISSUER_GROUP, + RATING_OF_ASSET, + RATING_OF_THE_IS, + RATING_OF_THE_GU, + PRICE_INFORMATION, + VALUATION_METHODOLOGY, + TYPE_OF_OPERATION, + NCB_COMMENT, + SNAPSHOT_DATE, + REPORTING_NCB, + ID, + VERSION, + IS_CORRECTION, + META_INFORMATION_ID, + META_INFORMATION_TYPE, + USED_SNAPSHOT_DATE, + PRICING_DATE, + RIAD_ENTTY_ID, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_ELA_T_ELA_INFO_OU_C2D_ELA_INFO_REPLICATION_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN.sql b/dbt/models/mopdb/targets/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN.sql new file mode 100644 index 0000000..338bb6f --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN.sql @@ -0,0 +1,19 @@ + +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "FMCO_REPORTS", + alias = "POP_LIST", + tags = ["m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN"] +) }} + +SELECT +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY , +SNAPSHOT_DATE, +MFI_ID, +ISIN_CODE +FROM + {{ ref( + "m_MOPDB_FMCO_REPORTS_POP_LIST_OU_C2D_EA_RIAD_MFI_ISIN_SQ" + ) }} + diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET.sql new file mode 100644 index 0000000..cb33cf4 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET.sql @@ -0,0 +1,37 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_BALANCE_SHEET", + tags = ["m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET"], + meta={ + "author": "Adelina Borisova", + "created": "2025-03-31", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE NVL(REF_DATE,TO_DATE('31129999000000','DDMMYYYYHH24MISS')) IN ( + SELECT DISTINCT REFERENCE_DATE + FROM {{ ref('m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ') }} + WHERE MAX_KEY = A_WORKFLOW_HISTORY_KEY)" + + ) +}} + +SELECT + REFERENCE_DATE AS REF_DATE, + VERSION, + COUNTRY AS COUNTRY_ID, + ITEM_LEVEL, + POSITION, + ITEM_TYPE, + FULLY_QUALIFIED_POSITION, + NAME, + AMOUNT, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ref("m_MOPDB_LM_T_BALANCESHEET_OU_LM_BALANCESHEET_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS.sql new file mode 100644 index 0000000..02de49c --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS.sql @@ -0,0 +1,44 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_CURRENT_ACCOUNTS", + tags = ["m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-01", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE NVL(REFERENCE_DATE,TO_DATE('31129999000000','DDMMYYYYHH24MISS')) IN ( + SELECT DISTINCT REFERENCE_DATE + FROM {{ ref('m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ') }} + WHERE MAX_KEY= A_WORKFLOW_HISTORY_KEY)" + + ) +}} + +SELECT + REVISION_NUMBER, + REFERENCE_DATE, + CURRENT_ACCOUNT_BS_TOTAL, + MRR_FORECAST_TOTAL, + CURRENT_ACCOUNT_MESSAGE_TOTAL, + MRR_MESSAGE_TOTAL, + CAST(PERIODICITY AS VARCHAR2(50 CHAR)) AS PERIODICITY, + COUNTRY, + MFI_CODE, + BANK_NAME, + CURRENT_ACCOUNT, + MINIMUM_RESERVE_REQUIREMENT, + COMMENT_, + FREE_TEXT, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ ref("m_MOPDB_LM_T_CURRENT_ACCOUNTS_OU_MRR_CURRENT_ACCOUNTS_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY + + diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT.sql new file mode 100644 index 0000000..1090d85 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT.sql @@ -0,0 +1,36 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_FC_ADH_ADJUSTMENTS", + tags = ["m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-07", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE NVL(ADJUSTMENT_DATE,TO_DATE('31129999000000','DDMMYYYYHH24MISS')) + IN ( + SELECT DISTINCT ADJUSTMENT_DATE + FROM {{ ref('m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ') }} + WHERE MAX_KEY = A_WORKFLOW_HISTORY_KEY + )" + ) +}} + + SELECT + ADJUSTMENT_DATE, + VERSION, + COUNTRY, + EFFECTIVE_DATE, + LAST_DATE_NOT_FORECAST, + FORECAST_NAME, + ADJUSTMENT_AMOUNT, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ ref("m_MOPDB_LM_T_FC_ADH_ADJUSTMENTS_OU_LM_ADHOC_ADJUSTMENT_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT.sql new file mode 100644 index 0000000..15cc3d3 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT.sql @@ -0,0 +1,46 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_FC_CSM_ADJUSTMENTS", + tags = ["m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-08", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = + "DELETE FROM {{ this }} + WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT DISTINCT + YEAR, + MONTH + FROM {{ ref('m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ')}} + WHERE A_WORKFLOW_HISTORY_KEY = MAX_KEY + ) s + WHERE + {{ this }}.YEAR = s.YEAR + AND {{ this }}.MONTH = s.MONTH + )" + ) + +}} + +SELECT + YEAR, + MONTH, + VERSION, + COUNTRY, + EFFECTIVE_DATE, + LAST_DATE_NOT_FORECAST, + FORECAST_NAME, + ADJUSTMENT_AMOUNT, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ ref("m_MOPDB_LM_T_FC_CSM_ADJUSTMENTS_OU_LM_CSM_ADJUSTMENT_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT.sql new file mode 100644 index 0000000..c92a7fa --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT.sql @@ -0,0 +1,45 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_FC_QRE_ADJUSTMENTS", + tags = ["m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-04-02", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = + "DELETE FROM {{ this }} + WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT DISTINCT + YEAR, + QUARTER + FROM {{ ref('m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ')}} + WHERE A_WORKFLOW_HISTORY_KEY = MAX_KEY + ) s + WHERE + {{ this }}.YEAR = s.YEAR + AND {{ this }}.QUARTER = s.QUARTER + )" + ) +}} + +SELECT + YEAR, + QUARTER, + VERSION, + COUNTRY, + EFFECTIVE_DATE, + LAST_DATE_NOT_FORECAST, + FORECAST_NAME, + ADJUSTMENT_AMOUNT, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ ref("m_MOPDB_LM_T_FC_QRE_ADJUSTMENTS_OU_LM_QRE_ADJUSTMENT_SQ")}} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST.sql new file mode 100644 index 0000000..8c4b9d9 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_FORECAST", + tags = ["m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST"], + meta={ + "author": "Adelina Borisova", + "created": "2025-03-27", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE NVL(REF_DATE,TO_DATE('31129999000000','DDMMYYYYHH24MISS')) IN ( + SELECT DISTINCT REFERENCE_DATE + FROM {{ ref('m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ') }} + WHERE MAX_KEY = A_WORKFLOW_HISTORY_KEY)" + ) +}} + +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + REFERENCE_DATE AS REF_DATE, + REVISION, + COUNTRY AS COUNTRY_ID, + FORECAST_DATE AS FS_DATE, + FORECAST_NAME AS FS_ITEM, + FORECAST_VALUE AS FS_VALUE +FROM + {{ ref("m_MOPDB_LM_T_FORECAST_OU_LM_FORECAST_SQ") }} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY + diff --git a/dbt/models/mopdb/targets/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES.sql b/dbt/models/mopdb/targets/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES.sql new file mode 100644 index 0000000..8e4ce00 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "LM", + alias = "T_STANDING_FACILITIES", + tags = ["m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES"], + meta={ + "author": "Adelina Borisova", + "created": "2025-08-18", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-01-06 - ticket MARS-856" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE NVL(REF_DATE,TO_DATE('31129999000000','DDMMYYYYHH24MISS')) IN ( + SELECT DISTINCT REF_DATE + FROM {{ ref('m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ') }} + WHERE MAX_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT + REF_DATE, + REV_NUMBER, + MLF_BS_TOTAL, + DF_BS_TOTAL, + MLF_SF_TOTAL, + DF_SF_TOTAL, + COUNTRY, + MFI_ID, + MFI_NAME, + MARGINAL_LENDING_FACILITY, + DEPOSIT_FACILITY, + COMMENT_, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK +FROM + {{ ref("m_MOPDB_LM_T_STANDING_FACILITIES_OU_LM_STANDING_FACILITIES_SQ") }} +WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL.sql new file mode 100644 index 0000000..f8964b4 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL.sql @@ -0,0 +1,540 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "merge", + unique_key = ["RIAD_CODE","SNAPSHOT_DATE","A_MOPDB_LOAD_SET_FK"], + merge_update_columns=["IS_CURRENT"], + schema = "MPEC", + alias = "T_MPEC_FULL", + tags = ["m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-05-09", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 ticket MARS-1340" + } + ) +}} +WITH SQ AS ( + SELECT + --COUNT_KEYS, + CAST(NULL AS NUMBER(9,0)) AS ECB_ID_NUMBER, + RIAD_CODE, + MPEC_BIC AS BIC, + ELIG_DEPOSIT_FACILITY AS DEPOSIT_FACILITY, + ELIG_MARGINAL_LENDING_FACILITY AS MARGINAL_LENDING_FACILITY, + ELIG_STD_TENDER_OPERATIONS AS STANDARD_TENDER_OPERATIONS, + ELIG_ECB_DEBT_CERTIFICATE AS ECB_DEBT_CERTIFICATE, + CAST(NULL AS VARCHAR2(1)) AS FOREIGN_EXCHANGE_SWAP, + CAST(NULL AS VARCHAR2(1)) AS FINE_TUNING_REVERSE_OPERATION, + ELIG_FIX_TERM_DEPOSIT AS FIXED_TERM_DEPOSIT, + CASE + WHEN RTGS_ACCESS = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN RTGS_ACCESS = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(RTGS_ACCESS AS VARCHAR2(1 CHAR)) + END AS RTGS, + CASE + WHEN CLM_ACCESS = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN CLM_ACCESS = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + WHEN CLM_ACCESS IS NULL THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(CLM_ACCESS AS VARCHAR2(1 CHAR)) + END AS CLM, + CASE + WHEN INTRADAY_CREDIT_FACILITY = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN INTRADAY_CREDIT_FACILITY = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(INTRADAY_CREDIT_FACILITY AS VARCHAR2(1 CHAR)) + END AS INTRA_DAY_CREDIT, + PRUDENTIAL_SUPERVISION, + 'Y' AS RESERVE, + CASE + WHEN MRR_EXEMPTION_STDT < SYSDATE AND (MRR_EXEMPTION_ENDT > SYSDATE OR MRR_EXEMPTION_ENDT IS NULL) THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN MRR_EXEMPTION_REORG_STDT < SYSDATE AND (MRR_EXEMPTION_REORG_ENDT > SYSDATE OR MRR_EXEMPTION_REORG_ENDT IS NULL) THEN CAST('R' AS VARCHAR2(1 CHAR)) + ELSE CAST('N' AS VARCHAR2(1 CHAR)) + END AS EXEMPT, + CASE + WHEN MRR_TYPE = 'mrr_via_intermediary' + THEN CAST((MRR_INTERMEDIARY_HOST || MRR_INTERMEDIARY_ID) AS VARCHAR2(255 CHAR)) + ELSE CAST(NULL AS VARCHAR2(255 CHAR)) + END AS INTERMEDIARY_RESERVE, + VERIFIED_OPERATIONAL_CRITERIA, + STD_TENDER_OPE_SUSP_STDT AS STANDARD_TENDER_SUSP_START, + STD_TENDER_OPE_SUSP_ENDT AS STANDARD_TENDER_SUSP_END, + MARG_LEND_FACILITY_SUSP_STDT AS MARG_LEND_FACILITY_SUSP_START, + MARG_LEND_FACILITY_SUSP_ENDT AS MARG_LEND_FACILITY_SUSP_END, + MRR_EXEMPTION_STDT AS MRR_EXEMPT_START_DATE, + MRR_EXEMPTION_ENDT AS MRR_EXEMPT_END_DATE, + ACTION AS UPDATE_TYPE, + ECB_ENTRY_DATE AS DATE_OF_ENTRY_INTO_DATABAS, + CAST('Y' AS VARCHAR2(1 CHAR)) AS IS_CURRENT, + CAST(NULL AS NUMBER(9,0)) AS REQUEST_ID, + CAST(NULL AS NUMBER(10,0)) AS EXCHANGE_ID, + CAST(NULL AS NUMBER(9,0)) AS PARENT_ID, + CAST(NULL AS NUMBER(7,0)) AS OBJ_VERSION, + FINE_TUN_REVOPE_ABS_SUSP_STDT AS FINE_TUNING_ABSORB_SUSP_START, + FINE_TUN_REVOPE_ABS_SUSP_ENDT AS FINE_TUNING_ABSORB_SUSP_END, + FINE_TUN_REVOP_PROV_SUSP_STDT AS FINE_TUNING_PROV_SUSP_START, + FINE_TUN_REVOP_PROV_SUSP_ENDT AS FINE_TUNING_PROV_SUSP_END, + ELIG_FTRO_PROVIDING AS FINE_TUNING_PROVIDING, + ELIG_FTRO_ABSORBING AS FINE_TUNING_ABSORBING, + FX_SWAP_ABS_SUSP_STDT AS FX_SWAP_ABSORBING_SUSP_START, + FX_SWAP_ABS_SUSP_ENDT AS FX_SWAP_ABSORBING_SUSP_END, + ELIG_FX_SWAP_ABSORBING AS FX_SWAP_ABSORBING, + FX_SWAP_PROV_SUSP_STDT AS FX_SWAP_PROVIDING_SUSP_START, + FX_SWAP_PROV_SUSP_ENDT AS FX_SWAP_PROVIDING_SUSP_END, + ELIG_FX_SWAP_PROVIDING AS FX_SWAP_PROVIDING, + DEPOSIT_FACILITY_SUSP_STDT AS DEPOSIT_FACILITY_SUSP_START, + DEPOSIT_FACILITY_SUSP_ENDT AS DEPOSIT_FACILITY_SUSP_END, + ECB_DEBT_CERTIF_SUSP_STDT AS ECB_DEBT_CERT_SUSP_START, + ECB_DEBT_CERTIF_SUSP_ENDT AS ECB_DEBT_CERT_SUSP_END, + FIX_TERM_DEPOSIT_SUSP_STDT AS FIXED_TERM_DEPOSIT_SUSP_START, + FIX_TERM_DEPOSIT_SUSP_ENDT AS FIXED_TERM_DEPOSIT_SUSP_END, + DEPOSIT_FACILITY_EXCL_STDT AS DEPOSIT_FACILITY_EXCL_START, + DEPOSIT_FACILITY_EXCL_ENDT AS DEPOSIT_FACILITY_EXCL_END, + DEPOSIT_FACILITY_LIMIT_STDT AS DEPOSIT_FACILITY_LIMIT_START, + DEPOSIT_FACILITY_LIMIT_ENDT AS DEPOSIT_FACILITY_LIMIT_END, + ECB_DEBT_CERTIF_EXCL_STDT AS ECB_DEBT_CERT_EXCL_START, + ECB_DEBT_CERTIF_EXCL_ENDT AS ECB_DEBT_CERT_EXCL_END, + ECB_DEBT_CERTIF_LIMIT_STDT AS ECB_DEBT_CERT_LIMIT_START, + ECB_DEBT_CERTIF_LIMIT_ENDT AS ECB_DEBT_CERT_LIMIT_END, + FIX_TERM_DEPOSIT_EXCL_STDT AS FIXED_TERM_DEPOSIT_EXCL_START, + FIX_TERM_DEPOSIT_EXCL_ENDT AS FIXED_TERM_DEPOSIT_EXCL_END, + FIX_TERM_DEPOSIT_LIMIT_STDT AS FIXED_TERM_DEPOSIT_LIMIT_START, + FIX_TERM_DEPOSIT_LIMIT_ENDT AS FIXED_TERM_DEPOSIT_LIMIT_END, + MARG_LEND_FACILITY_EXCL_STDT AS MARG_LEND_FACILITY_EXCL_START, + MARG_LEND_FACILITY_EXCL_ENDT AS MARG_LEND_FACILITY_EXCL_END, + MARG_LEND_FACILITY_LIMIT_STDT AS MARG_LEND_FACILITY_LIMIT_START, + MARG_LEND_FACILITY_LIMIT_ENDT AS MARG_LEND_FACILITY_LIMIT_END, + FINE_TUN_REVOPE_ABS_EXCL_STDT AS FINE_TUNING_ABSORB_EXCL_START, + FINE_TUN_REVOPE_ABS_EXCL_ENDT AS FINE_TUNING_ABSORB_EXCL_END, + FINE_TUN_REVOPE_ABS_LIMIT_STDT AS FINE_TUNING_ABSORB_LIMIT_START, + FINE_TUN_REVOPE_ABS_LIMIT_ENDT AS FINE_TUNING_ABSORB_LIMIT_END, + FINE_TUN_REVOP_PROV_EXCL_STDT AS FINE_TUNING_PROV_EXCL_START, + FINE_TUN_REVOP_PROV_EXCL_ENDT AS FINE_TUNING_PROV_EXCL_END, + FINE_TUN_REVOP_PROV_LIMIT_STDT AS FINE_TUNING_PROV_LIMIT_START, + FINE_TUN_REVOP_PROV_LIMIT_ENDT AS FINE_TUNING_PROV_LIMIT_END, + FX_SWAP_ABS_EXCL_STDT AS FX_SWAP_ABSORBING_EXCL_START, + FX_SWAP_ABS_EXCL_ENDT AS FX_SWAP_ABSORBING_EXCL_END, + FX_SWAP_ABS_LIMIT_STDT AS FX_SWAP_ABSORBING_LIMIT_START, + FX_SWAP_ABS_LIMIT_ENDT AS FX_SWAP_ABSORBING_LIMIT_END, + FX_SWAP_PROV_EXCL_STDT AS FX_SWAP_PROVIDING_EXCL_START, + FX_SWAP_PROV_EXCL_ENDT AS FX_SWAP_PROVIDING_EXCL_END, + FX_SWAP_PROV_LIMIT_STDT AS FX_SWAP_PROVIDING_LIMIT_START, + FX_SWAP_PROV_LIMIT_ENDT AS FX_SWAP_PROVIDING_LIMIT_END, + STD_TENDER_OPE_EXCL_STDT AS STANDARD_TENDER_EXCL_START, + STD_TENDER_OPE_EXCL_ENDT AS STANDARD_TENDER_EXCL_END, + STD_TENDER_OPE_LIMIT_STDT AS STANDARD_TENDER_LIMIT_START, + STD_TENDER_OPE_LIMIT_ENDT AS STANDARD_TENDER_LIMIT_END, + DELETION_REASON, + NCB_COMMENT, + WORKFLOW_START AS SNAPSHOT_DATE, + CAST(NULL AS TIMESTAMP) AS CREATION_TIMESTAMP, + USD_OPERATIONS, + CAST(NULL AS NUMBER(38,0)) AS ENTTY_RIAD_ID, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK + FROM + {{ ref("m_MOPDB_MPEC_MPEC_FULL_OU_C2D_MPEC_CONTENT_FULL_SQ")}} + WHERE + MAX_KEY = A_WORKFLOW_HISTORY_KEY + {% if is_incremental() %} + AND TRUNC(WORKFLOW_START) >= ( + SELECT MAX(DISTINCT trg.SNAPSHOT_DATE) + FROM {{ this }} trg) + {% endif %} +) +{% if is_incremental() %} + , +-- Step 2: Mark current records as expired (based on IS_CURRENT ='Y') +expired_rows AS ( + SELECT + tgt.ECB_ID_NUMBER, + tgt.RIAD_CODE, + tgt.BIC, + tgt.DEPOSIT_FACILITY, + tgt.MARGINAL_LENDING_FACILITY, + tgt.STANDARD_TENDER_OPERATIONS, + tgt.ECB_DEBT_CERTIFICATE, + tgt.FOREIGN_EXCHANGE_SWAP, + tgt.FINE_TUNING_REVERSE_OPERATION, + tgt.FIXED_TERM_DEPOSIT, + tgt.RTGS, + tgt.CLM, + tgt.INTRA_DAY_CREDIT, + tgt.PRUDENTIAL_SUPERVISION, + tgt.RESERVE, + tgt.EXEMPT, + tgt.INTERMEDIARY_RESERVE, + tgt.VERIFIED_OPERATIONAL_CRITERIA, + tgt.STANDARD_TENDER_SUSP_START, + tgt.STANDARD_TENDER_SUSP_END, + tgt.MARG_LEND_FACILITY_SUSP_START, + tgt.MARG_LEND_FACILITY_SUSP_END, + tgt.MRR_EXEMPT_START_DATE, + tgt.MRR_EXEMPT_END_DATE, + tgt.UPDATE_TYPE, + tgt.DATE_OF_ENTRY_INTO_DATABAS, + CAST('N' AS VARCHAR2(1 CHAR)) AS IS_CURRENT, + tgt.REQUEST_ID, + tgt.EXCHANGE_ID, + tgt.PARENT_ID, + tgt.OBJ_VERSION, + tgt.FINE_TUNING_ABSORB_SUSP_START, + tgt.FINE_TUNING_ABSORB_SUSP_END, + tgt.FINE_TUNING_PROV_SUSP_START, + tgt.FINE_TUNING_PROV_SUSP_END, + tgt.FINE_TUNING_PROVIDING, + tgt.FINE_TUNING_ABSORBING, + tgt.FX_SWAP_ABSORBING_SUSP_START, + tgt.FX_SWAP_ABSORBING_SUSP_END, + tgt.FX_SWAP_ABSORBING, + tgt.FX_SWAP_PROVIDING_SUSP_START, + tgt.FX_SWAP_PROVIDING_SUSP_END, + tgt.FX_SWAP_PROVIDING, + tgt.DEPOSIT_FACILITY_SUSP_START, + tgt.DEPOSIT_FACILITY_SUSP_END, + tgt.ECB_DEBT_CERT_SUSP_START, + tgt.ECB_DEBT_CERT_SUSP_END, + tgt.FIXED_TERM_DEPOSIT_SUSP_START, + tgt.FIXED_TERM_DEPOSIT_SUSP_END, + tgt.DEPOSIT_FACILITY_EXCL_START, + tgt.DEPOSIT_FACILITY_EXCL_END, + tgt.DEPOSIT_FACILITY_LIMIT_START, + tgt.DEPOSIT_FACILITY_LIMIT_END, + tgt.ECB_DEBT_CERT_EXCL_START, + tgt.ECB_DEBT_CERT_EXCL_END, + tgt.ECB_DEBT_CERT_LIMIT_START, + tgt.ECB_DEBT_CERT_LIMIT_END, + tgt.FIXED_TERM_DEPOSIT_EXCL_START, + tgt.FIXED_TERM_DEPOSIT_EXCL_END, + tgt.FIXED_TERM_DEPOSIT_LIMIT_START, + tgt.FIXED_TERM_DEPOSIT_LIMIT_END, + tgt.MARG_LEND_FACILITY_EXCL_START, + tgt.MARG_LEND_FACILITY_EXCL_END, + tgt.MARG_LEND_FACILITY_LIMIT_START, + tgt.MARG_LEND_FACILITY_LIMIT_END, + tgt.FINE_TUNING_ABSORB_EXCL_START, + tgt.FINE_TUNING_ABSORB_EXCL_END, + tgt.FINE_TUNING_ABSORB_LIMIT_START, + tgt.FINE_TUNING_ABSORB_LIMIT_END, + tgt.FINE_TUNING_PROV_EXCL_START, + tgt.FINE_TUNING_PROV_EXCL_END, + tgt.FINE_TUNING_PROV_LIMIT_START, + tgt.FINE_TUNING_PROV_LIMIT_END, + tgt.FX_SWAP_ABSORBING_EXCL_START, + tgt.FX_SWAP_ABSORBING_EXCL_END, + tgt.FX_SWAP_ABSORBING_LIMIT_START, + tgt.FX_SWAP_ABSORBING_LIMIT_END, + tgt.FX_SWAP_PROVIDING_EXCL_START, + tgt.FX_SWAP_PROVIDING_EXCL_END, + tgt.FX_SWAP_PROVIDING_LIMIT_START, + tgt.FX_SWAP_PROVIDING_LIMIT_END, + tgt.STANDARD_TENDER_EXCL_START, + tgt.STANDARD_TENDER_EXCL_END, + tgt.STANDARD_TENDER_LIMIT_START, + tgt.STANDARD_TENDER_LIMIT_END, + tgt.DELETION_REASON, + tgt.NCB_COMMENT, + tgt.SNAPSHOT_DATE, + tgt.CREATION_TIMESTAMP, + tgt.USD_OPERATIONS, + tgt.ENTTY_RIAD_ID, + tgt.A_MOPDB_LOAD_SET_FK + FROM {{ this }} tgt + WHERE + tgt.IS_CURRENT ='Y' + AND EXISTS (SELECT 1 FROM SQ) +), +-- Step 3: Combine new and expired versions +final_rows AS ( + SELECT + ECB_ID_NUMBER, + RIAD_CODE, + BIC, + DEPOSIT_FACILITY, + MARGINAL_LENDING_FACILITY, + STANDARD_TENDER_OPERATIONS, + ECB_DEBT_CERTIFICATE, + FOREIGN_EXCHANGE_SWAP, + FINE_TUNING_REVERSE_OPERATION, + FIXED_TERM_DEPOSIT, + RTGS, + CLM, + INTRA_DAY_CREDIT, + PRUDENTIAL_SUPERVISION, + RESERVE, + EXEMPT, + INTERMEDIARY_RESERVE, + VERIFIED_OPERATIONAL_CRITERIA, + STANDARD_TENDER_SUSP_START, + STANDARD_TENDER_SUSP_END, + MARG_LEND_FACILITY_SUSP_START, + MARG_LEND_FACILITY_SUSP_END, + MRR_EXEMPT_START_DATE, + MRR_EXEMPT_END_DATE, + UPDATE_TYPE, + DATE_OF_ENTRY_INTO_DATABAS, + IS_CURRENT, + REQUEST_ID, + EXCHANGE_ID, + PARENT_ID, + OBJ_VERSION, + FINE_TUNING_ABSORB_SUSP_START, + FINE_TUNING_ABSORB_SUSP_END, + FINE_TUNING_PROV_SUSP_START, + FINE_TUNING_PROV_SUSP_END, + FINE_TUNING_PROVIDING, + FINE_TUNING_ABSORBING, + FX_SWAP_ABSORBING_SUSP_START, + FX_SWAP_ABSORBING_SUSP_END, + FX_SWAP_ABSORBING, + FX_SWAP_PROVIDING_SUSP_START, + FX_SWAP_PROVIDING_SUSP_END, + FX_SWAP_PROVIDING, + DEPOSIT_FACILITY_SUSP_START, + DEPOSIT_FACILITY_SUSP_END, + ECB_DEBT_CERT_SUSP_START, + ECB_DEBT_CERT_SUSP_END, + FIXED_TERM_DEPOSIT_SUSP_START, + FIXED_TERM_DEPOSIT_SUSP_END, + DEPOSIT_FACILITY_EXCL_START, + DEPOSIT_FACILITY_EXCL_END, + DEPOSIT_FACILITY_LIMIT_START, + DEPOSIT_FACILITY_LIMIT_END, + ECB_DEBT_CERT_EXCL_START, + ECB_DEBT_CERT_EXCL_END, + ECB_DEBT_CERT_LIMIT_START, + ECB_DEBT_CERT_LIMIT_END, + FIXED_TERM_DEPOSIT_EXCL_START, + FIXED_TERM_DEPOSIT_EXCL_END, + FIXED_TERM_DEPOSIT_LIMIT_START, + FIXED_TERM_DEPOSIT_LIMIT_END, + MARG_LEND_FACILITY_EXCL_START, + MARG_LEND_FACILITY_EXCL_END, + MARG_LEND_FACILITY_LIMIT_START, + MARG_LEND_FACILITY_LIMIT_END, + FINE_TUNING_ABSORB_EXCL_START, + FINE_TUNING_ABSORB_EXCL_END, + FINE_TUNING_ABSORB_LIMIT_START, + FINE_TUNING_ABSORB_LIMIT_END, + FINE_TUNING_PROV_EXCL_START, + FINE_TUNING_PROV_EXCL_END, + FINE_TUNING_PROV_LIMIT_START, + FINE_TUNING_PROV_LIMIT_END, + FX_SWAP_ABSORBING_EXCL_START, + FX_SWAP_ABSORBING_EXCL_END, + FX_SWAP_ABSORBING_LIMIT_START, + FX_SWAP_ABSORBING_LIMIT_END, + FX_SWAP_PROVIDING_EXCL_START, + FX_SWAP_PROVIDING_EXCL_END, + FX_SWAP_PROVIDING_LIMIT_START, + FX_SWAP_PROVIDING_LIMIT_END, + STANDARD_TENDER_EXCL_START, + STANDARD_TENDER_EXCL_END, + STANDARD_TENDER_LIMIT_START, + STANDARD_TENDER_LIMIT_END, + DELETION_REASON, + NCB_COMMENT, + SNAPSHOT_DATE, + CREATION_TIMESTAMP, + USD_OPERATIONS, + ENTTY_RIAD_ID, + A_MOPDB_LOAD_SET_FK + FROM SQ + UNION ALL + SELECT + ECB_ID_NUMBER, + RIAD_CODE, + BIC, + DEPOSIT_FACILITY, + MARGINAL_LENDING_FACILITY, + STANDARD_TENDER_OPERATIONS, + ECB_DEBT_CERTIFICATE, + FOREIGN_EXCHANGE_SWAP, + FINE_TUNING_REVERSE_OPERATION, + FIXED_TERM_DEPOSIT, + RTGS, + CLM, + INTRA_DAY_CREDIT, + PRUDENTIAL_SUPERVISION, + RESERVE, + EXEMPT, + INTERMEDIARY_RESERVE, + VERIFIED_OPERATIONAL_CRITERIA, + STANDARD_TENDER_SUSP_START, + STANDARD_TENDER_SUSP_END, + MARG_LEND_FACILITY_SUSP_START, + MARG_LEND_FACILITY_SUSP_END, + MRR_EXEMPT_START_DATE, + MRR_EXEMPT_END_DATE, + UPDATE_TYPE, + DATE_OF_ENTRY_INTO_DATABAS, + IS_CURRENT, + REQUEST_ID, + EXCHANGE_ID, + PARENT_ID, + OBJ_VERSION, + FINE_TUNING_ABSORB_SUSP_START, + FINE_TUNING_ABSORB_SUSP_END, + FINE_TUNING_PROV_SUSP_START, + FINE_TUNING_PROV_SUSP_END, + FINE_TUNING_PROVIDING, + FINE_TUNING_ABSORBING, + FX_SWAP_ABSORBING_SUSP_START, + FX_SWAP_ABSORBING_SUSP_END, + FX_SWAP_ABSORBING, + FX_SWAP_PROVIDING_SUSP_START, + FX_SWAP_PROVIDING_SUSP_END, + FX_SWAP_PROVIDING, + DEPOSIT_FACILITY_SUSP_START, + DEPOSIT_FACILITY_SUSP_END, + ECB_DEBT_CERT_SUSP_START, + ECB_DEBT_CERT_SUSP_END, + FIXED_TERM_DEPOSIT_SUSP_START, + FIXED_TERM_DEPOSIT_SUSP_END, + DEPOSIT_FACILITY_EXCL_START, + DEPOSIT_FACILITY_EXCL_END, + DEPOSIT_FACILITY_LIMIT_START, + DEPOSIT_FACILITY_LIMIT_END, + ECB_DEBT_CERT_EXCL_START, + ECB_DEBT_CERT_EXCL_END, + ECB_DEBT_CERT_LIMIT_START, + ECB_DEBT_CERT_LIMIT_END, + FIXED_TERM_DEPOSIT_EXCL_START, + FIXED_TERM_DEPOSIT_EXCL_END, + FIXED_TERM_DEPOSIT_LIMIT_START, + FIXED_TERM_DEPOSIT_LIMIT_END, + MARG_LEND_FACILITY_EXCL_START, + MARG_LEND_FACILITY_EXCL_END, + MARG_LEND_FACILITY_LIMIT_START, + MARG_LEND_FACILITY_LIMIT_END, + FINE_TUNING_ABSORB_EXCL_START, + FINE_TUNING_ABSORB_EXCL_END, + FINE_TUNING_ABSORB_LIMIT_START, + FINE_TUNING_ABSORB_LIMIT_END, + FINE_TUNING_PROV_EXCL_START, + FINE_TUNING_PROV_EXCL_END, + FINE_TUNING_PROV_LIMIT_START, + FINE_TUNING_PROV_LIMIT_END, + FX_SWAP_ABSORBING_EXCL_START, + FX_SWAP_ABSORBING_EXCL_END, + FX_SWAP_ABSORBING_LIMIT_START, + FX_SWAP_ABSORBING_LIMIT_END, + FX_SWAP_PROVIDING_EXCL_START, + FX_SWAP_PROVIDING_EXCL_END, + FX_SWAP_PROVIDING_LIMIT_START, + FX_SWAP_PROVIDING_LIMIT_END, + STANDARD_TENDER_EXCL_START, + STANDARD_TENDER_EXCL_END, + STANDARD_TENDER_LIMIT_START, + STANDARD_TENDER_LIMIT_END, + DELETION_REASON, + NCB_COMMENT, + SNAPSHOT_DATE, + CREATION_TIMESTAMP, + USD_OPERATIONS, + ENTTY_RIAD_ID, + A_MOPDB_LOAD_SET_FK + FROM expired_rows +) +-- Final output +SELECT + ECB_ID_NUMBER, + RIAD_CODE, + BIC, + DEPOSIT_FACILITY, + MARGINAL_LENDING_FACILITY, + STANDARD_TENDER_OPERATIONS, + ECB_DEBT_CERTIFICATE, + FOREIGN_EXCHANGE_SWAP, + FINE_TUNING_REVERSE_OPERATION, + FIXED_TERM_DEPOSIT, + RTGS, + CLM, + INTRA_DAY_CREDIT, + PRUDENTIAL_SUPERVISION, + RESERVE, + EXEMPT, + INTERMEDIARY_RESERVE, + VERIFIED_OPERATIONAL_CRITERIA, + STANDARD_TENDER_SUSP_START, + STANDARD_TENDER_SUSP_END, + MARG_LEND_FACILITY_SUSP_START, + MARG_LEND_FACILITY_SUSP_END, + MRR_EXEMPT_START_DATE, + MRR_EXEMPT_END_DATE, + UPDATE_TYPE, + DATE_OF_ENTRY_INTO_DATABAS, + IS_CURRENT, + REQUEST_ID, + EXCHANGE_ID, + PARENT_ID, + OBJ_VERSION, + FINE_TUNING_ABSORB_SUSP_START, + FINE_TUNING_ABSORB_SUSP_END, + FINE_TUNING_PROV_SUSP_START, + FINE_TUNING_PROV_SUSP_END, + FINE_TUNING_PROVIDING, + FINE_TUNING_ABSORBING, + FX_SWAP_ABSORBING_SUSP_START, + FX_SWAP_ABSORBING_SUSP_END, + FX_SWAP_ABSORBING, + FX_SWAP_PROVIDING_SUSP_START, + FX_SWAP_PROVIDING_SUSP_END, + FX_SWAP_PROVIDING, + DEPOSIT_FACILITY_SUSP_START, + DEPOSIT_FACILITY_SUSP_END, + ECB_DEBT_CERT_SUSP_START, + ECB_DEBT_CERT_SUSP_END, + FIXED_TERM_DEPOSIT_SUSP_START, + FIXED_TERM_DEPOSIT_SUSP_END, + DEPOSIT_FACILITY_EXCL_START, + DEPOSIT_FACILITY_EXCL_END, + DEPOSIT_FACILITY_LIMIT_START, + DEPOSIT_FACILITY_LIMIT_END, + ECB_DEBT_CERT_EXCL_START, + ECB_DEBT_CERT_EXCL_END, + ECB_DEBT_CERT_LIMIT_START, + ECB_DEBT_CERT_LIMIT_END, + FIXED_TERM_DEPOSIT_EXCL_START, + FIXED_TERM_DEPOSIT_EXCL_END, + FIXED_TERM_DEPOSIT_LIMIT_START, + FIXED_TERM_DEPOSIT_LIMIT_END, + MARG_LEND_FACILITY_EXCL_START, + MARG_LEND_FACILITY_EXCL_END, + MARG_LEND_FACILITY_LIMIT_START, + MARG_LEND_FACILITY_LIMIT_END, + FINE_TUNING_ABSORB_EXCL_START, + FINE_TUNING_ABSORB_EXCL_END, + FINE_TUNING_ABSORB_LIMIT_START, + FINE_TUNING_ABSORB_LIMIT_END, + FINE_TUNING_PROV_EXCL_START, + FINE_TUNING_PROV_EXCL_END, + FINE_TUNING_PROV_LIMIT_START, + FINE_TUNING_PROV_LIMIT_END, + FX_SWAP_ABSORBING_EXCL_START, + FX_SWAP_ABSORBING_EXCL_END, + FX_SWAP_ABSORBING_LIMIT_START, + FX_SWAP_ABSORBING_LIMIT_END, + FX_SWAP_PROVIDING_EXCL_START, + FX_SWAP_PROVIDING_EXCL_END, + FX_SWAP_PROVIDING_LIMIT_START, + FX_SWAP_PROVIDING_LIMIT_END, + STANDARD_TENDER_EXCL_START, + STANDARD_TENDER_EXCL_END, + STANDARD_TENDER_LIMIT_START, + STANDARD_TENDER_LIMIT_END, + DELETION_REASON, + NCB_COMMENT, + SNAPSHOT_DATE, + CREATION_TIMESTAMP, + USD_OPERATIONS, + ENTTY_RIAD_ID, + A_MOPDB_LOAD_SET_FK +FROM final_rows +{% else %} +-- below part is used only if model is executed 1st time and target table doesn't exist +-- Final output +SELECT + * +FROM SQ +{% endif %} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT.sql b/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT.sql new file mode 100644 index 0000000..7d0c19c --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT.sql @@ -0,0 +1,273 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "merge", + unique_key = ["RIAD_CODE","VALID_FROM","A_MOPDB_LOAD_SET_FK"], + merge_update_columns=["VALID_TO","IS_CURRENT"], + schema = "MPEC", + alias = "T_MPEC", + tags = ["m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-05-05", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 ticket MARS-1340" + } + ) +}} + +-- Step 1: Get new or updated source rows +WITH SQ AS( + SELECT DISTINCT + CAST(NULL AS NUMBER(9,0)) AS ECB_ID_NUMBER, + RIAD_CODE, + MPEC_BIC AS BIC, + ELIG_DEPOSIT_FACILITY AS DEPOSIT_FACILITY, + ELIG_MARGINAL_LENDING_FACILITY AS MARGINAL_LENDING_FACILITY, + ELIG_STD_TENDER_OPERATIONS AS STANDARD_TENDER_OPERATIONS, + ELIG_ECB_DEBT_CERTIFICATE AS ECB_DEBT_CERTIFICATE, + CAST(NULL AS VARCHAR2(1)) AS FOREIGN_EXCHANGE_SWAP, + CAST(NULL AS VARCHAR2(1)) AS FINE_TUNING_REVERSE_OPERATION, + ELIG_FIX_TERM_DEPOSIT AS FIXED_TERM_DEPOSIT, + CASE + WHEN RTGS_ACCESS = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN RTGS_ACCESS = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(RTGS_ACCESS AS VARCHAR2(1 CHAR)) + END AS RTGS, + CASE + WHEN CLM_ACCESS = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN CLM_ACCESS = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + WHEN CLM_ACCESS IS NULL THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(CLM_ACCESS AS VARCHAR2(1 CHAR)) + END AS CLM, + CASE + WHEN INTRADAY_CREDIT_FACILITY = '1' THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN INTRADAY_CREDIT_FACILITY = '0' THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST(INTRADAY_CREDIT_FACILITY AS VARCHAR2(1 CHAR)) + END AS INTRA_DAY_CREDIT, + PRUDENTIAL_SUPERVISION, + CAST('Y' AS VARCHAR2(1 CHAR)) AS RESERVE, + CASE + WHEN MRR_EXEMPTION_STDT < SYSDATE AND (MRR_EXEMPTION_ENDT > SYSDATE OR MRR_EXEMPTION_ENDT IS NULL) THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN MRR_EXEMPTION_REORG_STDT < SYSDATE AND (MRR_EXEMPTION_REORG_ENDT > SYSDATE OR MRR_EXEMPTION_REORG_ENDT IS NULL) THEN CAST('R' AS VARCHAR2(1 CHAR)) + ELSE CAST('N' AS VARCHAR2(1 CHAR)) + END AS EXEMPT, + CASE + WHEN MRR_TYPE = 'mrr_via_intermediary' + THEN CAST((MRR_INTERMEDIARY_HOST || MRR_INTERMEDIARY_ID) AS VARCHAR2(255 CHAR)) + ELSE CAST(NULL AS VARCHAR2(255 CHAR)) + END AS INTERMEDIARY_RESERVE, + VERIFIED_OPERATIONAL_CRITERIA, + STD_TENDER_OPE_SUSP_STDT AS STANDARD_TENDER_SUSP_START, + STD_TENDER_OPE_SUSP_ENDT AS STANDARD_TENDER_SUSP_END, + MARG_LEND_FACILITY_SUSP_STDT AS MARG_LEND_FACILITY_SUSP_START, + MARG_LEND_FACILITY_SUSP_ENDT AS MARG_LEND_FACILITY_SUSP_END, + MRR_EXEMPTION_STDT AS MRR_EXEMPT_START_DATE, + MRR_EXEMPTION_ENDT AS MRR_EXEMPT_END_DATE, + ACTION AS UPDATE_TYPE, + ECB_ENTRY_DATE AS DATE_OF_ENTRY_INTO_DATABAS, + CASE + WHEN ACTION IN ('U', 'N') AND row_num2 = max_row_num3 THEN CAST('Y' AS VARCHAR2(1 CHAR)) + WHEN ACTION = 'D' AND row_num2 = max_row_num3 THEN CAST('N' AS VARCHAR2(1 CHAR)) + ELSE CAST('N' AS VARCHAR2(1 CHAR)) + END AS IS_CURRENT, + CAST(NULL AS NUMBER(9,0)) AS REQUEST_ID, + CAST(NULL AS NUMBER(10,0)) AS EXCHANGE_ID, + CAST(NULL AS NUMBER(9,0)) AS PARENT_ID, + CAST(NULL AS NUMBER(7,0)) AS OBJ_VERSION, + FINE_TUN_REVOPE_ABS_SUSP_STDT AS FINE_TUNING_ABSORB_SUSP_START, + FINE_TUN_REVOPE_ABS_SUSP_ENDT AS FINE_TUNING_ABSORB_SUSP_END, + FINE_TUN_REVOP_PROV_SUSP_STDT AS FINE_TUNING_PROV_SUSP_START, + FINE_TUN_REVOP_PROV_SUSP_ENDT AS FINE_TUNING_PROV_SUSP_END, + ELIG_FTRO_PROVIDING AS FINE_TUNING_PROVIDING, + ELIG_FTRO_ABSORBING AS FINE_TUNING_ABSORBING, + FX_SWAP_ABS_SUSP_STDT AS FX_SWAP_ABSORBING_SUSP_START, + FX_SWAP_ABS_SUSP_ENDT AS FX_SWAP_ABSORBING_SUSP_END, + ELIG_FX_SWAP_ABSORBING AS FX_SWAP_ABSORBING, + FX_SWAP_PROV_SUSP_STDT AS FX_SWAP_PROVIDING_SUSP_START, + FX_SWAP_PROV_SUSP_ENDT AS FX_SWAP_PROVIDING_SUSP_END, + ELIG_FX_SWAP_PROVIDING AS FX_SWAP_PROVIDING, + DEPOSIT_FACILITY_SUSP_STDT AS DEPOSIT_FACILITY_SUSP_START, + DEPOSIT_FACILITY_SUSP_ENDT AS DEPOSIT_FACILITY_SUSP_END, + ECB_DEBT_CERTIF_SUSP_STDT AS ECB_DEBT_CERT_SUSP_START, + ECB_DEBT_CERTIF_SUSP_ENDT AS ECB_DEBT_CERT_SUSP_END, + FIX_TERM_DEPOSIT_SUSP_STDT AS FIXED_TERM_DEPOSIT_SUSP_START, + FIX_TERM_DEPOSIT_SUSP_ENDT AS FIXED_TERM_DEPOSIT_SUSP_END, + DEPOSIT_FACILITY_EXCL_STDT AS DEPOSIT_FACILITY_EXCL_START, + DEPOSIT_FACILITY_EXCL_ENDT AS DEPOSIT_FACILITY_EXCL_END, + DEPOSIT_FACILITY_LIMIT_STDT AS DEPOSIT_FACILITY_LIMIT_START, + DEPOSIT_FACILITY_LIMIT_ENDT AS DEPOSIT_FACILITY_LIMIT_END, + ECB_DEBT_CERTIF_EXCL_STDT AS ECB_DEBT_CERT_EXCL_START, + ECB_DEBT_CERTIF_EXCL_ENDT AS ECB_DEBT_CERT_EXCL_END, + ECB_DEBT_CERTIF_LIMIT_STDT AS ECB_DEBT_CERT_LIMIT_START, + ECB_DEBT_CERTIF_LIMIT_ENDT AS ECB_DEBT_CERT_LIMIT_END, + FIX_TERM_DEPOSIT_EXCL_STDT AS FIXED_TERM_DEPOSIT_EXCL_START, + FIX_TERM_DEPOSIT_EXCL_ENDT AS FIXED_TERM_DEPOSIT_EXCL_END, + FIX_TERM_DEPOSIT_LIMIT_STDT AS FIXED_TERM_DEPOSIT_LIMIT_START, + FIX_TERM_DEPOSIT_LIMIT_ENDT AS FIXED_TERM_DEPOSIT_LIMIT_END, + MARG_LEND_FACILITY_EXCL_STDT AS MARG_LEND_FACILITY_EXCL_START, + MARG_LEND_FACILITY_EXCL_ENDT AS MARG_LEND_FACILITY_EXCL_END, + MARG_LEND_FACILITY_LIMIT_STDT AS MARG_LEND_FACILITY_LIMIT_START, + MARG_LEND_FACILITY_LIMIT_ENDT AS MARG_LEND_FACILITY_LIMIT_END, + FINE_TUN_REVOPE_ABS_EXCL_STDT AS FINE_TUNING_ABSORB_EXCL_START, + FINE_TUN_REVOPE_ABS_EXCL_ENDT AS FINE_TUNING_ABSORB_EXCL_END, + FINE_TUN_REVOPE_ABS_LIMIT_STDT AS FINE_TUNING_ABSORB_LIMIT_START, + FINE_TUN_REVOPE_ABS_LIMIT_ENDT AS FINE_TUNING_ABSORB_LIMIT_END, + FINE_TUN_REVOP_PROV_EXCL_STDT AS FINE_TUNING_PROV_EXCL_START, + FINE_TUN_REVOP_PROV_EXCL_ENDT AS FINE_TUNING_PROV_EXCL_END, + FINE_TUN_REVOP_PROV_LIMIT_STDT AS FINE_TUNING_PROV_LIMIT_START, + FINE_TUN_REVOP_PROV_LIMIT_ENDT AS FINE_TUNING_PROV_LIMIT_END, + FX_SWAP_ABS_EXCL_STDT AS FX_SWAP_ABSORBING_EXCL_START, + FX_SWAP_ABS_EXCL_ENDT AS FX_SWAP_ABSORBING_EXCL_END, + FX_SWAP_ABS_LIMIT_STDT AS FX_SWAP_ABSORBING_LIMIT_START, + FX_SWAP_ABS_LIMIT_ENDT AS FX_SWAP_ABSORBING_LIMIT_END, + FX_SWAP_PROV_EXCL_STDT AS FX_SWAP_PROVIDING_EXCL_START, + FX_SWAP_PROV_EXCL_ENDT AS FX_SWAP_PROVIDING_EXCL_END, + FX_SWAP_PROV_LIMIT_STDT AS FX_SWAP_PROVIDING_LIMIT_START, + FX_SWAP_PROV_LIMIT_ENDT AS FX_SWAP_PROVIDING_LIMIT_END, + STD_TENDER_OPE_EXCL_STDT AS STANDARD_TENDER_EXCL_START, + STD_TENDER_OPE_EXCL_ENDT AS STANDARD_TENDER_EXCL_END, + STD_TENDER_OPE_LIMIT_STDT AS STANDARD_TENDER_LIMIT_START, + STD_TENDER_OPE_LIMIT_ENDT AS STANDARD_TENDER_LIMIT_END, + DELETION_REASON, + NCB_COMMENT, + VALID_FROM, + CASE + WHEN ACTION = 'D' AND row_num2 = max_row_num3 THEN VALID_FROM + ELSE VALID_TO + END AS VALID_TO, + CREATION_TIME AS CREATION_TIMESTAMP, + USD_OPERATIONS, + CAST(NULL AS NUMBER(38,0)) AS ENTTY_RIAD_ID, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK + FROM + {{ ref('m_MOPDB_MPEC_MPEC_OU_C2D_MPEC_CONTENT_SQ')}} + {% if is_incremental() %} + WHERE VALID_FROM > ( + SELECT MAX(trg.VALID_FROM) + FROM {{ this }} trg + WHERE RIAD_CODE = trg.RIAD_CODE) + {% endif %} +) +{% if is_incremental() %} + , +-- Step 2: Mark current records as expired (based on same RIAD_CODE) +expired_rows AS ( + SELECT + tgt.ECB_ID_NUMBER, + tgt.RIAD_CODE, + tgt.BIC, + tgt.DEPOSIT_FACILITY, + tgt.MARGINAL_LENDING_FACILITY, + tgt.STANDARD_TENDER_OPERATIONS, + tgt.ECB_DEBT_CERTIFICATE, + tgt.FOREIGN_EXCHANGE_SWAP, + tgt.FINE_TUNING_REVERSE_OPERATION, + tgt.FIXED_TERM_DEPOSIT, + tgt.RTGS, + tgt.CLM, + tgt.INTRA_DAY_CREDIT, + tgt.PRUDENTIAL_SUPERVISION, + tgt.RESERVE, + tgt.EXEMPT, + tgt.INTERMEDIARY_RESERVE, + tgt.VERIFIED_OPERATIONAL_CRITERIA, + tgt.STANDARD_TENDER_SUSP_START, + tgt.STANDARD_TENDER_SUSP_END, + tgt.MARG_LEND_FACILITY_SUSP_START, + tgt.MARG_LEND_FACILITY_SUSP_END, + tgt.MRR_EXEMPT_START_DATE, + tgt.MRR_EXEMPT_END_DATE, + tgt.UPDATE_TYPE, + tgt.DATE_OF_ENTRY_INTO_DATABAS, + CAST('N' AS VARCHAR2(1 CHAR)) AS IS_CURRENT, + tgt.REQUEST_ID, + tgt.EXCHANGE_ID, + tgt.PARENT_ID, + tgt.OBJ_VERSION, + tgt.FINE_TUNING_ABSORB_SUSP_START, + tgt.FINE_TUNING_ABSORB_SUSP_END, + tgt.FINE_TUNING_PROV_SUSP_START, + tgt.FINE_TUNING_PROV_SUSP_END, + tgt.FINE_TUNING_PROVIDING, + tgt.FINE_TUNING_ABSORBING, + tgt.FX_SWAP_ABSORBING_SUSP_START, + tgt.FX_SWAP_ABSORBING_SUSP_END, + tgt.FX_SWAP_ABSORBING, + tgt.FX_SWAP_PROVIDING_SUSP_START, + tgt.FX_SWAP_PROVIDING_SUSP_END, + tgt.FX_SWAP_PROVIDING, + tgt.DEPOSIT_FACILITY_SUSP_START, + tgt.DEPOSIT_FACILITY_SUSP_END, + tgt.ECB_DEBT_CERT_SUSP_START, + tgt.ECB_DEBT_CERT_SUSP_END, + tgt.FIXED_TERM_DEPOSIT_SUSP_START, + tgt.FIXED_TERM_DEPOSIT_SUSP_END, + tgt.DEPOSIT_FACILITY_EXCL_START, + tgt.DEPOSIT_FACILITY_EXCL_END, + tgt.DEPOSIT_FACILITY_LIMIT_START, + tgt.DEPOSIT_FACILITY_LIMIT_END, + tgt.ECB_DEBT_CERT_EXCL_START, + tgt.ECB_DEBT_CERT_EXCL_END, + tgt.ECB_DEBT_CERT_LIMIT_START, + tgt.ECB_DEBT_CERT_LIMIT_END, + tgt.FIXED_TERM_DEPOSIT_EXCL_START, + tgt.FIXED_TERM_DEPOSIT_EXCL_END, + tgt.FIXED_TERM_DEPOSIT_LIMIT_START, + tgt.FIXED_TERM_DEPOSIT_LIMIT_END, + tgt.MARG_LEND_FACILITY_EXCL_START, + tgt.MARG_LEND_FACILITY_EXCL_END, + tgt.MARG_LEND_FACILITY_LIMIT_START, + tgt.MARG_LEND_FACILITY_LIMIT_END, + tgt.FINE_TUNING_ABSORB_EXCL_START, + tgt.FINE_TUNING_ABSORB_EXCL_END, + tgt.FINE_TUNING_ABSORB_LIMIT_START, + tgt.FINE_TUNING_ABSORB_LIMIT_END, + tgt.FINE_TUNING_PROV_EXCL_START, + tgt.FINE_TUNING_PROV_EXCL_END, + tgt.FINE_TUNING_PROV_LIMIT_START, + tgt.FINE_TUNING_PROV_LIMIT_END, + tgt.FX_SWAP_ABSORBING_EXCL_START, + tgt.FX_SWAP_ABSORBING_EXCL_END, + tgt.FX_SWAP_ABSORBING_LIMIT_START, + tgt.FX_SWAP_ABSORBING_LIMIT_END, + tgt.FX_SWAP_PROVIDING_EXCL_START, + tgt.FX_SWAP_PROVIDING_EXCL_END, + tgt.FX_SWAP_PROVIDING_LIMIT_START, + tgt.FX_SWAP_PROVIDING_LIMIT_END, + tgt.STANDARD_TENDER_EXCL_START, + tgt.STANDARD_TENDER_EXCL_END, + tgt.STANDARD_TENDER_LIMIT_START, + tgt.STANDARD_TENDER_LIMIT_END, + tgt.DELETION_REASON, + tgt.NCB_COMMENT, + tgt.VALID_FROM, + SQ.VALID_FROM AS VALID_TO, + tgt.CREATION_TIMESTAMP, + tgt.USD_OPERATIONS, + tgt.ENTTY_RIAD_ID, + tgt.A_MOPDB_LOAD_SET_FK + FROM {{ this }} tgt + INNER JOIN SQ + ON tgt.RIAD_CODE = SQ.RIAD_CODE + WHERE + tgt.VALID_TO = TO_DATE('31129999', 'dd-mm-yy') AND tgt.IS_CURRENT ='Y' +), +-- Step 3: Combine new and expired versions +final_rows AS ( + SELECT * FROM SQ + UNION ALL + SELECT * FROM expired_rows +) +-- Final output +SELECT + * +FROM final_rows +{% else %} +-- below part is used only if model is executed 1st time and target table doesn't exist +-- Final output +SELECT + * +FROM SQ +{% endif %} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL.sql b/dbt/models/mopdb/targets/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL.sql new file mode 100644 index 0000000..9ae8de6 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "append", + schema = "MPEC", + alias = "T_MPEC_MID_FULL_MARS", + tags = ["m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL"], + meta={ + "author": "Adelina Borisova", + "created": "2025-05-15", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-05-15" + } + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + WORKFLOW_START AS SNAPSHOT_DATE, + RIAD_CODE, + BIC, + COUNTRY_OF_REGISTRATION, + NAME, + BOX, + ADDRESS, + POSTAL, + CITY, + CATEGORY, + HEAD_COUNTRY_OF_REGISTRATION, + HEAD_NAME, + HEAD_RIAD_CODE, + RESERVE, + EXEMPT +FROM + {{ ref("m_MOPDB_MPEC_T_MPEC_MID_FULL_OU_C2D_MPEC_MID_FULL_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK.sql new file mode 100644 index 0000000..c83506b --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_CLSLNK_PSTNZ_CURRENT", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_CLSLNK_PSTNZ_CURRENT_SQ)" +) }} + +SELECT + CLSLNK_ENTTY_RIAD_ID_1, + CLSLNK_ENTTY_RIAD_ID_2, + CLSNK_PSTN, + CLSLNK_SNPSHT_DT, + SHARE_OWNERSHIP AS SHR_OWNERSHIP, + PATH_ID, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref( + "m_MOPDB_RIAD_T_CLSLNK_PSTNZ_CURRENT_OU_RIAD_RIAD_CLOSE_LINK_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK.sql new file mode 100644 index 0000000..e81573d --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_CLSLNK_PSTNZ_HIST", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK"], + pre_hook = "DELETE FROM {{ this }} where to_date(to_char(CLSLNK_SNPSHT_DT,'YYYYMMDD'),'YYYYMMDD')=to_date(to_char(sysdate,'YYYYMMDD'),'YYYYMMDD')" +) }} + +SELECT + --iif (TRUNC(CLSLNK_SNPSHT_DT , 'DD') < TRUNC( SESSSTARTTIME , 'DD') ,SESSSTARTTIME ,CLSLNK_SNPSHT_DT ) + CASE WHEN TRUNC(CLSLNK_SNPSHT_DT) < TRUNC(SYSDATE) THEN (SYSDATE) ELSE CLSLNK_SNPSHT_DT END AS CLSLNK_SNPSHT_DT, + CLSLNK_ENTTY_RIAD_ID_1, + CLSLNK_ENTTY_RIAD_ID_2, + SHARE_OWNERSHIP AS SHR_OWNERSHIP, + --SESSSTARTTIME + SYSDATE AS LD_DATE, + PATH_ID, + --setvariable($$Var_m_mopdb_load_set_key,null) + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref( + "m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK.sql new file mode 100644 index 0000000..74a712a --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_CLSLNK_PSTNZ_HIST_ADHOCK", + tags = ["m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK"], + pre_hook = "DELETE FROM {{ this }} a where exists ( select distinct CLSLNK_SNPSHT_DT from RIAD.T_CLSLNK_PSTNZ_HIST_ADHOCK_SQ b where a.CLSLNK_SNPSHT_DT = b.CLSLNK_SNPSHT_DT )" +) }} + +SELECT + CLSLNK_SNPSHT_DT, + CLSLNK_ENTTY_RIAD_ID_1, + CLSLNK_ENTTY_RIAD_ID_2, + SHARE_OWNERSHIP AS SHR_OWNERSHIP, + SYSDATE AS LD_DATE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref( + "m_MOPDB_RIAD_T_CLSLNK_PSTNZ_HIST_OU_RIAD_RIAD_CLOSE_LINK_ADHOCK_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql new file mode 100644 index 0000000..aa320bb --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_C2D_CD_HIST", + tags = ["m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_C2D_CD_HIST_SQ)" +) }} + +SELECT +ENTTY_RIAD_ID , +ENTTY_CD, +TYP_ENTTY_CD, +ENTRY_DATE, +BSNSS_VLD_FRM, +BSNSS_VLD_T, +VRSN_VLD_FRM, +VRSN_VLD_T, +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + FROM + {{ ref( + "m_MOPDB_RIAD_T_ENTTY_C2D_CD_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql new file mode 100644 index 0000000..06f193c --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D.sql @@ -0,0 +1,60 @@ + +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_C2D_HIST", + tags = ["m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_C2D_HIST_SQ)" +) }} + +select +ENTTY_RIAD_ID , +CNTRY , +DT_BRTH , +DT_CLS , +ENTTY_RIAD_CD , +NM_ENTTY , +ECNMC_ACTVTY , +INSTTTNL_SCTR , +INSTTTNL_SCTR_DTL , +NMBR_EMPLYS , +ENTRPRS_SZ , +TYP_BNKNG_LCNS , +LGL_FRM , +PSTL_BX_CD , +STRT , +PSTL_CD , +CTY , +IS_INCTV , +IS_UCITS_CMPLNT , +INTRNTNL_ORGNSTN_CD , +BLNC_SHT_TTL_DMSTC , +BLNC_SHT_TTL , +BLNC_SHT_TTL_GRP , +NT_ASSTS_DMSTC , +COLLATERALGROUP , +INSTTTNL_SCTR_CNTRL , +CMMNT , +IS_C2D_RLVNT , +IS_RAR_RLVNT , +ARTCL_123_STTS , +CSPP_ASSSSMNT , +ENTRY_DATE , +BRANCHHEAD_ENTTY_RIAD_ID , +DIRECTHEAD_ENTTY_RIAD_ID , +ULTIMATEHEAD_ENTTY_RIAD_CD , +ULTIMATEHEAD_ENTTY_RIAD_ID , +BSNSS_VLD_FRM , +BSNSS_VLD_T , +VRSN_VLD_FRM , +VRSN_VLD_T , +FNCL_HLDNG_INVSTMNT_FRM as RIAD_FNCL_HLDNG_INVSTMNT_FRM , +IS_CNTRL_GVRNMNT_TRTD_LG as RIAD_IS_CNTRL_GVRNMNT_TRTD_LG , +IS_LCL_GVRNMNT_TRTD_PS as RIAD_IS_LCL_GVRNMNT_TRTD_PS , +IS_CNTRL_GVRNMNT_TRTD_PS as RIAD_IS_CNTRL_GVRNMNT_TRTD_PS , +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + FROM + {{ ref( + "m_MOPDB_RIAD_T_ENTTY_C2D_HIST_OU_RIAD_ENTTY_FLTTND_C2D_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql new file mode 100644 index 0000000..171d0c2 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_ECMS_CD_HIST", + tags = ["m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_ECMS_CD_HIST_SQ)" +) }} + +SELECT + ENTTY_RIAD_ID, + ENTTY_CD, + TYP_ENTTY_CD, + ENTRY_DATE, + BSNSS_VLD_FRM, + BSNSS_VLD_T, + VRSN_VLD_FRM, + VRSN_VLD_T, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + FROM + {{ ref( + "m_MOPDB_RIAD_T_ENTTY_ECMS_CD_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql new file mode 100644 index 0000000..032f492 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_ECMS_HIST", + tags = ["m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_ECMS_HIST_SQ)" +) }} + +Select +ENTTY_RIAD_ID , +CNTRY , +DT_BRTH , +DT_CLS , +ENTTY_RIAD_CD , +NM_ENTTY , +ECNMC_ACTVTY , +INSTTTNL_SCTR , +INSTTTNL_SCTR_DTL , +NMBR_EMPLYS , +ENTRPRS_SZ , +TYP_BNKNG_LCNS , +LGL_FRM , +PSTL_BX_CD , +STRT , +PSTL_CD , +CTY , +IS_INCTV , +IS_UCITS_CMPLNT , +INTRNTNL_ORGNSTN_CD , +BLNC_SHT_TTL_DMSTC , +BLNC_SHT_TTL , +0 BLNC_SHT_TTL_GRP , +NT_ASSTS_DMSTC , +COLLATERALGROUP , +INSTTTNL_SCTR_CNTRL , +CMMNT , +'Y' IS_ECMS_RLVNT , +IS_C2D_RLVNT , +IS_RAR_RLVNT , +ARTCL_123_STTS , +CSPP_ASSSSMNT , +ENTRY_DATE , +BRANCHHEAD_ENTTY_RIAD_ID , +DIRECTHEAD_ENTTY_RIAD_ID , +ULTIMATEHEAD_ENTTY_RIAD_CD , +ULTIMATEHEAD_ENTTY_RIAD_ID , +BSNSS_VLD_FRM , +BSNSS_VLD_T , +VRSN_VLD_FRM , +VRSN_VLD_T , + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY , +FNCL_HLDNG_INVSTMNT_FRM AS RIAD_FNCL_HLDNG_INVSTMNT_FRM , +IS_CNTRL_GVRNMNT_TRTD_LG AS RIAD_IS_CNTRL_GVRNMNT_TRTD_LG , +IS_LCL_GVRNMNT_TRTD_PS AS RIAD_IS_LCL_GVRNMNT_TRTD_PS , +IS_CNTRL_GVRNMNT_TRTD_PS AS RIAD_IS_CNTRL_GVRNMNT_TRTD_PS + FROM + {{ ref( + "m_MOPDB_RIAD_T_ENTTY_ECMS_HIST_OU_RIAD_ENTTY_FLTTND_ECMS_SQ" + ) }} + \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql new file mode 100644 index 0000000..14e97f1 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_FLTTND_ESSNTL_CURRENT", + tags = ["m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_FLTTND_ESSNTL_CURRENT_SQ)" +) }} + + SELECT + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY , +ENTTY_RIAD_ID, +BSNSS_VLD_FRM, +BSNSS_VLD_T, +CNTRY, +DT_BRTH, +DT_CLS, +ENTTY_RIAD_CD, +NM_ENTTY, +ECNMC_ACTVTY, +INSTTTNL_SCTR, +INSTTTNL_SCTR_DTL, +PSTL_BX_CD, +STRT, +PSTL_CD, +CTY, +IS_INCTV, +IS_UCITS_CMPLNT, +INTRNTNL_ORGNSTN_CD, +BLNC_SHT_TTL_DMSTC, +NT_ASSTS_DMSTC, +COLLATERALGROUP, +INSTTTNL_SCTR_CNTRL, +IS_C2D_RLVNT, +IS_RAR_RLVNT, +ARTCL_123_STTS, +CSPP_ASSSSMNT, +ENTRY_DATE, +RIAD, +AVID, +BIC, +LEI, +NCB, +REU, +UCDB_CD, +VAT, +TMS_CD, +BLMBRG_CD, +C2D_CD, +ULTIMATEHEAD_ENTTY_RIAD_CD, +ULTIMATEHEAD_ENTTY_RIAD_ID, +NMBR_EMPLYS, +ENTRPRS_SZ, +TYP_BNKNG_LCNS, +LGL_FRM, +BLNC_SHT_TTL, +BLNC_SHT_TTL_GRP, +CMMNT, +FNCL_HLDNG_INVSTMNT_FRM AS RIAD_FNCL_HLDNG_INVSTMNT_FRM, +IS_CNTRL_GVRNMNT_TRTD_LG AS RIAD_IS_CNTRL_GVRNMNT_TRTD_LG, +IS_LCL_GVRNMNT_TRTD_PS AS RIAD_IS_LCL_GVRNMNT_TRTD_PS, +IS_CNTRL_GVRNMNT_TRTD_PS AS RIAD_IS_CNTRL_GVRNMNT_TRTD_PS +FROM +{{ref("m_MOPDB_RIAD_T_ENTTY_FLTTND_ESSNTL_CURRENT_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ")}} + WHERE + MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS.sql b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS.sql new file mode 100644 index 0000000..a1bcc8d --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "RIAD", + alias = "T_ENTTY_RIAD_CD_ALS", + tags = ["m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS"], + pre_hook = "DELETE FROM {{ this }} WHERE EXISTS (select distinct 1 as key from RIAD.T_ENTTY_RIAD_CD_ALS_SQ)" +) }} + +SELECT + ENTTY_RIAD_ID, + ENTTY_RIAD_CD_ALS, + BSNSS_VLD_FRM, + BSNSS_VLD_T, + VRSN_VLD_FRM, + VRSN_VLD_T, + ENTRY_DATE, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY + FROM + {{ ref( + "m_MOPDB_RIAD_T_ENTTY_RIAD_CD_ALS_OU_RIAD_ENTTY_RIAD_CD_ALS_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS.sql new file mode 100644 index 0000000..819daca --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS.sql @@ -0,0 +1,94 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_1_ALL', + schema='RQSD' +) }} + +WITH source_clean AS ( +SELECT + OBLIGATIONMODULEREFERENCEDATE, + REPORTINGENTITYCOLLECTIONUNIQUEID, + RECEIVEDFILEVERSIONNUMBER, + RECEIVEDFILERECEIVEDDATE, + REVALIDATIONDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS CTP_MFI_ID, + LOWER(DATAPOINTSETDATAPOINTIDENTIFIER) AS DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS VARCHAR(200)), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBSERVATIONVALUE, + DATACOLLECTIONCODE, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULEVERSIONNUMBER, + REPORTINGENTITYNAME, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREMITTANCEDATE, + OBLIGATIONMODULEEXPECTED, + REVALIDATIONVERSIONNUMBER, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_1_ALL_ODS_RQSD_OBSERVATIONS_SQ') }}), +pivoted_data AS ( + SELECT * + FROM source_clean + PIVOT ( + MAX(OBSERVATIONVALUE) -- Aggregate function + FOR DATAPOINTSETDATAPOINTIDENTIFIER IN ( + -- List of unique datapointsetdatapointidentifier values goes here + 'ann_1_1_ref_date' AS ANN_1_1_REF_DATE, + 'ctp_name' AS CTP_NAME, + 'ctp_country' AS CTP_COUNTRY, + 'ctp_legal_status' AS CTP_LEGAL_STATUS, + 'ctp_lprt' AS CTP_LPRT, + 'ctp_measure_start_date' AS CTP_MEASURE_START_DATE, + 'ann_1_1_etwdr_mfi_id' AS ANN_1_1_ETWDR_MFI_ID, + 'ann_1_1_etwdr_lei' AS ANN_1_1_ETWDR_LEI, + 'ann_1_1_submitter_comment' AS ANN_1_1_SUBMITTER_COMMENT + ) + ) +) +SELECT CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(ANN_1_1_REF_DATE AS VARCHAR2(200)) AS ANN_1_1_REF_DATE, + CAST(CTP_MFI_ID AS VARCHAR2(200)) AS CTP_MFI_ID, + CAST(CTP_NAME AS VARCHAR2(200)) AS CTP_NAME, + CAST(CTP_COUNTRY AS VARCHAR2(200)) AS CTP_COUNTRY, + CAST(CTP_LEGAL_STATUS AS VARCHAR2(200)) AS CTP_LEGAL_STATUS, + CAST(CTP_LPRT AS VARCHAR2(200)) AS CTP_LPRT, + CAST(CTP_MEASURE_START_DATE AS VARCHAR2(200)) AS CTP_MEASURE_START_DATE, + CAST(ANN_1_1_ETWDR_MFI_ID AS VARCHAR2(200)) AS ANN_1_1_ETWDR_MFI_ID, + CAST(ANN_1_1_ETWDR_LEI AS VARCHAR2(200)) AS ANN_1_1_ETWDR_LEI, + CAST(ANN_1_1_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_1_SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM pivoted_data \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql new file mode 100644 index 0000000..24f2b29 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql @@ -0,0 +1,94 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_1_FIN_ALL', + schema='RQSD' +) }} + +WITH source_clean AS ( +SELECT + OBLIGATIONMODULEREFERENCEDATE, + REPORTINGENTITYCOLLECTIONUNIQUEID, + RECEIVEDFILEVERSIONNUMBER, + RECEIVEDFILERECEIVEDDATE, + REVALIDATIONDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS CTP_MFI_ID, + LOWER(DATAPOINTSETDATAPOINTIDENTIFIER) AS DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS VARCHAR(200)), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBSERVATIONVALUE, + DATACOLLECTIONCODE, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULEVERSIONNUMBER, + REPORTINGENTITYNAME, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREMITTANCEDATE, + OBLIGATIONMODULEEXPECTED, + REVALIDATIONVERSIONNUMBER, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ') }}), +pivoted_data AS ( + SELECT * + FROM source_clean + PIVOT ( + MAX(OBSERVATIONVALUE) -- Aggregate function + FOR DATAPOINTSETDATAPOINTIDENTIFIER IN ( + -- List of unique datapointsetdatapointidentifier values goes here + 'ann_1_1_ref_date' AS ANN_1_1_REF_DATE, + 'ctp_name' AS CTP_NAME, + 'ctp_country' AS CTP_COUNTRY, + 'ctp_legal_status' AS CTP_LEGAL_STATUS, + 'ctp_lprt' AS CTP_LPRT, + 'ctp_measure_start_date' AS CTP_MEASURE_START_DATE, + 'ann_1_1_etwdr_mfi_id' AS ANN_1_1_ETWDR_MFI_ID, + 'ann_1_1_etwdr_lei' AS ANN_1_1_ETWDR_LEI, + 'ann_1_1_submitter_comment' AS ANN_1_1_SUBMITTER_COMMENT + ) + ) +) +SELECT CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(ANN_1_1_REF_DATE AS VARCHAR2(200)) AS ANN_1_1_REF_DATE, + CAST(CTP_MFI_ID AS VARCHAR2(200)) AS CTP_MFI_ID, + CAST(CTP_NAME AS VARCHAR2(200)) AS CTP_NAME, + CAST(CTP_COUNTRY AS VARCHAR2(200)) AS CTP_COUNTRY, + CAST(CTP_LEGAL_STATUS AS VARCHAR2(200)) AS CTP_LEGAL_STATUS, + CAST(CTP_LPRT AS VARCHAR2(200)) AS CTP_LPRT, + CAST(CTP_MEASURE_START_DATE AS VARCHAR2(200)) AS CTP_MEASURE_START_DATE, + CAST(ANN_1_1_ETWDR_MFI_ID AS VARCHAR2(200)) AS ANN_1_1_ETWDR_MFI_ID, + CAST(ANN_1_1_ETWDR_LEI AS VARCHAR2(200)) AS ANN_1_1_ETWDR_LEI, + CAST(ANN_1_1_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_1_SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM pivoted_data \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS.sql new file mode 100644 index 0000000..45672cd --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS.sql @@ -0,0 +1,94 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_2_ALL', + schema='RQSD' +) }} + +WITH source_clean AS ( +SELECT + OBLIGATIONMODULEREFERENCEDATE, + REPORTINGENTITYCOLLECTIONUNIQUEID, + RECEIVEDFILEVERSIONNUMBER, + RECEIVEDFILERECEIVEDDATE, + REVALIDATIONDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS ETWDR_COMP_KEY, + LOWER(DATAPOINTSETDATAPOINTIDENTIFIER) AS DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS VARCHAR(200)), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBSERVATIONVALUE, + DATACOLLECTIONCODE, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULEVERSIONNUMBER, + REPORTINGENTITYNAME, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREMITTANCEDATE, + OBLIGATIONMODULEEXPECTED, + REVALIDATIONVERSIONNUMBER, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_2_ALL_ODS_RQSD_OBSERVATIONS_SQ') }}), +pivoted_data AS ( + SELECT * + FROM source_clean + PIVOT ( + MAX(OBSERVATIONVALUE) -- Aggregate function + FOR DATAPOINTSETDATAPOINTIDENTIFIER IN ( + -- List of unique datapointsetdatapointidentifier values goes here + 'ann_1_2_ref_date' AS ANN_1_2_REF_DATE, + 'etwdr_mfi_id' AS ETWDR_MFI_ID, + 'etwdr_lei' AS ETWDR_LEI, + 'etwdr_name' AS ETWDR_NAME, + 'etwdr_country' AS ETWDR_COUNTRY, + 'etwdr_in_eu' AS ETWDR_IN_EU, + 'etwdr_source' AS ETWDR_SOURCE, + 'etwdr_submitter' AS ETWDR_SUBMITTER, + 'ann_1_2_submitter_comment' AS ANN_1_2_SUBMITTER_COMMENT + ) + ) +) +SELECT CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(ANN_1_2_REF_DATE AS VARCHAR2(200)) AS ANN_1_2_REF_DATE, + CAST(ETWDR_COMP_KEY AS VARCHAR2(200)) AS ETWDR_COMP_KEY, + CAST(ETWDR_MFI_ID AS VARCHAR2(200)) AS ETWDR_MFI_ID, + CAST(ETWDR_LEI AS VARCHAR2(200)) AS ETWDR_LEI, + CAST(ETWDR_NAME AS VARCHAR2(200)) AS ETWDR_NAME, + CAST(ETWDR_COUNTRY AS VARCHAR2(200)) AS ETWDR_COUNTRY, + CAST(ETWDR_IN_EU AS VARCHAR2(200)) AS ETWDR_IN_EU, + CAST(ETWDR_SOURCE AS VARCHAR2(200)) AS ETWDR_SOURCE, + CAST(ETWDR_SUBMITTER AS VARCHAR2(200)) AS ETWDR_SUBMITTER, + CAST(ANN_1_2_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_2_SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM pivoted_data \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql new file mode 100644 index 0000000..0e7138d --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS.sql @@ -0,0 +1,94 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_1_2_FIN_ALL', + schema='RQSD' +) }} + +WITH source_clean AS ( +SELECT + OBLIGATIONMODULEREFERENCEDATE, + REPORTINGENTITYCOLLECTIONUNIQUEID, + RECEIVEDFILEVERSIONNUMBER, + RECEIVEDFILERECEIVEDDATE, + REVALIDATIONDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS ETWDR_COMP_KEY, + LOWER(DATAPOINTSETDATAPOINTIDENTIFIER) AS DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS VARCHAR(200)), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBSERVATIONVALUE, + DATACOLLECTIONCODE, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULEVERSIONNUMBER, + REPORTINGENTITYNAME, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREMITTANCEDATE, + OBLIGATIONMODULEEXPECTED, + REVALIDATIONVERSIONNUMBER, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_SQ') }}), +pivoted_data AS ( + SELECT * + FROM source_clean + PIVOT ( + MAX(OBSERVATIONVALUE) -- Aggregate function + FOR DATAPOINTSETDATAPOINTIDENTIFIER IN ( + -- List of unique datapointsetdatapointidentifier values goes here + 'ann_1_2_ref_date' AS ANN_1_2_REF_DATE, + 'etwdr_mfi_id' AS ETWDR_MFI_ID, + 'etwdr_lei' AS ETWDR_LEI, + 'etwdr_name' AS ETWDR_NAME, + 'etwdr_country' AS ETWDR_COUNTRY, + 'etwdr_in_eu' AS ETWDR_IN_EU, + 'etwdr_source' AS ETWDR_SOURCE, + 'etwdr_submitter' AS ETWDR_SUBMITTER, + 'ann_1_2_submitter_comment' AS ANN_1_2_SUBMITTER_COMMENT + ) + ) +) +SELECT CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(ANN_1_2_REF_DATE AS VARCHAR2(200)) AS ANN_1_2_REF_DATE, + CAST(ETWDR_COMP_KEY AS VARCHAR2(200)) AS ETWDR_COMP_KEY, + CAST(ETWDR_MFI_ID AS VARCHAR2(200)) AS ETWDR_MFI_ID, + CAST(ETWDR_LEI AS VARCHAR2(200)) AS ETWDR_LEI, + CAST(ETWDR_NAME AS VARCHAR2(200)) AS ETWDR_NAME, + CAST(ETWDR_COUNTRY AS VARCHAR2(200)) AS ETWDR_COUNTRY, + CAST(ETWDR_IN_EU AS VARCHAR2(200)) AS ETWDR_IN_EU, + CAST(ETWDR_SOURCE AS VARCHAR2(200)) AS ETWDR_SOURCE, + CAST(ETWDR_SUBMITTER AS VARCHAR2(200)) AS ETWDR_SUBMITTER, + CAST(ANN_1_2_SUBMITTER_COMMENT AS VARCHAR2(200)) AS ANN_1_2_SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM pivoted_data \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS.sql new file mode 100644 index 0000000..a58cab0 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS.sql @@ -0,0 +1,164 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='ANNEX_2_ALL', + schema='RQSD' +) }} + +WITH source_clean AS ( +SELECT + OBLIGATIONMODULEREFERENCEDATE, + REPORTINGENTITYCOLLECTIONUNIQUEID, + RECEIVEDFILEVERSIONNUMBER, + RECEIVEDFILERECEIVEDDATE, + REVALIDATIONDATE, + SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS INST_COMP_KEY, + LOWER(DATAPOINTSETDATAPOINTIDENTIFIER) AS DATAPOINTSETDATAPOINTIDENTIFIER, + NVL(NVL(CAST(OBSERVATIONVALUE AS VARCHAR(200)), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBSERVATIONVALUE, + DATACOLLECTIONCODE, + REPORTINGCYCLENAME, + REPORTINGCYCLESTATUS, + MODULECODE, + MODULEVERSIONNUMBER, + REPORTINGENTITYNAME, + ENTITYATTRIBUTECOUNTRY, + ENTITYGROUPENTITYNAME, + OBLIGATIONMODULEREMITTANCEDATE, + OBLIGATIONMODULEEXPECTED, + REVALIDATIONVERSIONNUMBER, + RECEIVEDFILESYSTEMFILENAME, + OBLIGATIONSTATUSSTATUS, + FILESTATUSSETSUBMISSIONSTATUS, + FILESTATUSSETVALIDATIONSTATUS, + NUMBEROFERRORS, + NUMBEROFWARNINGS, + DELAYINDAYS, + FAILEDATTEMPTS, + TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_SQ') }}), +pivoted_data AS ( + SELECT * + FROM source_clean + PIVOT ( + MAX(OBSERVATIONVALUE) -- Aggregate function + FOR DATAPOINTSETDATAPOINTIDENTIFIER IN ( + -- List of unique datapointsetdatapointidentifier values goes here + 'ref_date' AS REF_DATE, + 'mfi_id' AS MFI_ID, + 'legal_entity_id' AS LEGAL_ENTITY_ID, + 'inst_name' AS INST_NAME, + 'currency' AS CURRENCY, + 'reported_by_supervisor' AS REPORTED_BY_SUPERVISOR, + 'confirmed_by_supervisor' AS CONFIRMED_BY_SUPERVISOR, + 'exp_corep_cons' AS EXP_COREP_CONS, + 'exp_corep_solo' AS EXP_COREP_SOLO, + 'exp_finrep_cons' AS EXP_FINREP_CONS, + 'exp_finrep_solo' AS EXP_FINREP_SOLO, + 'exp_lev_cons' AS EXP_LEV_CONS, + 'exp_lev_solo' AS EXP_LEV_SOLO, + 'exp_lcr_cons' AS EXP_LCR_CONS, + 'exp_lcr_solo' AS EXP_LCR_SOLO, + 'exp_nsfr_cons' AS EXP_NSFR_CONS, + 'exp_nsfr_solo' AS EXP_NSFR_SOLO, + 'cons_cet1_amt' AS CONS_CET1_AMT, + 'cons_tier1_amt' AS CONS_TIER1_AMT, + 'cons_tot_cap_amt' AS CONS_TOT_CAP_AMT, + 'cons_cet1_ratio' AS CONS_CET1_RATIO, + 'çons_tier1_ratio' AS CONS_TIER1_RATIO, + 'cons_tot_cap_ratio' AS CONS_TOT_CAP_RATIO, + 'cons_risk_wght_assets' AS CONS_RISK_WGHT_ASSETS, + 'solo_cet1_amt' AS SOLO_CET1_AMT, + 'solo_tier1_amt' AS SOLO_TIER1_AMT, + 'solo_tot_cap_amt' AS SOLO_TOT_CAP_AMT, + 'solo_cet1_ratio' AS SOLO_CET1_RATIO, + 'solo_tier1_ratio' AS SOLO_TIER1_RATIO, + 'solo_tot_cap_ratio' AS SOLO_TOT_CAP_RATIO, + 'solo_risk_wght_assets' AS SOLO_RISK_WGHT_ASSETS, + 'cons_tot_assets' AS CONS_TOT_ASSETS, + 'solo_tot_assets' AS SOLO_TOT_ASSETS, + 'cons_lev_ratio_full' AS CONS_LEV_RATIO_FULL, + 'cons_lev_ratio_trans' AS CONS_LEV_RATIO_TRANS, + 'cons_lev_ratio_req' AS CONS_LEV_RATIO_REQ, + 'solo_lev_ratio_full' AS SOLO_LEV_RATIO_FULL, + 'solo_lev_ratio_trans' AS SOLO_LEV_RATIO_TRANS, + 'solo_lev_ratio_req' AS SOLO_LEV_RATIO_REQ, + 'cons_lc_ratio' AS CONS_LC_RATIO, + 'solo_lc_ratio' AS SOLO_LC_RATIO, + 'cons_nsfr_ratio' AS CONS_NSFR_RATIO, + 'solo_nsfr_ratio' AS SOLO_NSFR_RATIO, + 'submitter_comment' AS SUBMITTER_COMMENT + ) + ) +) +SELECT CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(REF_DATE AS VARCHAR2(200)) AS REF_DATE, + CAST(INST_COMP_KEY AS VARCHAR2(200)) AS INST_COMP_KEY, + CAST(MFI_ID AS VARCHAR2(200)) AS MFI_ID, + CAST(LEGAL_ENTITY_ID AS VARCHAR2(200)) AS LEGAL_ENTITY_ID, + CAST(INST_NAME AS VARCHAR2(200)) AS INST_NAME, + CAST(CURRENCY AS VARCHAR2(200)) AS CURRENCY, + CAST(REPORTED_BY_SUPERVISOR AS VARCHAR2(200)) AS REPORTED_BY_SUPERVISOR, + CAST(CONFIRMED_BY_SUPERVISOR AS VARCHAR2(200)) AS CONFIRMED_BY_SUPERVISOR, + CAST(EXP_COREP_CONS AS VARCHAR2(200)) AS EXP_COREP_CONS, + CAST(EXP_COREP_SOLO AS VARCHAR2(200)) AS EXP_COREP_SOLO, + CAST(EXP_FINREP_CONS AS VARCHAR2(200)) AS EXP_FINREP_CONS, + CAST(EXP_FINREP_SOLO AS VARCHAR2(200)) AS EXP_FINREP_SOLO, + CAST(EXP_LEV_CONS AS VARCHAR2(200)) AS EXP_LEV_CONS, + CAST(EXP_LEV_SOLO AS VARCHAR2(200)) AS EXP_LEV_SOLO, + CAST(EXP_LCR_CONS AS VARCHAR2(200)) AS EXP_LCR_CONS, + CAST(EXP_LCR_SOLO AS VARCHAR2(200)) AS EXP_LCR_SOLO, + CAST(EXP_NSFR_CONS AS VARCHAR2(200)) AS EXP_NSFR_CONS, + CAST(EXP_NSFR_SOLO AS VARCHAR2(200)) AS EXP_NSFR_SOLO, + CAST(CONS_CET1_AMT AS VARCHAR2(200)) AS CONS_CET1_AMT, + CAST(CONS_TIER1_AMT AS VARCHAR2(200)) AS CONS_TIER1_AMT, + CAST(CONS_TOT_CAP_AMT AS VARCHAR2(200)) AS CONS_TOT_CAP_AMT, + CAST(CONS_CET1_RATIO AS VARCHAR2(200)) AS CONS_CET1_RATIO, + CAST(CONS_TIER1_RATIO AS VARCHAR2(200)) AS CONS_TIER1_RATIO, + CAST(CONS_TOT_CAP_RATIO AS VARCHAR2(200)) AS CONS_TOT_CAP_RATIO, + CAST(CONS_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS CONS_RISK_WGHT_ASSETS, + CAST(SOLO_CET1_AMT AS VARCHAR2(200)) AS SOLO_CET1_AMT, + CAST(SOLO_TIER1_AMT AS VARCHAR2(200)) AS SOLO_TIER1_AMT, + CAST(SOLO_TOT_CAP_AMT AS VARCHAR2(200)) AS SOLO_TOT_CAP_AMT, + CAST(SOLO_CET1_RATIO AS VARCHAR2(200)) AS SOLO_CET1_RATIO, + CAST(SOLO_TIER1_RATIO AS VARCHAR2(200)) AS SOLO_TIER1_RATIO, + CAST(SOLO_TOT_CAP_RATIO AS VARCHAR2(200)) AS SOLO_TOT_CAP_RATIO, + CAST(SOLO_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS SOLO_RISK_WGHT_ASSETS, + CAST(CONS_TOT_ASSETS AS VARCHAR2(200)) AS CONS_TOT_ASSETS, + CAST(SOLO_TOT_ASSETS AS VARCHAR2(200)) AS SOLO_TOT_ASSETS, + CAST(CONS_LEV_RATIO_FULL AS VARCHAR2(200)) AS CONS_LEV_RATIO_FULL, + CAST(CONS_LEV_RATIO_TRANS AS VARCHAR2(200)) AS CONS_LEV_RATIO_TRANS, + CAST(CONS_LEV_RATIO_REQ AS VARCHAR2(200)) AS CONS_LEV_RATIO_REQ, + CAST(SOLO_LEV_RATIO_FULL AS VARCHAR2(200)) AS SOLO_LEV_RATIO_FULL, + CAST(SOLO_LEV_RATIO_TRANS AS VARCHAR2(200)) AS SOLO_LEV_RATIO_TRANS, + CAST(SOLO_LEV_RATIO_REQ AS VARCHAR2(200)) AS SOLO_LEV_RATIO_REQ, + CAST(CONS_LC_RATIO AS VARCHAR2(200)) AS CONS_LC_RATIO, + CAST(SOLO_LC_RATIO AS VARCHAR2(200)) AS SOLO_LC_RATIO, + CAST(CONS_NSFR_RATIO AS VARCHAR2(200)) AS CONS_NSFR_RATIO, + CAST(SOLO_NSFR_RATIO AS VARCHAR2(200)) AS SOLO_NSFR_RATIO, + CAST(SUBMITTER_COMMENT AS VARCHAR2(200)) AS SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM pivoted_data \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA.sql b/dbt/models/mopdb/targets/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA.sql new file mode 100644 index 0000000..1d27eb4 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA.sql @@ -0,0 +1,112 @@ +{{ config( + materialized='incremental', + incremental_strategy='append', + tags=["m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA","m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_MANUAL"], + alias='OUTPUT_CUR_HIST', + schema='RQSD', + pre_hook=""" + DECLARE + table_exists NUMBER; + BEGIN + -- Check if the table exists in the ALL_TABLES metadata view + SELECT COUNT(1) + INTO table_exists + FROM ALL_TABLES + WHERE OWNER = '{{ this.schema | upper }}' + AND TABLE_NAME = '{{ this.table | upper }}'; + + -- If the table exists, run the UPDATE statement + IF table_exists > 0 THEN + EXECUTE IMMEDIATE 'UPDATE {{ this }} SET VALID_TO = SYSDATE WHERE VALID_TO = CAST(''31-DEC-9999'' AS DATE)'; + END IF; + END; + """ +) }} + +SELECT + A_WORKFLOW_HISTORY_KEY, + SYSDATE AS VALID_FROM, + CAST('31-DEC-9999' as DATE) AS VALID_TO, + ANN_1_1_REF_DATE, + CTP_MFI_ID, + CTP_NAME, + CTP_COUNTRY, + CTP_LEGAL_STATUS, + CTP_LPRT, + CTP_MEASURE_START_DATE, + ANN_1_2_REF_DATE, + ETWDR_MFI_ID, + ETWDR_LEI, + ETWDR_NAME, + ETWDR_COUNTRY, + ETWDR_IN_EU, + ETWDR_SOURCE, + ETWDR_SUBMITTER, + REF_DATE, + CURRENCY, + REPORTED_BY_SUPERVISOR, + CONFIRMED_BY_SUPERVISOR, + EXP_COREP_CONS, + EXP_COREP_SOLO, + EXP_FINREP_CONS, + EXP_FINREP_SOLO, + EXP_LEV_CONS, + EXP_LEV_SOLO, + EXP_LCR_CONS, + EXP_LCR_SOLO, + EXP_NSFR_CONS, + EXP_NSFR_SOLO, + CONS_CET1_AMT, + CONS_CET1_AMT_EUR, + CONS_TIER1_AMT, + CONS_TIER1_AMT_EUR, + CONS_TOT_CAP_AMT, + CONS_TOT_CAP_AMT_EUR, + CONS_CET1_RATIO, + CONS_TIER1_RATIO, + CONS_TOT_CAP_RATIO, + CONS_RISK_WGHT_ASSETS, + CONS_RISK_WGHT_ASSETS_EUR, + SOLO_CET1_AMT, + SOLO_CET1_AMT_EUR, + SOLO_TIER1_AMT, + SOLO_TIER1_AMT_EUR, + SOLO_TOT_CAP_AMT, + SOLO_TOT_CAP_AMT_EUR, + SOLO_CET1_RATIO, + SOLO_TIER1_RATIO, + SOLO_TOT_CAP_RATIO, + SOLO_RISK_WGHT_ASSETS, + SOLO_RISK_WGHT_ASSETS_EUR, + CONS_TOT_ASSETS, + CONS_TOT_ASSETS_EUR, + SOLO_TOT_ASSETS, + SOLO_TOT_ASSETS_EUR, + CONS_LEV_RATIO_FULL, + CONS_LEV_RATIO_TRANS, + CONS_LEV_RATIO_REQ, + CONS_LEV_RATIO_ADJ, + SOLO_LEV_RATIO_FULL, + SOLO_LEV_RATIO_TRANS, + SOLO_LEV_RATIO_REQ, + SOLO_LEV_RATIO_ADJ, + CONS_LC_RATIO, + SOLO_LC_RATIO, + CONS_NSFR_RATIO, + SOLO_NSFR_RATIO, + SUBMITTER_COMMENT, + ANN_1_1_SUBMITTER_COMMENT, + ANN_1_2_SUBMITTER_COMMENT, + MPEC_SNAPSHOT_DATE, + ACCESS_TO, + SCOPF_OBLIGATIONMODULEREFERENCEDATE, + SCOPF_REPORTINGENTITYCOLLECTIONUNIQUEID, + SCOPF_RECEIVEDFILEVERSIONNUMBER, + SCOPF_RECEIVEDFILERECEIVEDDATE, + SCOPF_REVALIDATIONDATE, + DCT_OBLIGATIONMODULEREFERENCEDATE, + DCT_REPORTINGENTITYCOLLECTIONUNIQUEID, + DCT_RECEIVEDFILEVERSIONNUMBER, + DCT_RECEIVEDFILERECEIVEDDATE, + DCT_REVALIDATIONDATE +FROM {{ ref('m_MOPDB_RQSD_OUTPUT_CURR_RQSD_NCB_SUBA_SQ') }} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT.sql b/dbt/models/mopdb/targets/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT.sql new file mode 100644 index 0000000..a59b7a0 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema="T2", + alias="T_MAXIMA_CREDIT_LINE_AMOUNT_MARS", + on_schema_change='ignore', + tag=["m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT"], + pre_hook = "DELETE FROM {{ this }} TRG WHERE ( NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') ) IN + (SELECT DISTINCT NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') + FROM {{ ref('m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ') }} ) ", + meta={ + "author": "Atanas Georgiev", + "created": "2026-02-00", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2026-02-00" + } +) +}} + +select +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_WORKFLOW_HISTORY_KEY, +CAST(BUSINESS_DATE AS DATE) as BUSINESS_DATE, +CAST(CURRENCY AS VARCHAR2(3 CHAR)) as CURRENCY, +CAST(SYSTEM_ENTITY AS VARCHAR2(4 CHAR)) as SYSTEM_ENTITY, +CAST(PARTY_RIAD AS VARCHAR2(256 CHAR)) as PARTY_RIAD, +CAST(PARTY_BIC AS VARCHAR2(256 CHAR)) as PARTY_BIC, +CAST(MAXIMA_OF_CREDIT_LINE_AMOUNT AS NUMBER(28,10)) as MAXIMA_OF_CREDIT_LINE_AMOUNT +from + {{ref("m_MOPDB_T2_T_MAXIMA_CREDIT_LINE_AMOUNT_OU_T2_MAXIMA_CREDIT_LINE_AMOUNT_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE.sql b/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE.sql new file mode 100644 index 0000000..f9f2f15 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema="T2", + alias="T_PEAK_INTRADAY_CREDIT_USE_MARS", + on_schema_change='ignore', + tags=["m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE"], + pre_hook = "DELETE FROM {{ this }} TRG WHERE ( NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') ) IN + (SELECT DISTINCT NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') + FROM {{ ref('m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ') }} ) " +)}} + +select +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_WORKFLOW_HISTORY_KEY, +CAST(BUSINESS_DATE AS DATE) as BUSINESS_DATE, +CAST(CURRENCY AS VARCHAR2(3 CHAR)) as CURRENCY, +CAST(SYSTEM_ENTITY AS VARCHAR2(4 CHAR)) as SYSTEM_ENTITY, +CAST(PARTY_RIAD AS VARCHAR2(256 CHAR)) as PARTY_RIAD, +CAST(PARTY_BIC AS VARCHAR2(256 CHAR)) as PARTY_BIC, +CAST(PEAK_IDC_USE AS NUMBER(28,10)) as PEAK_IDC_USE +FROM + {{ref("m_MOPDB_T2_T_PEAK_INTRADAY_CREDIT_USE_OU_T2_PEAK_INTRADAY_CREDIT_USE_SQ")}} + diff --git a/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED.sql b/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED.sql new file mode 100644 index 0000000..e417974 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema="T2", + alias="T_PEAK_LIQUIDITY_NEED_MARS", + on_schema_change='ignore', + tags=["m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED"], + pre_hook = "DELETE FROM {{ this }} TRG WHERE ( NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') ) IN + (SELECT DISTINCT NVL(BUSINESS_DATE,TO_DATE('01019999000000','DDMMYYYYHH24MISS')), NVL(PARTY_RIAD,'PR'), NVL(PARTY_BIC,'PB') + FROM {{ ref('m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ') }} ) " +)}} + +Select +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_WORKFLOW_HISTORY_KEY, +CAST(BUSINESS_DATE AS DATE) as BUSINESS_DATE, +CAST(CURRENCY AS VARCHAR2(3 CHAR)) as CURRENCY, +CAST(SYSTEM_ENTITY AS VARCHAR2(4 CHAR)) as SYSTEM_ENTITY, +CAST(PARTY_RIAD AS VARCHAR2(256 CHAR)) as PARTY_RIAD, +CAST(PARTY_BIC AS VARCHAR2(256 CHAR)) as PARTY_BIC, +CAST(MRG_LEND_INDIC AS VARCHAR2(256 CHAR)) as MRG_LEND_INC, +CAST(PEAK_LIQUIDITY_NEED AS NUMBER(28,10)) as PEAK_LIQUIDITY_NEED, +CAST(TIMESTAMP AS TIMESTAMP(6)) as TIMESTAMP +FROM +{{ref("m_MOPDB_T2_T_PEAK_LIQUIDITY_NEED_OU_T2_PEAK_LIQUIDITY_NEED_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW.sql new file mode 100644 index 0000000..56569e3 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW.sql @@ -0,0 +1,118 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_ACMCURRENCYFLOW_MARS", + tags = ["m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-12", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-23 Alias changed to _MARS" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ')) }}" + ) +}} +SELECT + ACCOUNTINGTREATMENT_NAME, + ACMBOOKKEEPINGTYPE_ID_USER, + ACMCMMSTATE_ID_USER, + ACMCMMVIEWTYPE_ID_USER, + ACMCPTYRELATION_ID_USER, + ACMCURRENCYENTRYSTATE_ID_USER, + ACMCURRENCYPOSITION_ID_USER, + ACMENTRY_ACCOUNT_HOLDER_ID, + ACMENTRY_BANK_ACCOUNT, + ACMENTRY_BANK_ACCOUNT_CCY_ID, + ACMENTRY_BANK_ID, + ACMENTRY_BOOK_VALUE, + ACMENTRY_BOOKING_CURRENCY_ID, + ACMENTRY_C_PAYMENT_DATE, + ACMENTRY_CCY_FLOW_AMOUNT, + ACMENTRY_CCY_FLOW_BOOKING_AMOUNT, + ACMENTRY_CCY_GROUP_ID, + ACMENTRY_CCY_RESULT, + ACMENTRY_CLASSIFICATION_ID, + ACMENTRY_CMM_GROUP_ID, + ACMENTRY_CMM_PAYMENT_METHOD_ID, + ACMENTRY_CMM_RECONCILIATION_ID, + ACMENTRY_COUNTERPARTY_CODE, + ACMENTRY_COUNTERPARTY_GROUP_ID, + ACMENTRY_COUNTERPARTY_ID, + ACMENTRY_CURRENCY_ID, + ACMENTRY_DATE_BASIS, + ACMENTRY_DEAL_PRICE, + ACMENTRY_DEAL_RATE, + ACMENTRY_DESCRIPTION, + ACMENTRY_EVENT_DATE, + ACMENTRY_EVENT_ID, + ACMENTRY_FIGURE_ID, + ACMENTRY_FX_RATE, + ACMENTRY_ID, + ACMENTRY_INSTRUMENT_ID, + ACMENTRY_ISSUER_GROUP_ID, + ACMENTRY_ISSUER_ID, + ACMENTRY_LEG_GROUP, + ACMENTRY_LEG_ID, + ACMENTRY_LOAD_ID, + ACMENTRY_NOMINAL_AMOUNT, + ACMENTRY_NUMBER_, + ACMENTRY_ONE_TIME_COUNTERPARTY_ID, + ACMENTRY_PACKAGE_MAP_ID, + ACMENTRY_PACKAGE_TYPE_ID, + ACMENTRY_PARAM_0, + ACMENTRY_PARAM_1, + ACMENTRY_PARAM_10, + ACMENTRY_PARAM_11, + ACMENTRY_PARAM_12, + ACMENTRY_PARAM_13, + ACMENTRY_PARAM_14, + ACMENTRY_PARAM_15, + ACMENTRY_PARAM_16, + ACMENTRY_PARAM_17, + ACMENTRY_PARAM_18, + ACMENTRY_PARAM_19, + ACMENTRY_PARAM_2, + ACMENTRY_PARAM_3, + ACMENTRY_PARAM_4, + ACMENTRY_PARAM_5, + ACMENTRY_PARAM_6, + ACMENTRY_PARAM_7, + ACMENTRY_PARAM_8, + ACMENTRY_PARAM_9, + ACMENTRY_PORTFOLIO_ID, + ACMENTRY_REVERSE_EVENT_ID, + ACMENTRY_SOURCE_REFERENCE, + ACMENTRY_T_CURRENCY_2_ID, + ACMENTRY_T_CURRENCY_ID, + ACMENTRY_T_MATURITY_DATE, + ACMENTRY_T_OPENING_DATE, + ACMENTRY_T_VALUE_DATE, + ACMENTRY_ORIGIN_ID, + ACMENTRY_ORIGIN_ENTITY_ID, + ACMENTRY_ORIGIN_GROUP_ID, + ACMENTRYTYPE_ID_USER, + ACMEVENTTYPE_ID_USER, + ACMGROUPINGRULE_ID, + ACMGROUPINGRULE_ID_USER, + ACMLEDGER_ID, + ACMLEDGER_ID_USER, + ACMMAPPINGRULE_ID, + ACMMAPPINGRULE_ID_USER, + ACMPERIOD_ID_USER, + ACMPERIODORDERNUMBER_ORDER_NUMBER, + ACMPLCATEGORY_ID_USER, + ACMTRANSACTIONSIGN_NAME, + ACMVOUCHER_BOOKING_DATE, + ACMVOUCHER_ID, + ACMVOUCHER_POSTING_DATE, + ACMVOUCHER_VOUCHER_NUMBER, + TRANSACTIONKIND_NAME, + TRANSACTIONTYPE_NAME, + UMPATHNODE_ID, + UMPATHNODE_UPATH, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY +FROM + {{ ref("m_MOPDB_TMS_T_ACMCURRENCYFLOW_OU_TMS_ACMCURRENCYFLOW_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP.sql new file mode 100644 index 0000000..4eabd3e --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP.sql @@ -0,0 +1,210 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_ACMENTRYSTATELEDGERGROUP_MARS", + tags = ["m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-18", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-24 added on_schema_change" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ACCOUNTINGTREATMENT_NAME, + ACMACCOUNT1_ERP_ACCOUNT, + ACMACCOUNT1_ID_USER, + ACMACCOUNT1_NAME, + ACMACCOUNT2_ID_USER, + ACMBOOKKEEPINGTYPE_ID_USER, + ACMCMMSTATE_ID_USER, + ACMCMMVIEWTYPE_ID_USER, + ACMCOSTCENTER_ID_USER, + ACMCPTYRELATION_ID_USER, + ACMCTBTYPERULE_ID_USER, + ACMDRCR_ID_USER, + ACMENTRY_ACC_HOLDER_CODE, + ACMENTRY_ACC_HOLDER_GROUP_ID, + ACMENTRY_ACCOUNT_HOLDER_ID, + ACMENTRY_ACCOUNT_ID, + ACMENTRY_ADJ_DISCHARGE_DATE, + ACMENTRY_AGGREGATION_ID, + ACMENTRY_AMOUNT, + ACMENTRY_BANK_ACCOUNT, + ACMENTRY_BANK_ACCOUNT_CCY_ID, + ACMENTRY_BANK_ID, + ACMENTRY_BATCH_ID, + ACMENTRY_BOOK_VALUE, + ACMENTRY_BOOKING_AMOUNT, + ACMENTRY_BOOKING_CURRENCY_ID, + ACMENTRY_BOOKING_DATE, + ACMENTRY_C_PAYMENT_DATE, + ACMENTRY_CCY_AMOUNT, + ACMENTRY_CCY_BOOKING_AMOUNT, + ACMENTRY_CCY_GROUP_ID, + ACMENTRY_CCY_GROUPING_RULE_ID, + ACMENTRY_CCY_RESULT, + ACMENTRY_CLASSIFICATION_ID, + ACMENTRY_CMM_GROUP_ID, + ACMENTRY_CMM_PAYMENT_METHOD_ID, + ACMENTRY_CMM_RECONCILIATION_ID, + ACMENTRY_CMMSE_ID, + ACMENTRY_CONTINUATION_NUMBER, + ACMENTRY_COUNTERPARTY_CODE, + ACMENTRY_COUNTERPARTY_GROUP_ID, + ACMENTRY_COUNTERPARTY_ID, + ACMENTRY_COUNTERPARTY_UNIT_ID, + ACMENTRY_CTB_PREVIOUS_DATE, + ACMENTRY_CTB_SEQUENCE_ID, + ACMENTRY_CURRENCY_ID, + ACMENTRY_DATE_BASIS, + ACMENTRY_DEAL_PRICE, + ACMENTRY_DEAL_RATE, + ACMENTRY_DESCRIPTION, + ACMENTRY_EVENT_DATE, + ACMENTRY_EVENT_ID, + ACMENTRY_EXPORT_ID, + ACMENTRY_FAC_REP_GROUP, + ACMENTRY_FACILITY_ID, + ACMENTRY_FIGURE_AMOUNT, + ACMENTRY_FIGURE_BOOKING_AMOUNT, + ACMENTRY_FIGURE_ID, + ACMENTRY_FLAGS, + ACMENTRY_FLAGS_2, + ACMENTRY_FX_RATE, + ACMENTRY_HEDGE_BOOK_EFF_BAL, + ACMENTRY_HEDGE_BOOK_INEFF_BAL, + ACMENTRY_HEDGE_DESIGN_PERCENT, + ACMENTRY_HEDGE_DOC_EFF_BAL, + ACMENTRY_HEDGE_DOC_INEFF_BAL, + ACMENTRY_HEDGE_EFF_PERCENT, + ACMENTRY_HEDGE_ID, + ACMENTRY_HEDGE_RISK, + ACMENTRY_HEDGE_STRATEGY_CODE, + ACMENTRY_HEDGE_TEST_METHOD, + ACMENTRY_HEDGE_TYPE, + ACMENTRY_ID, + ACMENTRY_INSTRUMENT_ID, + ACMENTRY_ISSUER_GROUP_ID, + ACMENTRY_ISSUER_ID, + ACMENTRY_ISSUER_UNIT_ID, + ACMENTRY_LEDGER_ID, + ACMENTRY_LEG_GROUP, + ACMENTRY_LEG_ID, + ACMENTRY_LOAD_ID, + ACMENTRY_LOGICAL_REFERENCE_ID, + ACMENTRY_MAPPING_DESCRIPTION, + ACMENTRY_MATCH_ID, + ACMENTRY_MODIFICATION_TIME, + ACMENTRY_N_OPTION_DATE, + ACMENTRY_NOMINAL_AMOUNT, + ACMENTRY_NUMBER, + ACMENTRY_ONE_TIME_CPTY_ID, + ACMENTRY_ORIG_BOOKING_AMOUNT, + ACMENTRY_ORIGINAL_AMOUNT, + ACMENTRY_OWNER_UNIT_ID, + ACMENTRY_PACKAGE_MAP_ID, + ACMENTRY_PACKAGE_TYPE_ID, + ACMENTRY_PARAM_0, + ACMENTRY_PARAM_1, + ACMENTRY_PARAM_10, + ACMENTRY_PARAM_11, + ACMENTRY_PARAM_12, + ACMENTRY_PARAM_13, + ACMENTRY_PARAM_14, + ACMENTRY_PARAM_15, + ACMENTRY_PARAM_16, + ACMENTRY_PARAM_17, + ACMENTRY_PARAM_18, + ACMENTRY_PARAM_19, + ACMENTRY_PARAM_2, + ACMENTRY_PARAM_3, + ACMENTRY_PARAM_4, + ACMENTRY_PARAM_5, + ACMENTRY_PARAM_6, + ACMENTRY_PARAM_7, + ACMENTRY_PARAM_8, + ACMENTRY_PARAM_9, + ACMENTRY_PARENT_ID, + ACMENTRY_PASE_ID, + ACMENTRY_PORTFOLIO_ID, + ACMENTRY_PRE_VOUCHER_ID, + ACMENTRY_REFERENCE_DATE, + ACMENTRY_REVERSE_EVENT_ID, + ACMENTRY_SECONDARY_ACCOUNT_ID, + ACMENTRY_SOURCE_REFERENCE, + ACMENTRY_STATUS, + ACMENTRY_T_CURRENCY_2_ID, + ACMENTRY_T_CURRENCY_ID, + ACMENTRY_T_MATURITY_DATE, + ACMENTRY_T_OPENING_DATE, + ACMENTRY_T_VALUE_DATE, + ACMENTRY_TAG, + ACMENTRY_VOUCHER_ID, + ACMENTRYORIGIN_ID_USER, + ACMENTRYORIGINENTITY_ID_USER, + ACMENTRYORIGINGROUP_ID_USER, + ACMENTRYSTATE1_ID_USER, + ACMENTRYSTATE2_ID_USER, + ACMENTRYTYPE_ID_USER, + ACMEVENTTYPE_ID_USER, + ACMGROUPINGRULE1_ID, + ACMGROUPINGRULE1_ID_USER, + ACMGROUPINGRULE2_ID, + ACMGROUPINGRULE2_ID_USER, + ACMHEDGECOSTCAT_ID_USER, + ACMHEDGEEFFCAT_ID_USER, + ACMHEDGELEGCAT_ID_USER, + ACMHEDGEQUALIFCAT_ID_USER, + ACMLEDGER_CHART_ID, + ACMLEDGER_ID_USER, + ACMMAPPINGACTION_ID_USER, + ACMMAPPINGRULE_ID, + ACMMAPPINGRULE_ID_USER, + ACMPERIOD_ID_USER, + ACMPLCATEGORY_ID_USER, + ACMPROJECT_ID_USER, + ACMREBOOKINGTYPE_ID_USER, + ACMREVCANCELSTATE1_ID_USER, + ACMREVCANCELSTATE2_ID_USER, + ACMREVCANCELSTATE3_ID_USER, + ACMSIGN1_NAME, + ACMSIGN2_NAME, + ACMSPECIALTREATMENT_ID_USER, + ACMTRANSACTIONSIGN_NAME, + ACMVOUCHER_BOOKING_DATE, + ACMVOUCHER_BUNDLE_ID, + ACMVOUCHER_FLAGS, + ACMVOUCHER_MODIFICATION_TIME, + ACMVOUCHER_POSTING_DATE, + ACMVOUCHER_REQ_BOOKING_DATE, + ACMVOUCHER_REV_VOUCHER_ID, + ACMVOUCHER_REVERSAL_BUNDLE_ID, + ACMVOUCHER_VOUCHER_NUMBER, + ACMVOUCHERSTATE_ID_USER, + ACMVOUCHERTYPE_ID_USER, + ACMVOUCHERTYPE2_ID_USER, + CCY_GROUPING_RULE, + CCY_POSITION, + CCY_STATE, + CLIENT_UNIT_RELATION_ID, + CTB_CATEGORY, + HEDGEDEDESIGNATIONTYPE_NAME, + OPTION_SIGN, + ORIGINAL_TIME_BAND, + REPCCY, + REPORT_OWNER_UNIT_GROUP_ID, + REPORTCCYAMOUNT, + TRANSACTIONKIND_NAME, + TRANSACTIONTYPE_NAME, + UMPATHNODE_ID, + UMPATHNODE_UPATH, + VALUATIONHIERARCHY_NAME +FROM + {{ ref("m_MOPDB_TMS_T_ACMENTRYSTATELEDGERGROUP_OU_TMS_ACMENTRYSTATELEDGERGROUP_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY.sql new file mode 100644 index 0000000..eb22fbf --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY.sql @@ -0,0 +1,37 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_ACTIVITY_MARS", + tags = ["m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-26", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-25" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + GROUP_ID, + ID, + TYPE_ID, + DOMAIN_ID, + FLAGS, + NAME, + SERVICE_NAME, + SERVICE_GROUP, + START_DATE, + TIME_ZONE_ID, + END_DATE, + PREREQUISITE_ID, + OWNER_ID, + TYPE_NAME, + START_DATE_LOCAL, + END_DATE_LOCAL +FROM + {{ ref("m_MOPDB_TMS_T_ACTIVITY_OU_TMS_ACTIVITY_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH.sql new file mode 100644 index 0000000..a94cce3 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_BRANCH", + tags = ["m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-23", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + ID, + NAME, + NUMBER_, + ORDER_NUMBER +FROM + {{ ref("m_MOPDB_TMS_T_BRANCH_OU_TMS_BRANCH_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR.sql new file mode 100644 index 0000000..af5c298 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR.sql @@ -0,0 +1,26 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CALENDAR", + tags = ["m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-06", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + DEFAULT_HOLIDAYS, + HOLIDAY_DATE, + HOLIDAY_REASON, + ID, + NAME +FROM + {{ ref("m_MOPDB_TMS_T_CALENDAR_OU_TMS_CALENDAR_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT.sql new file mode 100644 index 0000000..0bcd8df --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT.sql @@ -0,0 +1,45 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CLIENTACCOUNT", + tags = ["m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-05-29", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + ACCOUNT_1, + ACCOUNT_2, + ACCOUNT_3, + ACCOUNT_4, + ACTIVE_SINCE, + ACTIVE_UNTIL, + BANK1, + BANK2, + BANK3, + BANK4, + CLIENT_ID, + COMMENT_, + CURRENCY_ID, + DELIVERY_VS_PAYMENT, + FULL_CHAIN, + KIND, + LAST_LINK, + NAME, + NOT_RULE_ID, + OWN_ACCOUNT, + PREFER_ACCOUNT, + PREFER_BANK, + PRIORITY, + RULE_ID +FROM + {{ ref("m_MOPDB_TMS_T_CLIENTACCOUNT_OU_TMS_CLIENTACCOUNT_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP.sql new file mode 100644 index 0000000..b0fb04e --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CLIENTGROUPMAP_MARS", + tags = ["m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-05", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-08-21 - change in load logic -> Jira ticket https://europeancentralbank.atlassian.net/browse/UCDB-3573" + } + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ECBOPERATION, + ECBTS, + CLIENT_ID, + MAIN_GROUP_ID, + GROUP_ID, + STAMP, + MAIN_GROUP_NAME, + GROUP_NAME +FROM + {{ ref("m_MOPDB_TMS_T_CLIENTGROUPMAP_OU_TMS_CLIENTGROUPMAP_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION.sql new file mode 100644 index 0000000..339c0e8 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION.sql @@ -0,0 +1,109 @@ + +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CLIENTINSTRUCTION_MARS", + tags = ["m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-01", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340, revert to write in _MARS - ticket MARS-1391" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ACCOUNT_1_COMMENT, + ACCOUNT_1_CURRENCY, + ACCOUNT_1_GROUP, + ACCOUNT_1_NAME, + ACCOUNT_1_NUMBER, + ACCOUNT_1_PARENT_ACCOUNT, + ACCOUNT_1_SECONDARY_NUMBER, + ACCOUNT_1_TARGET_CLOSING_DATE, + ACCOUNT_1_TYPE, + ACCOUNT_2_COMMENT, + ACCOUNT_2_CURRENCY, + ACCOUNT_2_GROUP, + ACCOUNT_2_NAME, + ACCOUNT_2_NUMBER, + ACCOUNT_2_PARENT_ACCOUNT, + ACCOUNT_2_SECONDARY_NUMBER, + ACCOUNT_2_TARGET_CLOSING_DATE, + ACCOUNT_2_TYPE, + ACCOUNT_3_COMMENT, + ACCOUNT_3_CURRENCY, + ACCOUNT_3_GROUP, + ACCOUNT_3_NAME, + ACCOUNT_3_NUMBER, + ACCOUNT_3_PARENT_ACCOUNT, + ACCOUNT_3_SECONDARY_NUMBER, + ACCOUNT_3_TARGET_CLOSING_DATE, + ACCOUNT_3_TYPE, + ACCOUNT_4_COMMENT, + ACCOUNT_4_CURRENCY, + ACCOUNT_4_GROUP, + ACCOUNT_4_NAME, + ACCOUNT_4_NUMBER, + ACCOUNT_4_PARENT_ACCOUNT, + ACCOUNT_4_SECONDARY_NUMBER, + ACCOUNT_4_TARGET_CLOSING_DATE, + ACCOUNT_4_TYPE, + ACTIVE_SINCE, + ACTIVE_UNTIL, + BANK_1, + BANK_2, + BANK_3, + BANK_4, + CLIENT_ID, + CLIENT_NAME, + CLIENT_UNIT_ID, + COMMENT_, + CURRENCY_ID, + INSTRUCTION_NAME, + KIND, + NOT_RULE_CF_CATEGORY_ID, + NOT_RULE_CF_SUBTYPE_ID, + NOT_RULE_CF_TYPE_ID, + NOT_RULE_COUNTERPARTY, + NOT_RULE_COUNTERPARTY_GROUP, + NOT_RULE_CPTY_MAIN_GROUP, + NOT_RULE_CURRENCY, + NOT_RULE_CURRENCY_2, + NOT_RULE_ID, + NOT_RULE_INSTRUMENT, + NOT_RULE_INSTRUMENT_GROUP, + NOT_RULE_NAME, + NOT_RULE_NUMBER, + NOT_RULE_OWNER, + NOT_RULE_PORTFOLIO, + NOT_RULE_TRANSACTION_KIND, + NOT_RULE_TRANSACTION_SIGN, + NOT_RULE_TRANSACTION_TYPE, + PRIORITY, + RULE_CASHFLOW_CATEGORY, + RULE_CF_SUBTYPE_ID, + RULE_CF_TYPE_ID, + RULE_CF_TYPE_NAME, + RULE_COUNTERPARTY, + RULE_COUNTERPARTY_GROUP, + RULE_CPTY_MAIN_GROUP, + RULE_CURRENCY, + RULE_CURRENCY_2, + RULE_ID, + RULE_INSTRUMENT, + RULE_INSTRUMENT_GROUP, + RULE_NAME, + RULE_NUMBER, + RULE_OWNER, + RULE_PORTFOLIO, + RULE_TRANSACTION_KIND, + RULE_TRANSACTION_SIGN, + RULE_TRANSACTION_TYPE +FROM + {{ ref("m_MOPDB_TMS_T_CLIENTINSTRUCTION_OU_TMS_CLIENTINSTRUCTION_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT.sql new file mode 100644 index 0000000..3602e2d --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT.sql @@ -0,0 +1,41 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CLIENT_MARS", + tags = ["m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT"], + meta={ + "author": "Adelina Borisova", + "created": "2025-06-02", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-25 added on_schema_change" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ')) }}" + ) +}} +SELECT + ECBOPERATION, + ECBTS, + CLIENT_ID, + DOMAIN_ID, + COUNTRY_ID, + ROLES, + GROUPS, + FLAGS, + STAMP, + TEMPLATE_ID, + NAME, + PARENT_ID, + SHORT_NAME, + POSTAL_ADDRESS, + STREET_ADDRESS, + PAYMENT_ADDRESS, + CITY, + RISK_COUNTRY_ID, + SWIFT_CODE, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + CLIENT_FLAGS +FROM + {{ ref("m_MOPDB_TMS_T_CLIENT_OU_TMS_CLIENT_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE.sql new file mode 100644 index 0000000..f9f3fff --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE.sql @@ -0,0 +1,46 @@ + +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_CUSTODYBALANCE_MARS", + tags = ["m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-03", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-25 added on_schema_change" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ID, + CLIENT_ID, + CLIENT_UNIT_ID, + CUSTODY_ID, + CUSTODY_UNIT_ID, + ACCOUNT_ID, + DATE_, + INSTRUMENT_ID, + CURRENCY_ID, + ISSUER_ID, + ISSUER_UNIT_ID, + MATURITY_DATE, + BALANCE_AMOUNT, + SELLABLE_AMOUNT, + BALANCE_FACE_AMOUNT, + SELLABLE_FACE_AMOUNT, + BALANCE_UNITS, + SELLABLE_UNITS, + COMMENT_, + FLAGS, + INSTRUMENT_GROUP, + ISIN_CODE, + CUSIP_CODE, + SEDOL_CODE +FROM + {{ ref("m_MOPDB_TMS_T_CUSTODYBALANCE_OU_TMS_CUSTODYBALANCE_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS.sql new file mode 100644 index 0000000..9d99e42 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS.sql @@ -0,0 +1,39 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_EFFECTIVEPERMISSIONS_MARS", + tags = ["m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-17", + "last_updated_by": "Adelina Borisova", + "last_updated": "2025-07-17" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ')) }}" + + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ID, + NAME, + ADMIN_CENTER_ID, + TIME_ZONE_ID, + MENU_NAME, + WEB_HOME_GROUP_ID, + COMMENT_, + FLAGS, + ACCESS_PROFILE_ID, + ACCESS_PROFILE_NAME, + ACCESS_PROFILE_TYPE_ID, + ACCESS_PROFILE_TYPE_NAME, + ENTITY_ID, + PERMISSION_ID, + RESOURCE_RULE, + ACTIVE_FROM, + ACTIVE_TO +FROM + {{ ref("m_MOPDB_TMS_T_EFFECTIVEPERMISSIONS_OU_TMS_EFFECTIVEPERMISSIONS_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE.sql new file mode 100644 index 0000000..65b4674 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE.sql @@ -0,0 +1,37 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_EFFECTIVEROLEPROFILE_MARS", + tags = ["m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-21", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-25 added on_schema_change" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, + ID, + NAME, + ADMIN_CENTER_ID, + TIME_ZONE_ID, + MENU_NAME, + WEB_HOME_GROUP_ID, + FLAGS, + ACCESS_ROLE_ID, + ACCESS_ROLE_NAME, + DIRECT_P, + ACTIVE_FROM, + ACTIVE_TO, + ACCESS_PROFILE_ID, + ACCESS_PROFILE_NAME, + ACCESS_PROFILE_TYPE_ID, + ACCESS_PROFILE_TYPE_NAME +FROM + {{ ref("m_MOPDB_TMS_T_EFFECTIVEROLEPROFILE_OU_TMS_EFFECTIVEROLEPROFILE_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG.sql new file mode 100644 index 0000000..9ebb303 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_HISTORY_LOG", + tags = ["m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG"], + meta={ + "author": "Adelina Borisova", + "created": "2025-08-20", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "DELETE FROM {{ this }} + WHERE trunc(DATE_ ) in ( SELECT DISTINCT trunc(DATE_ ) FROM {{ ref('m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ')}})" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + OBJECT_ID, + ACTION, + USER_ID, + DATE_, + KEY_1_STRING, + FIELD_NAME, + KEY_2_STRING, + KEY_3_STRING, + KEY_4_STRING, + FROM_VALUE, + TO_VALUE, + DOMAIN_ID +FROM + {{ ref("m_MOPDB_TMS_T_HISTORY_LOG_OU_TMS_HISTORY_LOG_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS.sql new file mode 100644 index 0000000..916f361 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_LIMIT_ACCESS", + tags = ["m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS"], + meta={ + "author": "Adelina Borisova", + "created": "2025-07-21", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + ECBTS, + LIMIT_ID, + USER_ID, + ECBUSRBDTS +FROM + {{ ref("m_MOPDB_TMS_T_LIMIT_ACCESS_OU_TMS_RARLIMITACCESS_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS.sql new file mode 100644 index 0000000..59de936 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized = "incremental", + on_schema_change = 'ignore', + incremental_strategy = "append", + schema = "TMS", + alias = "T_PORTFOLIO_ACCESS", + tags = ["m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS"], + meta={ + "author": "Adelina Borisova", + "created": "2025-10-24", + "last_updated_by": "Adelina Borisova", + "last_updated": "2026-02-16 MARS-1340" + }, + pre_hook = "{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ')) }}" + ) +}} +SELECT + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK, + ECBTS, + PORTFOLIO_ID, + USER_ID, + ECBUSRBDTS +FROM + {{ref("m_MOPDB_TMS_T_PORTFOLIO_ACCESS_OU_TMS_TEMP_RAR_PORTFOLIOACCESS_SQ")}} diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE.sql new file mode 100644 index 0000000..0d3258f --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE.sql @@ -0,0 +1,28 @@ +{{config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TMS", + alias = "T_ROLEPORTFOLIOPROFILE_MARS", + on_schema_change='ignore', + tags = ["m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE"], + pre_hook="{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ')) }}", + meta={ + "author": "Atanas Georgiev", + "created": "2025-11-14", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-11-14" + } + ) +}} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_WORKFLOW_HISTORY_KEY, +CAST(ROLE AS VARCHAR2(30 CHAR)) as ROLE, +CAST(INHERITED_ROLE AS VARCHAR2(30 CHAR)) as INHERITED_ROLE, +CAST(ACCESS_PROFILE_ID AS VARCHAR2(30 CHAR)) as ACCESS_PROFILE_ID, +CAST(ACCESS_PROFILE_TYPE_ID AS VARCHAR2(128 CHAR)) as ACCESS_PROFILE_TYPE_ID, +CAST(PERMISSION_ID AS VARCHAR2(255 CHAR)) as PERMISSION_ID, +CAST(ENTITY_ID AS VARCHAR2(999 CHAR)) as ENTITY_ID +from + {{ref("m_MOPDB_TMS_T_ROLEPORTFOLIOPROFILE_OU_TMS_ROLEPORTFOLIOPROFILE_SQ")}} + \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION.sql b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION.sql new file mode 100644 index 0000000..a10151f --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION.sql @@ -0,0 +1,67 @@ +{{ + config( + materialized="incremental", + incremental_strategy= "append", + schema="TMS", + alias="T_SECURITYPOSITION_MARS", + on_schema_change='ignore', + tags=["m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION"], + pre_hook="{{ conditional_truncate(this, ref('m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ')) }}", + meta={ + "author": "Atanas Georgiev", + "created": "2025-11-14", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-11-14" + } + ) +}} + +SELECT +{{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY, +DATE_, +PORTFOLIO_ID, +INSTRUMENT_ID , +CONTEXTS, +CURRENCY_ID , +PACKAGE_MAIN_TYPE_ID, +PACKAGE_TYPE_ID , +CAST(PACKAGE_ID AS NUMBER(10,0)) as PACKAGE_ID, +CAST(INFLOW_NOMINAL_AMOUNT AS NUMBER(19,4)) as INFLOW_NOMINAL_AMOUNT, +CAST(OUTFLOW_NOMINAL_AMOUNT AS NUMBER(19,4)) as OUTFLOW_NOMINAL_AMOUNT, +CAST(NOMINAL_AMOUNT AS NUMBER(19,4)) as NOMINAL_AMOUNT, +CAST(INFLOW_UNITS AS NUMBER(19,4)) as INFLOW_UNITS, +CAST(OUTFLOW_UNITS AS NUMBER(19,4)) as OUTFLOW_UNITS, +CAST(UNITS AS NUMBER(19,4)) as UNITS, +CAST(INFLOW_BOOK_VALUE AS NUMBER(19,4)) as INFLOW_BOOK_VALUE, +CAST(OUTFLOW_BOOK_VALUE AS NUMBER(19,4)) as OUTFLOW_BOOK_VALUE, +CAST(BOOK_VALUE AS NUMBER(19,4)) as BOOK_VALUE, +CAST(REDEMPTION_AMOUNT AS NUMBER(19,4)) as REDEMPTION_AMOUNT, +CAST(ACCRUED_PROFIT AS NUMBER(19,4)) as ACCRUED_PROFIT, +CAST(INDEXATION_PROFIT AS NUMBER(19,4)) as INDEXATION_PROFIT, +CAST(DISCOUNT_PREMIUM AS NUMBER(19,4)) as DISCOUNT_PREMIUM, +CAST(ACCRUED_BOOK_VALUE AS NUMBER(19,4)) as ACCRUED_BOOK_VALUE, +CAST(REALIZED_MTOM_PROFIT AS NUMBER(19,4)) as REALIZED_MTOM_PROFIT, +CAST(REALIZED_ACCRUED_PROFIT AS NUMBER(19,4)) as REALIZED_ACCRUED_PROFIT, +CAST(REALIZED_INDEXATION_PROFIT AS NUMBER(19,4)) as REALIZED_INDEXATION_PROFIT, +CAST(BVC_MTOM_PROFIT AS NUMBER(19,4)) as BVC_MTOM_PROFIT, +CAST(MTOM_PROFIT AS NUMBER(19,4)) as MTOM_PROFIT, +CAST(MARKET_VALUE AS NUMBER(19,4)) as MARKET_VALUE, +CAST(ACCRUED_INTEREST AS NUMBER(19,4)) as ACCRUED_INTEREST, +CAST(ACCRUED_INTEREST_BOUGHT AS NUMBER(19,4)) as ACCRUED_INTEREST_BOUGHT, +CAST(ACCRUED_INTEREST_SOLD AS NUMBER(19,4)) as ACCRUED_INTEREST_SOLD, +CAST(YIELD AS FLOAT) as YIELD, +CAST(PRICE AS FLOAT) as PRICE, +CAST(MARKET_QUOTE AS FLOAT) as MARKET_QUOTE, +CAST(INDEX_VALUE AS FLOAT) as INDEX_VALUE, +CAST(ISSUER_ID AS VARCHAR2(20 CHAR)) as ISSUER_ID, +CAST(ISSUE_DATE AS DATE) as ISSUE_DATE, +MATURITY_DATE, +ISIN_CODE, +ACCRUAL_METHOD, +ACCRUAL_INTEREST_TYPE, +ACCRUAL_DATE_BASIS, +INPUT_DATE, +CAST(FLAGS AS NUMBER(10,0)) as FLAGS, +CAST(FACE_AMOUNT AS NUMBER(19,4)) as FACE_AMOUNT +FROM +{{ref("m_MOPDB_TMS_T_SECURITYPOSITION_OU_TMS_SECURITYPOSITION_SQ")}} \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT.sql new file mode 100644 index 0000000..9955f05 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_AGGREGATED_ALLOTMENT", + tags = ["m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_MOPDB_LOAD_SET_FK , +CAST(REF_NO AS VARCHAR2(4000 CHAR)) as REF_NO, +CAST(TIMESTAMP_SENDER AS VARCHAR2(4000 CHAR)) as TIMESTAMP_SENDER, +CAST(ID AS NUMBER(28,0)) as ID, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM, +CAST(REVISION AS NUMBER(28,0)) as REVISION, +CAST(TRANSACTION_TYPE AS VARCHAR2(255 CHAR)) as TRANSACTION_TYPE, +CAST(OPERATION_CURRENCY AS VARCHAR2(3 CHAR)) as OPERATION_CURRENCY, +CAST(OPERATION_NAME AS VARCHAR2(255 CHAR)) as OPERATION_NAME, +CAST(OPERATION_CATEGORY AS VARCHAR2(50 CHAR)) as OPERATION_CATEGORY, +CAST(OPERATION_TYPE AS VARCHAR2(255 CHAR)) as OPERATION_TYPE, +CAST(PROCEDURE AS VARCHAR2(255 CHAR)) as PROCEDURE, +CAST(AUCTION_TYPE AS VARCHAR2(255 CHAR)) as AUCTION_TYPE, +CAST(ALLOTMENT_METHOD AS VARCHAR2(255 CHAR)) as ALLOTMENT_METHOD, +CAST(QUOTATION AS VARCHAR2(255 CHAR)) as QUOTATION, +CAST(ALLOTMENT_DATE AS DATE) as ALLOTMENT_DATE, +CAST(VALUE_DATE AS DATE) as VALUE_DATE, +CAST(MATURITY_DATE AS DATE) as MATURITY_DATE, +CAST(DURATION_OF_OPERATION AS NUMBER(10,0)) as DURATION_OF_OPERATION, +CAST(REFERENCE_ENTITY AS VARCHAR2(60 CHAR)) as REFERENCE_ENTITY, +CAST(SPOT_RATE AS NUMBER(20,9)) as SPOT_RATE, +CAST(REPAID_REF_NO AS VARCHAR2(8 CHAR)) as REPAID_REF_NO, +CAST(BUSINESS_VALUE_TYPE AS VARCHAR2(20 CHAR)) as BUSINESS_VALUE_TYPE, +CAST(BID_CURRENCY AS VARCHAR2(3 CHAR)) as BID_CURRENCY, +CAST(OTHER_CURRENCY AS VARCHAR2(3 CHAR)) as OTHER_CURRENCY, +CAST(TOTAL_NUMBER_BIDDERS AS NUMBER(28,0)) as TOTAL_NUMBER_BIDDERS, +CAST(TOTAL_BID_AMOUNT AS NUMBER(28,8)) as TOTAL_BID_AMOUNT, +CAST(TOTAL_ALLOTTED_AMOUNT AS NUMBER(28,8)) as TOTAL_ALLOTTED_AMOUNT, +CAST(NCB_SENDER_REFERENCE AS VARCHAR2(3 CHAR)) as NCB_SENDER_REFERENCE, +CAST(NCB_NUMBER_BIDDERS AS NUMBER(28,0)) as NCB_NUMBER_BIDDERS, +CAST(NCB_BID_AMOUNT AS NUMBER(28,8)) as NCB_BID_AMOUNT, +CAST(NCB_ALLOTTED_AMOUNT AS NUMBER(28,8)) as NCB_ALLOTTED_AMOUNT, +CAST(BIDDER_ID AS VARCHAR2(15 CHAR)) as BIDDER_ID, +CAST(BIDDER_NAME AS VARCHAR2(255 CHAR)) as BIDDER_NAME, +CAST(BIDDER_BID_AMOUNT AS NUMBER(28,8)) as BIDDER_BID_AMOUNT, +CAST(BIDDER_ALLOTTED_AMOUNT AS NUMBER(28,8)) as BIDDER_ALLOTTED_AMOUNT, +CAST(BIDDER_ALLOTTED_BUSINESS_VALUE AS NUMBER(28,8)) as BIDDER_ALLOTTED_BUSINESS_VALUE, +CAST(BID_AMOUNT AS NUMBER(28,8)) as BID_AMOUNT, +CAST(BID_BUSINESS_VALUE AS NUMBER(28,8)) as BID_BUSINESS_VALUE, +CAST(BID_ALLOTTED_AMOUNT AS NUMBER(28,8)) as BID_ALLOTTED_AMOUNT, +CAST(BID_ALLOTTED_BUSINESS_VALUE AS NUMBER(28,8)) as BID_ALLOTTED_BUSINESS_VALUE, +CAST(BID_ALLOTTED_PERCENTAGE AS NUMBER(5,2)) as BID_ALLOTTED_PERCENTAGE +FROM + {{ ref("m_MOPDB_TOP_T_AGGREGATED_ALLOTMENT_OU_TOP_AGGREGATED_ALLOTMENT_SQ") }} +WHERE + MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR.sql new file mode 100644 index 0000000..7fd8839 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR.sql @@ -0,0 +1,33 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_ALLOTMENT_MODIFICATIONS_AGGREGATED", + tags = ["m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_LOAD"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)", + meta={ + "author": "Atanas Georgiev", + "created": "2025-11-14", + "last_updated_by": "Atanas Georgiev", + "last_updated": "2025-11-14" + } +) }} + +select +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM, +CAST(ID AS NUMBER(38,0)) as ID, +CAST(OPERATION_CURRENCY AS VARCHAR2(3 CHAR)) as OPERATION_CURRENCY, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST(REVISION AS NUMBER(38,0)) as REVISION, +CAST(TIMESTAMP_SENDER AS DATE) as TIMESTAMP_SENDER, +CAST(AMOUNT_ITEM AS NUMBER(30,8)) as AMOUNT_ITEM, +CAST(BIDDER_ID AS VARCHAR2(15 CHAR)) as BIDDER_ID, +CAST(FREE_TEXT AS VARCHAR2(4000 CHAR)) as FREE_TEXT, +CAST(SENDER_REFERENCE AS VARCHAR2(3 CHAR)) as SENDER_REFERENCE, +CAST(VALUE_DATE AS DATE) as VALUE_DATE, +CAST(NEW_BIDDER AS VARCHAR2(4000 CHAR)) as NEW_BIDDER +FROM + {{ref("m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_AGGREGATED_OU_TOP_ALLOTMENT_MOD_AGR_SQ")}} +WHERE + MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS.sql new file mode 100644 index 0000000..31ea4d2 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_ALLOTMENT_MODIFICATIONS", + tags = ["m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +case when instr(TIMESTAMP_, '-')<> 0 then to_date (TIMESTAMP_ , 'YYYY-MM-DD"T"HH24:MI:SS' ) else null end AS TIMESTAMP_ , +NVL(TO_NUMBER(REVISION) , -1 ) as REVISION , +TO_DATE(REPLACE(TIMESTAMP_SENDER,'T',' '),'YYYY-MM-DD HH24:MI:SS') as TIMESTAMP_SENDER, +CAST(ID AS NUMBER(38,0)) as ID, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM, +CAST(AMOUNT_ITEM AS NUMBER(30,8)) as AMOUNT_ITEM, +TO_DATE(VALUE_DATE,'YYYY-MM-DD') as VALUE_DATE, +CAST(FREE_TEXT AS VARCHAR2(4000 CHAR)) as FREE_TEXT, +CAST(REF_NO AS VARCHAR2(4000 BYTE)) as REF_NO, +CAST(OPERATION_CURRENCY AS VARCHAR2(4000 BYTE)) as OPERATION_CURRENCY, +CAST(SENDER_REFERENCE AS VARCHAR2(4000 BYTE)) as SENDER_REFERENCE, +CAST(BIDDER_ID AS VARCHAR2(4000 BYTE)) as BIDDER_ID, +CAST(FLOW_TYPE AS VARCHAR2(4000 BYTE)) as FLOW_TYPE +FROM + {{ref("m_MOPDB_TOP_T_ALLOTMENT_MODIFICATIONS_OU_TOP_ALLOTMENT_MODIFICATIONS_SQ")}} +WHERE + MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT.sql new file mode 100644 index 0000000..21e3682 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT.sql @@ -0,0 +1,48 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_ALLOTMENT", + on_schema_change='ignore', + tags = ["m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST(NULL AS VARCHAR2(3 CHAR)) as CURRENCY_INFO, -- not presented in ODS +CAST(SENDER_REF AS VARCHAR2(3 CHAR)) as SENDER_REF, +CAST(USER_REF AS VARCHAR2(20 CHAR)) as USER_REF, +to_date(TIMESTAMP , 'YYYY-MM-DD"T"HH24:MI:SS' ) as TIMESTAMP, +CAST(VERSION AS VARCHAR2(2 CHAR)) as VERSION, +CAST(AMOUNT AS NUMBER(30,8)) as AMOUNT, +CAST(NUMBER_BIDDERS AS NUMBER(32,0)) as NUMBER_BIDDERS, +CAST(CUTOFF_LEVEL AS NUMBER(30,8)) as CUTOFF_LEVEL, +CAST(AMNT_ALLOTED AS NUMBER(30,8)) as AMNT_ALLOTED, +CAST(MIN_ALLOTMENT AS NUMBER(20,9)) as MIN_ALLOTMENT, +CAST(PERC_ALLOTED AS NUMBER(30,8)) as PERC_ALLOTED, +CAST(MARGINAL_RATE_TYPE AS VARCHAR2(20 CHAR)) as MARGINAL_RATE_TYPE, +CAST(MARGINAL_RATE AS NUMBER(20,9)) as MARGINAL_RATE, +CAST(MIN_RATE_TYPE AS VARCHAR2(20 CHAR)) as MIN_RATE_TYPE, +CAST(MIN_RATE AS NUMBER(20,9)) as MIN_RATE, +CAST(MAX_RATE_TYPE AS VARCHAR2(20 CHAR)) as MAX_RATE_TYPE, +CAST(MAX_RATE AS NUMBER(20,9)) as MAX_RATE, +CAST(AVG_RATE_TYPE AS VARCHAR2(20 CHAR)) as AVG_RATE_TYPE, +CAST(AVG_RATE AS NUMBER(20,9)) as AVG_RATE, +CAST(SPOT_RATE AS NUMBER(20,9)) as SPOT_RATE, +CAST(FREE_TEXT AS VARCHAR2(4000 CHAR)) as FREE_TEXT, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM, +CAST(NULL AS DATE) as PRINT_DATE, -- not presented in ODS +CAST(ISIN AS VARCHAR2(30 CHAR)) as ISIN, +CAST(REF_ENTITY AS VARCHAR2(255 CHAR)) as REF_ENTITY, +CAST(REF_ENTITY_VALUE AS VARCHAR2(255 CHAR)) as REF_ENTITY_VALUE, +CAST(SPOT_RATE_WITH_MARGIN AS NUMBER(20,9)) as SPOT_RATE_WITH_MARGIN, +to_date (TIMESTAMP_SENDER , 'YYYY-MM-DD"T"HH24:MI:SS' ) as TIMESTAMP_SENDER, +CAST(ID AS VARCHAR2(255 CHAR)) as ID, +CAST(REPAID_REF_NO AS VARCHAR2(8 CHAR)) as REPAID_REF_NO, +NVL(CAST(REVISION AS NUMBER(38,0)),-1) as REVISION + from + {{ ref("m_MOPDB_TOP_T_ALLOTMENT_OU_TOP_ALLOTMENT_SQ") }} +WHERE + MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT.sql new file mode 100644 index 0000000..5beee01 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_ANNOUNCEMENT", + tags = ["m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST(NULL AS VARCHAR2(3 CHAR)) as CURRENCYINFO, +CAST(SENDER_REF AS VARCHAR2(3 CHAR)) as SENDER_REF, +CAST(USER_REF AS VARCHAR2(20 CHAR)) as USER_REF, +TO_DATE( trim(TIMESTAMP) , 'YYYY-MM-DD"T"HH24:MI:SS' ) AS "TIMESTAMP" , +CAST( VERSION AS VARCHAR2(2 CHAR)) as VERSION , +CAST(OPERATION_TYPE AS VARCHAR2(255 CHAR)) as OPERATION_TYPE, +CAST(PROCEDURE AS VARCHAR2(255 CHAR)) as PROCEDURE, +CAST(TRANSACTION_TYPE AS VARCHAR2(255 CHAR)) as TRANSACTION_TYPE, +CAST(OPERATION_NAME AS VARCHAR2(255 CHAR)) as OPERATION_NAME, +CAST(AUCTION_TYPE AS VARCHAR2(255 CHAR)) as AUCTION_TYPE, +CAST(ALL_METHOD AS VARCHAR2(255 CHAR)) as ALL_METHOD, +CAST(QUOTATION AS VARCHAR2(255 CHAR)) as QUOTATION, +CAST(BUSINESS_VALUE_TYPE AS VARCHAR2(20 CHAR)) as BUSINESS_VALUE_TYPE, +CAST(BUSINESS_VALUE AS NUMBER(20,9)) as BUSINESS_VALUE, +TO_DATE(ALLOTMENT_DATE, 'YYYY-MM-DD"T"HH24:MI:SS' ) AS ALLOTMENT_DATE, +TO_DATE(VALUE_DATE,'YYYY-MM-DD') AS VALUE_DATE, +TO_DATE(MATURITY_DATE,'YYYY-MM-DD') AS MATURITY_DATE, +CAST(DURATION_OF_OPERATION AS NUMBER(10,0)) as DURATION_OF_OPERATION, +CAST(SPOT_RATE AS NUMBER(20,9)) as SPOT_RATE, +CAST(OTHER_CURRENCY AS VARCHAR2(3 CHAR)) as OTHER_CURRENCY, +CAST(OPERATION_CURRENCY_LEG AS VARCHAR2(255 CHAR)) as OPERATION_CURRENCY_LEG, +CAST(MIN_BID_AMT AS NUMBER(20,5)) as MIN_BID_AMT, +CAST(MAX_BID_AMT AS NUMBER(20,5)) as MAX_BID_AMT, +CAST(INTENDED_VOLUME AS NUMBER(20,5)) as INTENDED_VOLUME, +CAST(MIN_ALLOTMENT AS NUMBER(20,5)) as MIN_ALLOTMENT, +CAST(MIN_ALLOTMENT_RATIO AS NUMBER(20,5)) as MIN_ALLOTMENT_RATIO, +TO_DATE(DEADLINE_CP,'YYYY-MM-DD"T"HH24:MI:SS')AS DEADLINE_CP, +TO_DATE(DEADLINE_NCB,'YYYY-MM-DD"T"HH24:MI:SS') AS DEADLINE_NCB, +CAST(PUBLIC_ANNOUNCEMENT AS VARCHAR2(1 CHAR)) as PUBLIC_ANNOUNCEMENT, +CAST(FREE_TEXT AS VARCHAR2(4000 CHAR)) as FREE_TEXT, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM, +CAST(NULL AS DATE) AS PRINT_DATE, +CAST(OPER_CURRENCY AS VARCHAR2(3 CHAR)) as OPER_CURRENCY, +CAST(ISIN AS VARCHAR2(30 CHAR)) as ISIN, +CAST(REF_ENTITY AS VARCHAR2(60 CHAR)) as REF_ENTITY, +CAST(SPOT_RATE_WITH_MARGIN AS NUMBER(20,9)) as SPOT_RATE_WITH_MARGIN, +CAST(MAX_BIDS_COUNTER_PARTY AS NUMBER(10,0)) as MAX_BIDS_COUNTER_PARTY, +CAST(EXACT_DENOMINATION AS NUMBER(20,5)) as EXACT_DENOMINATION, +CAST(BID_CURRENCY AS VARCHAR2(3 CHAR)) as BID_CURRENCY, +TO_DATE(TIMESTAMP_SENDER,'YYYY-MM-DD"T"HH24:MI:SS') AS TIMESTAMP_SENDER, +CAST(ID AS VARCHAR2(255 CHAR)) as ID, +CAST(REPAID_REF_NO AS VARCHAR2(8 CHAR)) as REPAID_REF_NO, +NVL(CAST(REVISION AS NUMBER(38,0)),-1) as REVISION, +CAST(OPERATION_CATEGORY AS VARCHAR2(50 CHAR)) as OPERATION_CATEGORY +FROM + {{ ref("m_MOPDB_TOP_T_ANNOUNCEMENT_OU_TOP_ANNOUNCEMENT_SQ") }} OU_TOP_ANNOUNCEMENT +WHERE + REF_NO IS NOT NULL AND A_WORKFLOW_HISTORY_KEY=MAX_A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM.sql new file mode 100644 index 0000000..9ed5706 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_FBL_ITEM", + on_schema_change='ignore', + tags = ["m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_LOAD"], + pre_hook = "DELETE FROM {{ this }} WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ') }} WHERE MAX_A_WORKFLOW_HISTORY_KEY = A_WORKFLOW_HISTORY_KEY)" +) }} + +SELECT +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +CAST(SEND_REF AS VARCHAR2(3 CHAR)) as SEND_REF, +CAST(LARGE_BIDDER_NAME AS VARCHAR2(255 CHAR)) as LARGE_BIDDER_NAME, +CAST(LARGE_BIDDER_ID AS VARCHAR2(15 CHAR)) as LARGE_BIDDER_ID, +CAST(AMOUNT AS NUMBER(30,8)) as AMOUNT, +CAST(AMOUNT_BUS_VALUE_TYPE AS VARCHAR2(20 CHAR)) as AMOUNT_BUS_VALUE_TYPE, +CAST(AMOUNT_BUS_VALUE AS NUMBER(20,9)) as AMOUNT_BUS_VALUE, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST( NVL(TO_NUMBER(REVISION),-1) AS NUMBER(38,0)) AS REVISION +FROM +{{ref("m_MOPDB_TOP_T_FBL_ITEM_OU_TOP_FBL_ITEM_SQ")}} +WHERE + A_WORKFLOW_HISTORY_KEY=MAX_A_WORKFLOW_HISTORY_KEY \ No newline at end of file diff --git a/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED.sql b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED.sql new file mode 100644 index 0000000..fb58624 --- /dev/null +++ b/dbt/models/mopdb/targets/m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = "incremental", + incremental_strategy = "append", + schema = "TOP", + alias = "T_FULLBID_ARRAY_COMPILED", + tags = ["m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_LOAD"], + on_schema_change='ignore', + pre_hook = "DELETE FROM {{ this }} TRG WHERE REF_NO IN (SELECT DISTINCT REF_NO FROM {{ ref('m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ') }} SQ WHERE TRG.REF_NO=SQ.REF_NO)" +) }} + +select +CAST( {{ get_workflow_history_key() }} AS NUMBER(38,0)) as A_MOPDB_LOAD_SET_FK, +CAST(REF_NO AS VARCHAR2(8 CHAR)) as REF_NO, +CAST(OPER_CURR AS VARCHAR2(3 CHAR)) as OPER_CURR, +CAST(CURR_INFO AS VARCHAR2(255 CHAR)) as CURR_INFO, +CAST(NUM_BIDDERS AS NUMBER(32,0)) as NUM_BIDDERS, +CAST(NUM_BIDS AS NUMBER(32,0)) as NUM_BIDS, +CAST(MIN_RATE_TYPE AS VARCHAR2(20 CHAR)) as MIN_RATE_TYPE, +CAST(MIN_RATE AS NUMBER(20,9)) as MIN_RATE, +CAST(MAX_RATE_TYPE AS VARCHAR2(20 CHAR)) as MAX_RATE_TYPE, +CAST(MAX_RATE AS NUMBER(20,9)) as MAX_RATE, +CAST(IS_ASCENDING AS VARCHAR2(255 CHAR)) as IS_ASCENDING, +CAST(ID AS VARCHAR2(255 CHAR)) as ID, +CAST(BUSINESS_VALUE AS NUMBER(20,9)) as BUSINESS_VALUE, +CAST(BUSINESS_VALUE_TYPE AS VARCHAR2(20 CHAR)) as BUSINESS_VALUE_TYPE, +CAST(REPAID_REF_NO AS VARCHAR2(8 CHAR)) as REPAID_REF_NO, +TO_DATE(TIMESTAMP_SENDER,'YYYY-MM-DD"T"HH24:MI:SS' ) as TIMESTAMP_SENDER, +CAST(REVISION AS NUMBER(38,0)) as REVISION, +CAST(CHECKSUM AS VARCHAR2(4000 CHAR)) as CHECKSUM +FROM +{{ref("m_MOPDB_TOP_T_FULLBID_ARRAY_COMPILED_OU_TOP_FULLBID_ARRAY_COMPILED_SQ")}} \ No newline at end of file diff --git a/dbt/models/rar/_sequences.yml b/dbt/models/rar/_sequences.yml new file mode 100644 index 0000000..0458b67 --- /dev/null +++ b/dbt/models/rar/_sequences.yml @@ -0,0 +1,12 @@ + +version: 2 + +sources: + - name: dw_rar_sequences + schema: dw_rar + tables: + - name: AK_NH_ASSET + - name: ct_rar_sequences + schema: ct_rar + tables: + - name: AK_ERR_LOG \ No newline at end of file diff --git a/dbt/models/rar/lookups/LKP_ASSET_CODE_TYPE_NO.sql b/dbt/models/rar/lookups/LKP_ASSET_CODE_TYPE_NO.sql new file mode 100644 index 0000000..67f5f50 --- /dev/null +++ b/dbt/models/rar/lookups/LKP_ASSET_CODE_TYPE_NO.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized="ephemeral", + tags=["m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET"], + ) +}} +-- In IPC this lookup has policy "Use First Value" +select code_type_no, code_type_id + from + ( + select + code_type_no, + code_type_id, + row_number() over ( + partition by code_type_id order by code_type_no + ) as rn + from {{ source("dw_ref_main", "ASSET_CODE_TYPE") }} + where + a_valid_to = to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ) + where rn = 1 diff --git a/dbt/models/rar/lookups/LKP_MU_ASSET_CODE_MAP_ASSET_FK.sql b/dbt/models/rar/lookups/LKP_MU_ASSET_CODE_MAP_ASSET_FK.sql new file mode 100644 index 0000000..5e7fab9 --- /dev/null +++ b/dbt/models/rar/lookups/LKP_MU_ASSET_CODE_MAP_ASSET_FK.sql @@ -0,0 +1,12 @@ +{{ + config( + materialized="ephemeral", + tags=["m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT"], + ) +}} + +select act.code_type_id, acm.code_value, acm.asset_fk, acm.a_valid_from, acm.a_valid_to +from {{ source("dw_ref_main", "ASSET_CODE_TYPE") }} act, {{ ref("m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET") }} acm +where act.code_type_no = acm.code_type_no_id +and acm.a_valid_to = to_date('31.12.9999', 'DD.MM.YYYY') +and act.a_valid_to = to_date('31.12.9999', 'DD.MM.YYYY') diff --git a/dbt/models/rar/source_qualifiers/m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ.sql b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ.sql new file mode 100644 index 0000000..684e949 --- /dev/null +++ b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ.sql @@ -0,0 +1,42 @@ +{{ + config( + materialized="table", + tags=["m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET", "A_TASK_HISTORY_TARGET_insert"], + alias="MU_ASSET_CODE_MAP_SQ", + schema="DW_RAR", + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('SQ') }}", + ) +}} +select + sf_c2d_ea.isin, + sf_c2d_ea.moody_programme_id as moody_code_value, + sf_c2d_ea.a_workflow_history_key +from + ( + select + isin, + a_workflow_history_key, + case + when + moody_programme_id + in ('NOT_COMPLIANT', 'NOT_RATED', 'NOT_AVAILABLE') + then null + when action_type = 'U' + then moody_programme_id + else null + end as moody_programme_id + from + ( + select + trim(sf_c2d_ea_eligible_asset.isin) as isin, + sf_c2d_ea_eligible_asset.a_workflow_history_key, + trim( + sf_c2d_ea_eligible_asset.moody_programme_id + ) as moody_programme_id, + sf_c2d_ea_eligible_asset.action_type + from + {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET") }} sf_c2d_ea_eligible_asset + where action_type = 'U' + ) + ) sf_c2d_ea diff --git a/dbt/models/rar/source_qualifiers/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ.sql b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ.sql new file mode 100644 index 0000000..390fec5 --- /dev/null +++ b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ.sql @@ -0,0 +1,168 @@ +-- This model creates table DW_RAR.NH_ASSET_TEST_SQ. +-- The model is a part of the mapping m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD. +-- Pre-/post-hook log data into the control tables. +{{ config(materialized='table', + alias = 'NH_ASSET_TEST_SQ', + tags = ['m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_pre_snapshot', 'A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('SQ') }}", + schema = 'DW_RAR') }} + +-- Select the required columns for sourse qualifier from DW_TMP.TE_ASSET. +SELECT {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY, + DW_TMP_TE_ASSET.ASSET_KEY, + DW_TMP_TE_ASSET.IS_SECURED, + DW_TMP_TE_ASSET.CLASSIFICATION, + DW_TMP_TE_ASSET.IS_EXTENDIBLE, + DW_TMP_TE_ASSET.EA_IS_EMERGENCY_COLLATERAL, + DW_TMP_TE_ASSET.TMS_GROUP_PATH, + DW_TMP_TE_ASSET.TMS_GROUP_PATH_NAME, + DW_TMP_TE_ASSET.ISSUE_DATE, + DW_TMP_TE_ASSET.MATURITY_LAST_REDEMPTION_DATE, + DW_TMP_TE_ASSET.TMS_ACTIVE_SINCE, + DW_TMP_TE_ASSET.TMS_ACTIVE_UNTIL, + DW_TMP_TE_ASSET.CURRENCY, + DW_TMP_TE_ASSET.CURRENCY_2, + DW_TMP_TE_ASSET.INTEREST_RATE, + DW_TMP_TE_ASSET.TMS_TYPE, + DW_TMP_TE_ASSET.PERIOD_ID, + DW_TMP_TE_ASSET.PRICING_TYPE, + DW_TMP_TE_ASSET.ASSET_BBG_NAME, + DW_TMP_TE_ASSET.BBG_BM_IDX, + DW_TMP_TE_ASSET.BBG_FLT_BM_MULTIPLIER, + DW_TMP_TE_ASSET.BBG_COLLAT_TYPE, + DW_TMP_TE_ASSET.BBG_MARKET_ISSUE, + DW_TMP_TE_ASSET.BBG_MATURITY_TYPE, + DW_TMP_TE_ASSET.COUPON_TYPE, + DW_TMP_TE_ASSET.POOL_FACTOR, + DW_TMP_TE_ASSET.BBG_FLT_SPREAD, + DW_TMP_TE_ASSET.CSDB_REDEMPTION_TYPE, + DW_TMP_TE_ASSET.CSDB_REDEMPTION_FREQ, + DW_TMP_TE_ASSET.CSDB_REDEMPTION_PRICE, + DW_TMP_TE_ASSET.CSDB_ISSUE_PRICE, + DW_TMP_TE_ASSET.CSDB_STATUS, + DW_TMP_TE_ASSET.UOC_INTEREST_PAYMENT_TYPE, + DW_TMP_TE_ASSET.EA_CSD, + DW_TMP_TE_ASSET.C2D_ASSET_TYPE, + DW_TMP_TE_ASSET.EA_COUPON_TYPE, + DW_TMP_TE_ASSET.EA_REF_MARKET, + DW_TMP_TE_ASSET.EA_CRD_OR_EQUIV, + DW_TMP_TE_ASSET.EA_IS_ABS_LLD, + DW_TMP_TE_ASSET.EA_IS_ABS_LLD_DATE, + DW_TMP_TE_ASSET.EA_ABS_SURVEIL_DATE, + DW_TMP_TE_ASSET.EA_ABS_TYPE, + DW_TMP_TE_ASSET.EA_CEPH_IS_INFLAT, + DW_TMP_TE_ASSET.EA_CEPH_IS_CALL_PUT, + DW_TMP_TE_ASSET.EA_CEPH_IS_SINKABLE, + DW_TMP_TE_ASSET.EA_CEPH_COMMENT, + DW_TMP_TE_ASSET.EA_IS_OUTRIGHT_PURCHASE, + DW_TMP_TE_ASSET.EA_LIQ_CLASS, + DW_TMP_TE_ASSET.EA_DENOMINATION, + DW_TMP_TE_ASSET.EA_NAME, + DW_TMP_TE_ASSET.EA_VALIDATION_STATUS, + DW_TMP_TE_ASSET.EA_NCB_COMMENT, + DW_TMP_TE_ASSET.EA_ISSUANCE_DATE, + DW_TMP_TE_ASSET.C2D_MATURITY_DATE, + DW_TMP_TE_ASSET.MATURITY_DATE, + DW_TMP_TE_ASSET.UOC_BUCKET_SIZE, + DW_TMP_TE_ASSET.IS_ELIGIBLE_VIA_GUARANTOR, + DW_TMP_TE_ASSET.UOC_RESIDUAL_MATURITY, + DW_TMP_TE_ASSET.EA_SENDER_CODE, + DW_TMP_TE_ASSET.EA_NON_OWN_USE_HAIRCUT, + DW_TMP_TE_ASSET.ACCOUNTING_TYPE, + DW_TMP_TE_ASSET.GROUPING_VALUE, + DW_TMP_TE_ASSET.SPREAD_TYPE, + DW_TMP_TE_ASSET.TMS_ELIGIBILITY_TYPE, + DW_TMP_TE_ASSET.REF_AMOUNT_BUCKET, + DW_TMP_TE_ASSET.ISSUANCE_TYPE, + DW_TMP_TE_ASSET.BRANCH6_ID, + DW_TMP_TE_ASSET.BRANCH7_ID, + DW_TMP_TE_ASSET.BRANCH8_ID, + DW_TMP_TE_ASSET.BRANCH9_ID, + TRIM(DW_TMP_TE_ASSET.TMS_NAME) AS TMS_NAME, + DW_TMP_TE_ASSET.CSDB_CSD, + DW_TMP_TE_ASSET.CSDB_SECURITISATION_TYPE, + DW_TMP_TE_ASSET.CFI_CLASSIFICATION, + DW_TMP_TE_ASSET.INSTRUMENT_ESA_95_CLASS, + DW_TMP_TE_ASSET.INSTR_ESA_95_CLASS_VAL_TYPE, + DW_TMP_TE_ASSET.INSTRUMENT_ESA_2010_CLASS, + DW_TMP_TE_ASSET.REDEMPTION_CURRENCY, + DW_TMP_TE_ASSET.CSDB_SUPPL_INFO, + DW_TMP_TE_ASSET.UOC_RESET_PERIOD_MORE_ONE_YEAR, + DW_TMP_TE_ASSET.EA_EXTENSION_OPTION_EXERCISED, + DW_TMP_TE_ASSET.EA_EFFECTIVE_MATURITY_DATE, + DW_TMP_TE_ASSET.EA_COVERED_BOND_OU, + DW_TMP_TE_ASSET.EA_OWN_USE_HAIRCUT, + DW_TMP_TE_ASSET.EA_CQS, + DW_TMP_TE_ASSET.EA_EXTENDIBLE, + DW_TMP_TE_ASSET.EA_EXTENSION_TRIGGER, + DW_TMP_TE_ASSET.EA_EXTENDED_MATURITY_DATE, + DW_TMP_TE_ASSET.EA_ABS_PUBL_SURVEIL_DATE2, + DW_TMP_TE_ASSET.CAP, + DW_TMP_TE_ASSET.REFERENCERATE, + DW_TMP_TE_ASSET.REFERENCERATECOMMENT, + DW_TMP_TE_ASSET.BBG_TICKER, + DW_TMP_TE_ASSET.BBG_STRIP_TYPE, + DW_TMP_TE_ASSET.BBG_MTY_YEARS_TDY, + DW_TMP_TE_ASSET.BBG_EXCHANGE_CODE, + DW_TMP_TE_ASSET.BBG_PRVT_PLACE, + DW_TMP_TE_ASSET.BBG_PUTABLE, + DW_TMP_TE_ASSET.BBG_ISSUER_INDUSTRY, + DW_TMP_TE_ASSET.BBG_INFLATION_LINKED_INDICATOR, + DW_TMP_TE_ASSET.CSDB_IS_PRIVATE_PLACEMENT, + DW_TMP_TE_ASSET.CLASSIFICATION_C2D, + DW_TMP_TE_ASSET.CLASSIFICATION_TMS, + DW_TMP_TE_ASSET.ISSUE_DATE_TMS, + DW_TMP_TE_ASSET.ISSUE_DATE_CSDB, + DW_TMP_TE_ASSET.CURRENCY_TMS, + DW_TMP_TE_ASSET.CURRENCY_MDP, + DW_TMP_TE_ASSET.CURRENCY_CSDB, + DW_TMP_TE_ASSET.COUPON_TYPE_MDP, + DW_TMP_TE_ASSET.COUPON_TYPE_CSDB, + DW_TMP_TE_ASSET.POOL_FACTOR_CEPH, + DW_TMP_TE_ASSET.POOL_FACTOR_CSDB, + DW_TMP_TE_ASSET.MATURITY_DATE_TMS, + DW_TMP_TE_ASSET.MATURITY_DATE_MDP, + DW_TMP_TE_ASSET.CP_PROGRAMME, + DW_TMP_TE_ASSET.C2D_ELA_ABS_TYPE, + DW_TMP_TE_ASSET.C2D_EA_ABS_TYPE, + DW_TMP_TE_ASSET.C2D_EA_MATURITY_DATE, + DW_TMP_TE_ASSET.C2D_UC_MATURITY_DATE, + DW_TMP_TE_ASSET.C2D_UC_ASSET_TYPE, + DW_TMP_TE_ASSET.C2D_EA_ASSET_TYPE, + DW_TMP_TE_ASSET.C2D_ELA_ASSET_TYPE, + DW_TMP_TE_ASSET.C2D_UC_DENOMINATION, + DW_TMP_TE_ASSET.C2D_EA_DENOMINATION, + DW_TMP_TE_ASSET.C2D_UC_OA_ID, + DW_TMP_TE_ASSET.C2D_UC_CONTRACT_ID, + DW_TMP_TE_ASSET.C2D_UC_INSTRMNT_ID, + DW_TMP_TE_ASSET.MDP_CMA_DEFAULTED_DATE, + DW_TMP_TE_ASSET.MDP_CMA_DEFAULTED_STANDARD, + DW_TMP_TE_ASSET.MDP_CMA_QUOTE_CONVENTION, + DW_TMP_TE_ASSET.MDP_CMA_RESTRUCTURING_TYPE, + DW_TMP_TE_ASSET.MDP_CMA_MATURITY_DATE, + DW_TMP_TE_ASSET.MDP_CMA_TENOR, + DW_TMP_TE_ASSET.MDP_CMA_SENIORITY, + DW_TMP_TE_ASSET.MDP_CMA_INSTRUMENT_TYPE, + DW_TMP_TE_ASSET.MDP_CMA_CURRENCY, + DW_TMP_TE_ASSET.C2D_EA_REFERENCE_RATE, + DW_TMP_TE_ASSET.C2D_EA_GREEN_SUSTAINABLE, + DW_TMP_TE_ASSET.TMS_FUTURES_MATURITY_BUCKET, + DW_TMP_TE_ASSET.BBG_FLOATER, + DW_TMP_TE_ASSET.BBG_INDUSTRY_GROUP, + DW_TMP_TE_ASSET.CSDB_ESG1, + DW_TMP_TE_ASSET.CSDB_ESG2, + DW_TMP_TE_ASSET.CSDB_ESG3, + DW_TMP_TE_ASSET.REU_SECURITY_DESCRIPTION, + DW_TMP_TE_ASSET.REU_ASSET_CATEGORY, + DW_TMP_TE_ASSET.REU_ASSET_CATEGORY_DESCRIPTION, + DW_TMP_TE_ASSET.REU_ASSET_TYPE, + DW_TMP_TE_ASSET.REU_ASSET_TYPE_DESCRIPTION, + DW_TMP_TE_ASSET.REU_ASSET_SUBTYPE, + DW_TMP_TE_ASSET.REU_ASSET_SUBTYPE_DESCRIPTION, + DW_TMP_TE_ASSET.REU_ISO_CURRENCY, + DW_TMP_TE_ASSET.BBG_DES_NOTES, + DW_TMP_TE_ASSET.BBG_INDUSTRY_SUBGROUP, + DW_TMP_TE_ASSET.BBG_SECURITY_TYP, + DW_TMP_TE_ASSET.BBG_SECURITY_TYP2 +FROM {{ source("dw_tmp", "TE_ASSET") }} DW_TMP_TE_ASSET \ No newline at end of file diff --git a/dbt/models/rar/source_qualifiers/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ.sql b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ.sql new file mode 100644 index 0000000..1a24d35 --- /dev/null +++ b/dbt/models/rar/source_qualifiers/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ.sql @@ -0,0 +1,112 @@ +-- This model is materialised as a table DW_TMP.TE_ASSET_test_SQ and is a part of the mapping m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD. +{{ + config( + materialized="table", + tags=["m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD",'A_TASK_HISTORY_TARGET_insert'], + alias="TE_ASSET_test_SQ", + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('SQ') }}", + schema="DW_TMP", + ) +}} + +-- Select data from the source table: sf_c2d.EA_ELIGIBLE_ASSET. +-- Join with SF_SDW.EXCHANGE_RATE. +-- Join with DW_REF_MAIN.CURRENCY. +-- Perform calculations. +-- Filter data: keep rows where action type is 'U'. +WITH INIT_TABLE AS ( +SELECT 'ISIN' AS ISIN_CODE_TYPE, + RTRIM(LTRIM(ISIN)) AS ISIN_CODE_VALUE, + A_WORKFLOW_HISTORY_KEY, + EMERGENCY_COLLATERAL, + COUNTRY_OF_LOCATION, + ASSET_TYPE, + COUPON_DEFINITION, + REFERENCE_MARKET, + CRD_OR_EQUIVALENT, + LOAN_LEVEL_DATA, + LOAN_LEVEL_DATA_LAST_DATE, + ABS_PUBL_LAST_SURVEILLANCE, + ABS_TYPE, + CEPH_INFLATION_LINKED, + CEPH_CALLABLE_PUTABLE, + CEPH_SINKABLE, + CEPH_COMMENT, + OUTRIGHT_PURCHASE, + LIQUIDITY_CLASS, + DENOMINATION, + ASSET_NAME, + VALIDATION_STATUS, + NCB_COMMENT, + ISSUANCE_DATE, + MATURITY_DATE, + CASE + WHEN substr(ltrim(rtrim(SENDER_CODE)),1,2) = 'ID' THEN substr(ltrim(rtrim(SENDER_CODE)),3) + ELSE ltrim(rtrim(SENDER_CODE)) + END AS SENDER_CODE, + ABS_DATE_LAST_COUPON, + COUPON_RATE, + COUPON_FREQUENCY, + CASE + WHEN EUR_LEGACY IS NULL THEN 0 + ELSE 1 + END AS v_EUR_LEGACY, + NOMINAL_VAL_OUTSTANDING * 1000000 AS EA_NOMINAL_VALUE_OUTSTAND_LC, + ABS_PUBL_LAST_SURVEILLANCE2, + COVERED_BOND_OWN_USE, + CREDIT_QUALITY_RANK, + EXTENDED_MATURITY_DATE, + EXTENDIBLE, + EXTENSION_OPTION_EXERCISED, + EXTENSION_TRIGGER, + NON_OWN_USE_HAIRCUT* 1000000 AS NON_OWN_USE_HAIRCUT, + OWN_USE_HAIRCUT, + OWN_USE_HAIRCUT * 1000000 AS v_OWN_USE_HAIRCUT, + CASE + WHEN EXTENDED_MATURITY_DATE IS NULL THEN MATURITY_DATE + ELSE + CASE + WHEN EXTENSION_OPTION_EXERCISED IS NULL THEN MATURITY_DATE + ELSE EXTENDED_MATURITY_DATE + END + END AS EA_EFFECTIVE_MATURITY_DATE, + ACTUAL_OUTSTANDING_VALUE, + ACTUAL_OUTSTANDING_VALUE*1000000 AS EA_ACTUAL_VALUE_OUTSTAND_LC, + STRIPS, + REFERENCE_RATE, + GREEN_SUSTAINABLE, + SF_SDW_EXCHANGE_RATE.VALUE AS SF_SDW_EXCHANGE_RATE_VALUE +FROM {{ source("sf_c2d", "EA_ELIGIBLE_ASSET") }} EA +LEFT JOIN SF_SDW.EXCHANGE_RATE SF_SDW_EXCHANGE_RATE ON SF_SDW_EXCHANGE_RATE.CURRENCY = EA.DENOMINATION +LEFT JOIN (SELECT CURRENCY_CODE, EUR_LEGACY + FROM DW_REF_MAIN.CURRENCY + WHERE A_VALID_TO = TO_DATE('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS') AND EUR_LEGACY = 1 ) CURRENCY ON EA.DENOMINATION = CURRENCY.CURRENCY_CODE +WHERE EA.ACTION_TYPE = 'U'), + +-- Select all data from previous subquery. +-- Add the v_EXCHANGE_RATE_VALUE column: value is based on a condition and uses results from previous calculations. +TEMP_TABLE AS ( + SELECT INIT_TABLE.*, + CASE + WHEN DENOMINATION<>'EUR' AND v_EUR_LEGACY IS NULL THEN SF_SDW_EXCHANGE_RATE_VALUE + ELSE NULL + END AS v_EXCHANGE_RATE_VALUE + FROM INIT_TABLE +) + +--Select all data from previous subquery. +-- Add EA_NOMINAL_VALUE_OUTSTAND_EUR and EA_ACTUAL_VALUE_OUTSTAND_EUR columns: value is based on the previous calculations. +SELECT TEMP_TABLE.*, + CASE + WHEN DENOMINATION='EUR' OR v_EUR_LEGACY=1 THEN EA_NOMINAL_VALUE_OUTSTAND_LC + WHEN v_EXCHANGE_RATE_VALUE IS NULL THEN EA_NOMINAL_VALUE_OUTSTAND_LC + ELSE ROUND(EA_NOMINAL_VALUE_OUTSTAND_LC/v_EXCHANGE_RATE_VALUE,2) + END AS EA_NOMINAL_VALUE_OUTSTAND_EUR, + CASE + WHEN DENOMINATION='EUR' OR v_EUR_LEGACY=1 THEN EA_ACTUAL_VALUE_OUTSTAND_LC + WHEN v_EXCHANGE_RATE_VALUE IS NULL THEN EA_ACTUAL_VALUE_OUTSTAND_LC + ELSE ROUND(EA_ACTUAL_VALUE_OUTSTAND_LC/v_EXCHANGE_RATE_VALUE,2) + END AS EA_ACTUAL_VALUE_OUTSTAND_EUR +FROM TEMP_TABLE + diff --git a/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ.sql b/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ.sql new file mode 100644 index 0000000..1203bdc --- /dev/null +++ b/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ.sql @@ -0,0 +1,17 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET'], + schema='sf_c2d', + alias = 'EA_ADDITIONAL_GUARANTOR_TEST_SQ' +) }} + +SELECT + A_KEY AS A_KEY, + A_ETL_LOAD_SET_FK AS A_WORKFLOW_HISTORY_KEY, + A_EA_FK, + CODE_TYPE, + CODE_VALUE, + NAME, + COUNTRY_OF_INCORP, + LEI, + GUAR_ASSESSMENT +FROM {{ source("ou_c2d", "EA_ADDITIONAL_GUARANTOR") }} \ No newline at end of file diff --git a/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ.sql b/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ.sql new file mode 100644 index 0000000..0b4d6e4 --- /dev/null +++ b/dbt/models/rar/source_qualifiers/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ.sql @@ -0,0 +1,220 @@ +-- This model is materialised as a table sf_c2d.EA_ELIGIBLE_ASSET_TEST_SQ and is a part of the mapping m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET. +{{ config(materialized='table', + alias = 'EA_ELIGIBLE_ASSET_TEST_SQ', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET', 'A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('SQ') }}", + schema = 'sf_c2d') }} + +-- Select unique rows from the source table: ou_c2d.EA_ELIGIBLE_ASSET table where ISIN is not null. +-- Join with OU_C2D.EA_HEADER to get data on the creation time. +-- Update action type information based on a condition. +SELECT + A_KEY, + A_ETL_LOAD_SET_FK AS A_WORKFLOW_HISTORY_KEY, + A_EA_HEADER_FK, + ISIN, + ( CASE when ORN like '%EMPTY%' then NULL + else ORN + END)AS ORN, + ASSET_NAME, + DENOMINATION, + COUNTRY_OF_LOCATION, + ASSET_TYPE, + PRICE_QUOTATION, + COUPON_DEFINITION, + COUPON_RATE, + COUPON_FREQUENCY, + ISSUANCE_DATE, + MATURITY_DATE, + NOMINAL_VAL_OUTSTANDING, + ACTUAL_OUTSTANDING_VALUE, + STRIPS, + ISS_GROUP, + ISS_LT_RATING_ID, + ISS_LT_RATING_DATE, + ISS_ST_RATING_ID, + ISS_ST_RATING_DATE, + ISS_CODE_TYPE, + ISS_CODE_VALUE, + ISS_NAME, + ISS_OTHER_NAME, + ISS_RESIDENCE, + ISS_PSE, + ISS_LEI, + GUAR_GROUP, + GUAR_LT_RATING_ID, + GUAR_LT_RATING_DATE, + GUAR_CODE_TYPE, + GUAR_CODE_VALUE, + GUAR_NAME, + GUAR_RESIDENCE, + GUAR_PSE, + GUAR_LEI, + GUAR_ASSESSMENT, + NCB_COMMENT, + REFERENCE_MARKET, + LT_RATING_ID, + LT_RATING_DATE, + LT_RATING_PRG_NAME, + LT_RATING_2_ID, + LT_RATING_2_DATE, + LT_RATING_2_PRG_NAME, + ST_RATING_ID, + ST_RATING_DATE, + ST_RATING_PRG_NAME, + ST_RATING_2_ID, + ST_RATING_2_DATE, + ST_RATING_2_PRG_NAME, + ICAS_RATING, + LOAN_LEVEL_DATA, + LOAN_LEVEL_DATA_LAST_DATE, + ABS_PUBL_LAST_SURVEILLANCE, + ABS_PUBL_LAST_SURVEILLANCE2, + ABS_TYPE, + ABS_DATE_LAST_COUPON, + OUTRIGHT_PURCHASE, + CEPH_INFLATION_LINKED, + CEPH_CALLABLE_PUTABLE, + CEPH_SINKABLE, + CEPH_COMMENT, + EXTENDIBLE, + EXTENSION_TRIGGER, + EXTENDED_MATURITY_DATE, + EXTENSION_OPTION_EXERCISED, + EFFECTIVE_MATURITY_DATE, + CRD_OR_EQUIVALENT, + COVERED_BOND_OWN_USE, + DATE_TIME_OF_ENTRY, + SENDER_CODE, + STATUS, + UPDATED_BY, + DATE_TIME_OF_STATUS, + VALIDATION_STATUS, + LIQUIDITY_CLASS, + CREDIT_QUALITY_RANK, + NON_OWN_USE_HAIRCUT, + OWN_USE_HAIRCUT, + ECB_COMMENT, + WAL, + WAL_REF_DATE, + ( CASE WHEN (ACTION_TYPE in ('U','R') OR ACTION_TYPE IS NULL) AND (MATURITY_DATE DBT_INTERNAL_SOURCE.CODE_VALUE" + ], + merge_update_columns=["a_valid_to"], + schema="DW_RAR", + alias="MU_ASSET_CODE_MAP", + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + tags=["m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET", "A_TASK_HISTORY_TARGET_insert"], + ) +}} +{% set sequence_name = source("dw_rar_sequences", "AK_NH_ASSET") %} +with + lkp_asset_code_type_no as (select * from {{ ref("LKP_ASSET_CODE_TYPE_NO") }} + where code_type_id in ('MOODY_PROGRAM_ID', 'ISIN')), + sq as ( + select + {{ get_workflow_history_key() }} as a_workflow_history_key, + 'EADB' as source, + -- isin_code_type_no as isin_code_type_no, + -- moody_program_id_code_type_no as moody_code_type_no, + isin as isin_code_value, + moody_code_value as moody_code_value + from {{ ref("m_DWH_ASSET_CODE_MU_ASSET_CODE_MAP_SF_C2D_EA_ELIGIBLE_ASSET_SQ") }} + ) +{% if is_incremental() %} + , + lkp_dw_rar_mu_asset_code_map as ( + select + dw_rar_mu_asset_code_map.code_value as code_value, + dw_rar_mu_asset_code_map.source as source, + dw_rar_mu_asset_code_map.asset_fk as asset_fk, + dw_rar_mu_asset_code_map.code_type_no_id as code_type_no_id + from {{ this }} dw_rar_mu_asset_code_map + where + a_valid_to = to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + and code_type_no_id = ( + select code_type_no + from {{ source("dw_ref_main", "ASSET_CODE_TYPE") }} + where + a_valid_to + = to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + and code_type_id = 'MOODY_PROGRAM_ID' + ) + ), + all_assets as ( + select + sq.a_workflow_history_key, + sq.isin_code_value, + sq.moody_code_value, + sq.source, + trg.code_value, + trg.asset_fk, + trg.a_valid_from, -- needed for merge + trg.a_valid_to, + ( + select code_type_no + from lkp_asset_code_type_no + where code_type_id = 'ISIN' + ) as isin_code_type_no, + ( + select code_type_no + from lkp_asset_code_type_no + where code_type_id = 'MOODY_PROGRAM_ID' + ) as moody_code_type_no + from sq + full outer join + ( + select + acm.code_value, acm.asset_fk, acm.a_valid_from, acm.a_valid_to + from {{ source("dw_ref_main", "ASSET_CODE_TYPE") }} act, {{ this }} acm + where + act.code_type_id = 'ISIN' + and act.code_type_no = acm.code_type_no_id + and acm.a_valid_to = to_date('31.12.9999', 'DD.MM.YYYY') + and act.a_valid_to = to_date('31.12.9999', 'DD.MM.YYYY') + ) trg + on sq.isin_code_value = trg.code_value + ), + insert_update as ( + select + all_assets.asset_fk, + all_assets.a_workflow_history_key, + all_assets.a_valid_from, -- needed for merge + all_assets.a_valid_to, + all_assets.source, + all_assets.isin_code_type_no, + all_assets.moody_code_type_no, + all_assets.isin_code_value, + all_assets.moody_code_value, + case + when all_assets.asset_fk is null and all_assets.isin_code_value is not null + then 'I' + when + nvl(lkp_dw_rar_mu_asset_code_map.code_value, '>>NULL<<') + <> nvl(all_assets.moody_code_value, '>>NULL<<') + and lkp_dw_rar_mu_asset_code_map.source <> 'MANU' + then 'U' + -- when sq.isin like 'AT000B0779%' then 'I' --for tests + else 'N' + end ins_upd_flag + from all_assets + left join + lkp_dw_rar_mu_asset_code_map + on all_assets.asset_fk = lkp_dw_rar_mu_asset_code_map.asset_fk + and all_assets.moody_code_type_no + = lkp_dw_rar_mu_asset_code_map.code_type_no_id + ) + select + -- case when ins_upd_flag = 'I' then nextval else asset_fk end + -- asset_fk, -- + -- <- sequence used in this place is not good idea because a new value is + -- generated for every row even thought ins_upd_flag <> 'I' + case + when ins_upd_flag = 'I' + then /* bw_asset_tests_fnc()*/ + {{ sequence_next_value(sequence_name) }} + else asset_fk + end asset_fk, -- <- this is a workaround to above + code_type_no_id, + a_valid_from, + a_valid_to, + a_workflow_history_key, + code_value, + source, + indep_subprocess_fk + from + ( + -- insert for MOODY + -- open new row (insert new version of the asset_fk) + select + asset_fk, + moody_code_type_no as code_type_no_id, + sysdate as a_valid_from, + to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') as a_valid_to, + a_workflow_history_key, + moody_code_value as code_value, + source, + cast(null as number) as indep_subprocess_fk, -- IIF($$Var_m_main_load='Y',null ,setvariable($$Var_m_dwh_subprocess_load_set_key, null)) + ins_upd_flag + from insert_update + where + ins_upd_flag in ('U', 'I') + and nvl(moody_code_value, 'XYZXYZ123123') <> 'XYZXYZ123123' + union all + -- insert for ISIN + select + asset_fk, + isin_code_type_no as code_type_no_id, + sysdate as a_valid_from, + to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') as a_valid_to, + a_workflow_history_key, + isin_code_value as code_value, + source, + cast(null as number), -- IIF($$Var_m_main_load='Y',null ,setvariable($$Var_m_dwh_subprocess_load_set_key, null)) + ins_upd_flag + from insert_update + where + ins_upd_flag = 'I' + and nvl(moody_code_value, 'XYZXYZ123123') <> 'XYZXYZ123123' + union all + -- close old version of the row meaning update + select + asset_fk, + moody_code_type_no, + a_valid_from, + sysdate a_valid_to, + a_workflow_history_key, + moody_code_value as code_value, + source, + cast(null as number) as indep_subprocess_fk, + ins_upd_flag + from insert_update + where ins_upd_flag = 'U' + ) +-- below part is used only if model is executed 1st time and target table doesn't exist +{% else %} + -- initial insert for MOODY + select + {{ sequence_next_value(sequence_name) }} as asset_fk, + ( + select code_type_no + from lkp_asset_code_type_no + where code_type_id = 'MOODY_PROGRAM_ID' + ) as code_type_no_id, + sysdate as a_valid_from, + to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') as a_valid_to, + a_workflow_history_key, + moody_code_value as code_value, + source, + cast(null as number) as indep_subprocess_fk -- IIF($$Var_m_main_load='Y',null ,setvariable($$Var_m_dwh_subprocess_load_set_key, null)) + from sq + where nvl(moody_code_value, 'XYZXYZ123123') <> 'XYZXYZ123123' + union all + -- initial insert for ISIN + select + {{ sequence_next_value(sequence_name) }} as asset_fk, + ( + select code_type_no from lkp_asset_code_type_no where code_type_id = 'ISIN' + ) as code_type_no_id, + sysdate as a_valid_from, + to_date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') as a_valid_to, + a_workflow_history_key, + isin_code_value as code_value, + source, + cast(null as number) as indep_subprocess_fk -- IIF($$Var_m_main_load='Y',null ,setvariable($$Var_m_dwh_subprocess_load_set_key, null)) + from sq + where nvl(moody_code_value, 'XYZXYZ123123') <> 'XYZXYZ123123' +{% endif %} diff --git a/dbt/models/rar/targets/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD.sql b/dbt/models/rar/targets/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD.sql new file mode 100644 index 0000000..0582629 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD.sql @@ -0,0 +1,173 @@ +-- This model creates table DW_RAR.NH_ASSET_TEST. +-- The model is a part of the mapping m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD. +-- Pre-/post-hook log data into the control tables. +{{ config(materialized='table', + alias = 'NH_ASSET_TEST', + tags = ['m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_post_snapshot', 'A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + schema = 'DW_RAR') }} + +-- Select all the columns from snapshot. +-- Rename the dbt default historisation columns to the names used in the current setup. + +SELECT SS.ASSET_KEY, + SS.IS_SECURED, + SS.CLASSIFICATION, + SS.IS_EXTENDIBLE, + SS.EA_IS_EMERGENCY_COLLATERAL, + SS.TMS_GROUP_PATH, + SS.TMS_GROUP_PATH_NAME, + SS.ISSUE_DATE, + SS.MATURITY_LAST_REDEMPTION_DATE, + SS.TMS_ACTIVE_SINCE, + SS.TMS_ACTIVE_UNTIL, + SS.CURRENCY, + SS.CURRENCY_2, + SS.INTEREST_RATE, + SS.TMS_TYPE, + SS.PERIOD_ID, + SS.PRICING_TYPE, + SS.ASSET_BBG_NAME, + SS.BBG_BM_IDX, + SS.BBG_FLT_BM_MULTIPLIER, + SS.BBG_COLLAT_TYPE, + SS.BBG_MARKET_ISSUE, + SS.BBG_MATURITY_TYPE, + SS.COUPON_TYPE, + SS.POOL_FACTOR, + SS.BBG_FLT_SPREAD, + SS.CSDB_REDEMPTION_TYPE, + SS.CSDB_REDEMPTION_FREQ, + SS.CSDB_REDEMPTION_PRICE, + SS.CSDB_ISSUE_PRICE, + SS.CSDB_STATUS, + SS.UOC_INTEREST_PAYMENT_TYPE, + SS.EA_CSD, + SS.C2D_ASSET_TYPE, + SS.EA_COUPON_TYPE, + SS.EA_REF_MARKET, + SS.EA_CRD_OR_EQUIV, + SS.EA_IS_ABS_LLD, + SS.EA_IS_ABS_LLD_DATE, + SS.EA_ABS_SURVEIL_DATE, + SS.EA_ABS_TYPE, + SS.EA_CEPH_IS_INFLAT, + SS.EA_CEPH_IS_CALL_PUT, + SS.EA_CEPH_IS_SINKABLE, + SS.EA_CEPH_COMMENT, + SS.EA_IS_OUTRIGHT_PURCHASE, + SS.EA_LIQ_CLASS, + SS.EA_DENOMINATION, + SS.EA_NAME, + SS.EA_VALIDATION_STATUS, + SS.EA_NCB_COMMENT, + SS.EA_ISSUANCE_DATE, + SS.C2D_MATURITY_DATE, + SS.MATURITY_DATE, + SS.UOC_BUCKET_SIZE, + SS.IS_ELIGIBLE_VIA_GUARANTOR, + SS.UOC_RESIDUAL_MATURITY, + SS.EA_SENDER_CODE, + SS.EA_NON_OWN_USE_HAIRCUT, + SS.ACCOUNTING_TYPE, + SS.GROUPING_VALUE, + SS.SPREAD_TYPE, + SS.TMS_ELIGIBILITY_TYPE, + SS.REF_AMOUNT_BUCKET, + SS.ISSUANCE_TYPE, + SS.BRANCH6_ID, + SS.BRANCH7_ID, + SS.BRANCH8_ID, + SS.BRANCH9_ID, + SS.TMS_NAME, + SS.CSDB_CSD, + SS.CSDB_SECURITISATION_TYPE, + SS.CFI_CLASSIFICATION, + SS.INSTRUMENT_ESA_95_CLASS, + SS.INSTR_ESA_95_CLASS_VAL_TYPE, + SS.INSTRUMENT_ESA_2010_CLASS, + SS.REDEMPTION_CURRENCY, + SS.CSDB_SUPPL_INFO, + SS.CAP, + SS.REFERENCERATE, + SS.REFERENCERATECOMMENT, + SS.BBG_FLOATER, + SS.BBG_INDUSTRY_GROUP, + SS.CSDB_ESG1, + SS.CSDB_ESG2, + SS.CSDB_ESG3, + SS.REU_SECURITY_DESCRIPTION, + SS.REU_ASSET_CATEGORY, + SS.REU_ASSET_CATEGORY_DESCRIPTION, + SS.REU_ASSET_TYPE, + SS.REU_ASSET_TYPE_DESCRIPTION, + SS.REU_ASSET_SUBTYPE, + SS.REU_ASSET_SUBTYPE_DESCRIPTION, + SS.REU_ISO_CURRENCY, + SS.BBG_DES_NOTES, + SS.BBG_INDUSTRY_SUBGROUP, + SS.BBG_SECURITY_TYP, + SS.BBG_SECURITY_TYP2, + SS.UOC_RESET_PERIOD_MORE_ONE_YEAR, + SS.EA_EXTENSION_OPTION_EXERCISED, + SS.EA_EFFECTIVE_MATURITY_DATE, + SS.EA_COVERED_BOND_OU, + SS.EA_OWN_USE_HAIRCUT, + SS.EA_CQS, + SS.EA_EXTENDIBLE, + SS.EA_EXTENSION_TRIGGER, + SS.EA_EXTENDED_MATURITY_DATE, + SS.EA_ABS_PUBL_SURVEIL_DATE2, + SS.BBG_TICKER, + SS.BBG_STRIP_TYPE, + SS.BBG_EXCHANGE_CODE, + SS.BBG_PRVT_PLACE, + SS.BBG_PUTABLE, + SS.BBG_ISSUER_INDUSTRY, + SS.BBG_INFLATION_LINKED_INDICATOR, + SS.CSDB_IS_PRIVATE_PLACEMENT, + SS.CLASSIFICATION_C2D, + SS.CLASSIFICATION_TMS, + SS.ISSUE_DATE_TMS, + SS.ISSUE_DATE_CSDB, + SS.CURRENCY_TMS, + SS.CURRENCY_MDP, + SS.CURRENCY_CSDB, + SS.COUPON_TYPE_MDP, + SS.COUPON_TYPE_CSDB, + SS.POOL_FACTOR_CEPH, + SS.POOL_FACTOR_CSDB, + SS.MATURITY_DATE_TMS, + SS.MATURITY_DATE_MDP, + SS.CP_PROGRAMME, + SS.C2D_ELA_ABS_TYPE, + SS.C2D_EA_ABS_TYPE, + SS.C2D_EA_MATURITY_DATE, + SS.C2D_UC_MATURITY_DATE, + SS.C2D_UC_ASSET_TYPE, + SS.C2D_EA_ASSET_TYPE, + SS.C2D_ELA_ASSET_TYPE, + SS.C2D_UC_DENOMINATION, + SS.C2D_EA_DENOMINATION, + SS.C2D_UC_OA_ID, + SS.C2D_UC_CONTRACT_ID, + SS.C2D_UC_INSTRMNT_ID, + SS.MDP_CMA_DEFAULTED_DATE, + SS.MDP_CMA_DEFAULTED_STANDARD, + SS.MDP_CMA_QUOTE_CONVENTION, + SS.MDP_CMA_RESTRUCTURING_TYPE, + SS.MDP_CMA_MATURITY_DATE, + SS.MDP_CMA_TENOR, + SS.MDP_CMA_SENIORITY, + SS.MDP_CMA_INSTRUMENT_TYPE, + SS.MDP_CMA_CURRENCY, + SS.C2D_EA_REFERENCE_RATE, + SS.C2D_EA_GREEN_SUSTAINABLE, + SS.TMS_FUTURES_MATURITY_BUCKET, + SS.BBG_MTY_YEARS_TDY, + dbt_valid_to AS TO_DATE, + FROM_DATE, + dbt_valid_from AS A_INSERTION_DATE, + dbt_valid_to AS A_CORRECTION_DATE +FROM {{ ref("NH_ASSET_snapshot") }} SS \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET.sql b/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET.sql new file mode 100644 index 0000000..c072051 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET.sql @@ -0,0 +1,703 @@ +-- This model incrementally updates the target table DW_TMP.TE_ASSET_test by adding new rows and updating the already existing ones. +-- Wether the row is new or existing one is determined by ASSET_KEY column. +-- The model is a part of the mapping m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD. +-- Pre-/post-hook log data into the control tables. +-- A_TASK_HISTORY_TARGET_insert tag ensures logging of the target table data into the control tables. +{{ config(materialized='incremental', + incremental_strategy='merge', + unique_key='ASSET_KEY', + tags = ['m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD', 'A_TASK_HISTORY_TARGET_insert'], + schema='DW_TMP', + alias = 'TE_ASSET_test', + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + merge_update_columns=['ASSET_KEY', + 'CLASSIFICATION', + 'EA_IS_EMERGENCY_COLLATERAL', + 'C2D_ASSET_TYPE', + 'EA_CSD', + 'EA_COUPON_TYPE', + 'EA_REF_MARKET', + 'EA_CRD_OR_EQUIV', + 'EA_IS_ABS_LLD', + 'EA_IS_ABS_LLD_DATE', + 'EA_ABS_SURVEIL_DATE', + 'EA_ABS_TYPE', + 'EA_CEPH_IS_INFLAT', + 'EA_CEPH_IS_CALL_PUT', + 'EA_CEPH_IS_SINKABLE', + 'EA_CEPH_COMMENT', + 'EA_IS_OUTRIGHT_PURCHASE', + 'EA_LIQ_CLASS', + 'EA_DENOMINATION', + 'EA_NCB_COMMENT', + 'EA_NAME', + 'EA_VALIDATION_STATUS', + 'EA_ISSUANCE_DATE', + 'C2D_MATURITY_DATE', + 'EA_SENDER_CODE', + 'EA_NON_OWN_USE_HAIRCUT', + 'EA_EXTENSION_OPTION_EXERCISED', + 'EA_EFFECTIVE_MATURITY_DATE', + 'EA_COVERED_BOND_OU', + 'EA_OWN_USE_HAIRCUT', + 'EA_CQS', + 'EA_EXTENDIBLE', + 'EA_EXTENSION_TRIGGER', + 'EA_EXTENDED_MATURITY_DATE', + 'EA_ABS_PUBL_SURVEIL_DATE2', + 'CLASSIFICATION_C2D', + 'C2D_EA_ABS_TYPE', + 'C2D_EA_MATURITY_DATE', + 'C2D_EA_ASSET_TYPE', + 'C2D_EA_DENOMINATION', + 'CURRENCY', + 'C2D_EA_REFERENCE_RATE', + 'C2D_EA_GREEN_SUSTAINABLE'] + ) }} + + +-- Set sequence_name: to be used for the generation of the next sequence value. +{% set sequence_name = source("dw_rar_sequences", "AK_NH_ASSET") %} + +-- Select data from Source Qualifier +-- Implement the GET_ASSET_CODE_MAP_CACHE lookup: +-- - join with DW_REF_MAIN.ASSET_CODE_TYPE +-- - join with DW_RAR.MU_ASSET_CODE_MAP +-- - get the lookup ASSET_FK for the corresponding ISIN +-- Perform some data transformation +-- Filter data: keep rows where ASSET_FK is not null +WITH PASS_THROUGH AS ( +SELECT CASE + WHEN DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK IS NOT NULL THEN DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK + ELSE NULL + END AS ASSET_FK, + 'Marketable' AS CLASSIFICATION, + CASE + WHEN EMERGENCY_COLLATERAL IS NOT NULL THEN + CASE + WHEN REGEXP_LIKE(EMERGENCY_COLLATERAL,'^[0-9]+$') THEN + CASE + WHEN TO_NUMBER(EMERGENCY_COLLATERAL)=0 THEN 'N' + ELSE 'Y' + END + ELSE + CASE + WHEN UPPER(EMERGENCY_COLLATERAL) IN ('N','NO','') THEN 'N' + ELSE 'Y' + END + END + END AS EMERGENCY_COLLATERAL, + COUNTRY_OF_LOCATION, + ASSET_TYPE, + COUPON_DEFINITION, + REFERENCE_MARKET, + CRD_OR_EQUIVALENT, + CASE + WHEN LOAN_LEVEL_DATA IS NOT NULL THEN + CASE + WHEN REGEXP_LIKE(LOAN_LEVEL_DATA,'^[0-9]+$') THEN + CASE + WHEN TO_NUMBER(LOAN_LEVEL_DATA)=0 THEN 'N' + ELSE 'Y' + END + ELSE + CASE + WHEN UPPER(LOAN_LEVEL_DATA) IN ('N','NO','') THEN 'N' + ELSE 'Y' + END + END + END AS LOAN_LEVEL_DATA, + LOAN_LEVEL_DATA_LAST_DATE, + ABS_PUBL_LAST_SURVEILLANCE, + ABS_TYPE, + CASE + WHEN CEPH_INFLATION_LINKED IS NOT NULL THEN + CASE + WHEN REGEXP_LIKE(CEPH_INFLATION_LINKED,'^[0-9]+$') THEN + CASE + WHEN TO_NUMBER(CEPH_INFLATION_LINKED)=0 THEN 'N' + ELSE 'Y' + END + ELSE + CASE + WHEN UPPER(CEPH_INFLATION_LINKED) IN ('N','NO','') THEN 'N' + ELSE 'Y' + END + END + END AS CEPH_INFLATION_LINKED, + CEPH_CALLABLE_PUTABLE, + CASE + WHEN CEPH_SINKABLE IS NOT NULL THEN + CASE + WHEN REGEXP_LIKE(CEPH_SINKABLE,'^[0-9]+$') THEN + CASE + WHEN TO_NUMBER(CEPH_SINKABLE)=0 THEN 'N' + ELSE 'Y' + END + ELSE + CASE + WHEN UPPER(CEPH_SINKABLE) IN ('N','NO','') THEN 'N' + ELSE 'Y' + END + END + END AS CEPH_SINKABLE, + CEPH_COMMENT, + CASE + WHEN OUTRIGHT_PURCHASE IS NOT NULL THEN + CASE + WHEN REGEXP_LIKE(OUTRIGHT_PURCHASE,'^[0-9]+$') THEN + CASE + WHEN TO_NUMBER(OUTRIGHT_PURCHASE)=0 THEN 'N' + ELSE 'Y' + END + ELSE + CASE + WHEN UPPER(OUTRIGHT_PURCHASE) IN ('N','NO','') THEN 'N' + ELSE 'Y' + END + END + END AS OUTRIGHT_PURCHASE, + LIQUIDITY_CLASS, + DENOMINATION, + ASSET_NAME, + VALIDATION_STATUS, + NCB_COMMENT, + ISSUANCE_DATE, + MATURITY_DATE, + SENDER_CODE, + ABS_PUBL_LAST_SURVEILLANCE2, + COVERED_BOND_OWN_USE, + CREDIT_QUALITY_RANK, + EXTENDED_MATURITY_DATE, + EXTENDIBLE, + EXTENSION_OPTION_EXERCISED, + EXTENSION_TRIGGER, + NON_OWN_USE_HAIRCUT, + OWN_USE_HAIRCUT, + EA_EFFECTIVE_MATURITY_DATE, + REFERENCE_RATE, + GREEN_SUSTAINABLE, + ROW_NUMBER() OVER (PARTITION BY ASSET_FK ORDER BY ASSET_FK) AS rn +FROM {{ ref("m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ") }} SQ +LEFT JOIN (SELECT CODE_TYPE_NO, CODE_TYPE_ID + FROM DW_REF_MAIN.ASSET_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) DW_REF_ASSET_CODE_TYPE + ON DW_REF_ASSET_CODE_TYPE.CODE_TYPE_ID = UPPER(RTRIM(LTRIM(SQ.ISIN_CODE_TYPE))) +LEFT JOIN (SELECT ASSET_FK , CODE_TYPE_NO_ID, CODE_VALUE + FROM DW_RAR.MU_ASSET_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) DW_RAR_MU_ASSET_CODE_MAP + ON DW_RAR_MU_ASSET_CODE_MAP.CODE_TYPE_NO_ID = DW_REF_ASSET_CODE_TYPE.CODE_TYPE_NO + AND DW_RAR_MU_ASSET_CODE_MAP.CODE_VALUE = RTRIM(LTRIM(SQ.ISIN_CODE_VALUE)) +WHERE DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK IS NOT NULL +), + +-- Select data from the previous subquery. +-- Join with the target table DW_TMP.TE_ASSET_test. +-- Update information in columns based on the current target table data and the incoming values. +-- Remove duplicate rows. +CHECK_GET_ASSET_KEY AS ( +SELECT inp.ASSET_FK, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EMERGENCY_COLLATERAL + ELSE + CASE + WHEN lkp.EA_IS_EMERGENCY_COLLATERAL IS NOT NULL THEN lkp.EA_IS_EMERGENCY_COLLATERAL + ELSE inp.EMERGENCY_COLLATERAL + END + END AS CLASSIFICATION, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EMERGENCY_COLLATERAL + ELSE + CASE + WHEN lkp.EA_IS_EMERGENCY_COLLATERAL IS NOT NULL THEN lkp.EA_IS_EMERGENCY_COLLATERAL + ELSE inp.EMERGENCY_COLLATERAL + END + END AS EA_IS_EMERGENCY_COLLATERAL, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ASSET_TYPE + ELSE + CASE + WHEN lkp.C2D_ASSET_TYPE IS NOT NULL THEN lkp.C2D_ASSET_TYPE + ELSE inp.ASSET_TYPE + END + END AS C2D_ASSET_TYPE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.COUNTRY_OF_LOCATION + ELSE + CASE + WHEN lkp.EA_CSD IS NOT NULL THEN lkp.EA_CSD + ELSE inp.COUNTRY_OF_LOCATION + END + END AS EA_CSD, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.COUPON_DEFINITION + ELSE + CASE + WHEN lkp.EA_COUPON_TYPE IS NOT NULL THEN lkp.EA_COUPON_TYPE + ELSE inp.COUPON_DEFINITION + END + END AS EA_COUPON_TYPE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.REFERENCE_MARKET + ELSE + CASE + WHEN lkp.EA_REF_MARKET IS NOT NULL THEN lkp.EA_REF_MARKET + ELSE inp.REFERENCE_MARKET + END + END AS EA_REF_MARKET, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CRD_OR_EQUIVALENT + ELSE + CASE + WHEN lkp.EA_CRD_OR_EQUIV IS NOT NULL THEN lkp.EA_CRD_OR_EQUIV + ELSE inp.CRD_OR_EQUIVALENT + END + END AS EA_CRD_OR_EQUIV, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.LOAN_LEVEL_DATA + ELSE + CASE + WHEN lkp.EA_IS_ABS_LLD IS NOT NULL THEN lkp.EA_IS_ABS_LLD + ELSE inp.LOAN_LEVEL_DATA + END + END AS EA_IS_ABS_LLD, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.LOAN_LEVEL_DATA_LAST_DATE + ELSE + CASE + WHEN lkp.EA_IS_ABS_LLD_DATE IS NOT NULL THEN lkp.EA_IS_ABS_LLD_DATE + ELSE inp.LOAN_LEVEL_DATA_LAST_DATE + END + END AS EA_IS_ABS_LLD_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ABS_PUBL_LAST_SURVEILLANCE + ELSE + CASE + WHEN lkp.EA_ABS_SURVEIL_DATE IS NOT NULL THEN lkp.EA_ABS_SURVEIL_DATE + ELSE inp.ABS_PUBL_LAST_SURVEILLANCE + END + END AS EA_ABS_SURVEIL_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ABS_TYPE + ELSE + CASE + WHEN lkp.EA_ABS_TYPE IS NOT NULL THEN lkp.EA_ABS_TYPE + ELSE inp.ABS_TYPE + END + END AS EA_ABS_TYPE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CEPH_INFLATION_LINKED + ELSE + CASE + WHEN lkp.EA_CEPH_IS_INFLAT IS NOT NULL THEN lkp.EA_CEPH_IS_INFLAT + ELSE inp.CEPH_INFLATION_LINKED + END + END AS EA_CEPH_IS_INFLAT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CEPH_CALLABLE_PUTABLE + ELSE + CASE + WHEN lkp.EA_CEPH_IS_CALL_PUT IS NOT NULL THEN lkp.EA_CEPH_IS_CALL_PUT + ELSE inp.CEPH_CALLABLE_PUTABLE + END + END AS EA_CEPH_IS_CALL_PUT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CEPH_SINKABLE + ELSE + CASE + WHEN lkp.EA_CEPH_IS_SINKABLE IS NOT NULL THEN lkp.EA_CEPH_IS_SINKABLE + ELSE inp.CEPH_SINKABLE + END + END AS EA_CEPH_IS_SINKABLE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CEPH_COMMENT + ELSE + CASE + WHEN lkp.EA_CEPH_COMMENT IS NOT NULL THEN lkp.EA_CEPH_COMMENT + ELSE inp.CEPH_COMMENT + END + END AS EA_CEPH_COMMENT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.OUTRIGHT_PURCHASE + ELSE + CASE + WHEN lkp.EA_IS_OUTRIGHT_PURCHASE IS NOT NULL THEN lkp.EA_IS_OUTRIGHT_PURCHASE + ELSE inp.OUTRIGHT_PURCHASE + END + END AS EA_IS_OUTRIGHT_PURCHASE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.LIQUIDITY_CLASS + ELSE + CASE + WHEN lkp.EA_LIQ_CLASS IS NOT NULL THEN lkp.EA_LIQ_CLASS + ELSE inp.LIQUIDITY_CLASS + END + END AS EA_LIQ_CLASS, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.DENOMINATION + ELSE + CASE + WHEN lkp.EA_DENOMINATION IS NOT NULL THEN lkp.EA_DENOMINATION + ELSE inp.DENOMINATION + END + END AS EA_DENOMINATION, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.NCB_COMMENT + ELSE + CASE + WHEN lkp.EA_NCB_COMMENT IS NOT NULL THEN lkp.EA_NCB_COMMENT + ELSE inp.NCB_COMMENT + END + END AS EA_NCB_COMMENT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ASSET_NAME + ELSE + CASE + WHEN lkp.EA_NAME IS NOT NULL THEN lkp.EA_NAME + ELSE inp.ASSET_NAME + END + END AS EA_NAME, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.VALIDATION_STATUS + ELSE + CASE + WHEN lkp.EA_VALIDATION_STATUS IS NOT NULL THEN lkp.EA_VALIDATION_STATUS + ELSE inp.VALIDATION_STATUS + END + END AS EA_VALIDATION_STATUS, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ISSUANCE_DATE + ELSE + CASE + WHEN lkp.EA_ISSUANCE_DATE IS NOT NULL THEN lkp.EA_ISSUANCE_DATE + ELSE inp.ISSUANCE_DATE + END + END AS EA_ISSUANCE_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.MATURITY_DATE + ELSE + CASE + WHEN lkp.C2D_MATURITY_DATE IS NOT NULL THEN lkp.C2D_MATURITY_DATE + ELSE inp.MATURITY_DATE + END + END AS C2D_MATURITY_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.SENDER_CODE + ELSE + CASE + WHEN lkp.EA_SENDER_CODE IS NOT NULL THEN lkp.EA_SENDER_CODE + ELSE inp.SENDER_CODE + END + END AS EA_SENDER_CODE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.NON_OWN_USE_HAIRCUT + ELSE + CASE + WHEN lkp.EA_NON_OWN_USE_HAIRCUT IS NOT NULL THEN lkp.EA_NON_OWN_USE_HAIRCUT + ELSE inp.NON_OWN_USE_HAIRCUT + END + END AS EA_NON_OWN_USE_HAIRCUT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EXTENSION_OPTION_EXERCISED + ELSE + CASE + WHEN lkp.EA_EXTENSION_OPTION_EXERCISED IS NOT NULL THEN lkp.EA_EXTENSION_OPTION_EXERCISED + ELSE inp.EXTENSION_OPTION_EXERCISED + END + END AS EA_EXTENSION_OPTION_EXERCISED, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EA_EFFECTIVE_MATURITY_DATE + ELSE + CASE + WHEN lkp.EA_EFFECTIVE_MATURITY_DATE IS NOT NULL THEN lkp.EA_EFFECTIVE_MATURITY_DATE + ELSE inp.EA_EFFECTIVE_MATURITY_DATE + END + END AS EA_EFFECTIVE_MATURITY_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.COVERED_BOND_OWN_USE + ELSE + CASE + WHEN lkp.EA_COVERED_BOND_OU IS NOT NULL THEN lkp.EA_COVERED_BOND_OU + ELSE inp.COVERED_BOND_OWN_USE + END + END AS EA_COVERED_BOND_OU, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.OWN_USE_HAIRCUT + ELSE + CASE + WHEN lkp.EA_OWN_USE_HAIRCUT IS NOT NULL THEN lkp.EA_OWN_USE_HAIRCUT + ELSE inp.OWN_USE_HAIRCUT + END + END AS EA_OWN_USE_HAIRCUT, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CREDIT_QUALITY_RANK + ELSE + CASE + WHEN lkp.EA_CQS IS NOT NULL THEN lkp.EA_CQS + ELSE inp.CREDIT_QUALITY_RANK + END + END AS EA_CQS, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EXTENDIBLE + ELSE + CASE + WHEN lkp.EA_EXTENDIBLE IS NOT NULL THEN lkp.EA_EXTENDIBLE + ELSE inp.EXTENDIBLE + END + END AS EA_EXTENDIBLE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EXTENSION_TRIGGER + ELSE + CASE + WHEN lkp.EA_EXTENSION_TRIGGER IS NOT NULL THEN lkp.EA_EXTENSION_TRIGGER + ELSE inp.EXTENSION_TRIGGER + END + END AS EA_EXTENSION_TRIGGER, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.EXTENDED_MATURITY_DATE + ELSE + CASE + WHEN lkp.EA_EXTENDED_MATURITY_DATE IS NOT NULL THEN lkp.EA_EXTENDED_MATURITY_DATE + ELSE inp.EXTENDED_MATURITY_DATE + END + END AS EA_EXTENDED_MATURITY_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ABS_PUBL_LAST_SURVEILLANCE2 + ELSE + CASE + WHEN lkp.EA_ABS_PUBL_SURVEIL_DATE2 IS NOT NULL THEN lkp.EA_ABS_PUBL_SURVEIL_DATE2 + ELSE inp.ABS_PUBL_LAST_SURVEILLANCE2 + END + END AS EA_ABS_PUBL_SURVEIL_DATE2, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.CLASSIFICATION + ELSE + CASE + WHEN lkp.CLASSIFICATION_C2D IS NOT NULL THEN lkp.CLASSIFICATION_C2D + ELSE inp.CLASSIFICATION + END + END AS CLASSIFICATION_C2D, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ABS_TYPE + ELSE + CASE + WHEN lkp.C2D_EA_ABS_TYPE IS NOT NULL THEN lkp.C2D_EA_ABS_TYPE + ELSE inp.ABS_TYPE + END + END AS C2D_EA_ABS_TYPE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.MATURITY_DATE + ELSE + CASE + WHEN lkp.C2D_EA_MATURITY_DATE IS NOT NULL THEN lkp.C2D_EA_MATURITY_DATE + ELSE inp.MATURITY_DATE + END + END AS C2D_EA_MATURITY_DATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.ASSET_TYPE + ELSE + CASE + WHEN lkp.C2D_EA_ASSET_TYPE IS NOT NULL THEN lkp.C2D_EA_ASSET_TYPE + ELSE inp.ASSET_TYPE + END + END AS C2D_EA_ASSET_TYPE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.DENOMINATION + ELSE + CASE + WHEN lkp.C2D_EA_DENOMINATION IS NOT NULL THEN lkp.C2D_EA_DENOMINATION + ELSE inp.DENOMINATION + END + END AS C2D_EA_DENOMINATION, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.DENOMINATION + ELSE + CASE + WHEN lkp.CURRENCY IS NOT NULL THEN lkp.CURRENCY + ELSE inp.DENOMINATION + END + END AS CURRENCY, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.REFERENCE_RATE + ELSE + CASE + WHEN lkp.C2D_EA_REFERENCE_RATE IS NOT NULL THEN lkp.C2D_EA_REFERENCE_RATE + ELSE inp.REFERENCE_RATE + END + END AS C2D_EA_REFERENCE_RATE, + CASE + WHEN lkp.ASSET_KEY IS NULL THEN inp.GREEN_SUSTAINABLE + ELSE + CASE + WHEN lkp.C2D_EA_GREEN_SUSTAINABLE IS NOT NULL THEN lkp.C2D_EA_GREEN_SUSTAINABLE + ELSE inp.GREEN_SUSTAINABLE + END + END AS C2D_EA_GREEN_SUSTAINABLE +FROM PASS_THROUGH inp +LEFT JOIN DW_TMP.TE_ASSET_test lkp ON inp.ASSET_FK = lkp.ASSET_KEY +WHERE rn=1 +) + +-- Select data from previous subquery. +-- Generate a new ASSET_FK value if null. +-- Create columns with null values corresponding to the target table columns which are not updated anymore. +SELECT CASE WHEN ASSET_FK IS NULL THEN {{ sequence_next_value(sequence_name) }} + ELSE ASSET_FK + END AS ASSET_KEY, + CLASSIFICATION, + EA_IS_EMERGENCY_COLLATERAL, + C2D_ASSET_TYPE, + EA_CSD, + EA_COUPON_TYPE, + EA_REF_MARKET, + EA_CRD_OR_EQUIV, + EA_IS_ABS_LLD, + EA_IS_ABS_LLD_DATE, + EA_ABS_SURVEIL_DATE, + EA_ABS_TYPE, + EA_CEPH_IS_INFLAT, + EA_CEPH_IS_CALL_PUT, + EA_CEPH_IS_SINKABLE, + EA_CEPH_COMMENT, + EA_IS_OUTRIGHT_PURCHASE, + EA_LIQ_CLASS, + EA_DENOMINATION, + EA_NCB_COMMENT, + EA_NAME, + EA_VALIDATION_STATUS, + EA_ISSUANCE_DATE, + C2D_MATURITY_DATE, + EA_SENDER_CODE, + EA_NON_OWN_USE_HAIRCUT, + EA_EXTENSION_OPTION_EXERCISED, + EA_EFFECTIVE_MATURITY_DATE, + EA_COVERED_BOND_OU, + EA_OWN_USE_HAIRCUT, + EA_CQS, + EA_EXTENDIBLE, + EA_EXTENSION_TRIGGER, + EA_EXTENDED_MATURITY_DATE, + EA_ABS_PUBL_SURVEIL_DATE2, + CLASSIFICATION_C2D, + C2D_EA_ABS_TYPE, + C2D_EA_MATURITY_DATE, + C2D_EA_ASSET_TYPE, + C2D_EA_DENOMINATION, + CURRENCY, + C2D_EA_REFERENCE_RATE, + C2D_EA_GREEN_SUSTAINABLE, + CAST(NULL AS VARCHAR2(255)) AS IS_SECURED, + CAST(NULL AS VARCHAR2(255)) AS IS_EXTENDIBLE, + CAST(NULL AS VARCHAR2(255)) AS TMS_GROUP_PATH, + CAST(NULL AS VARCHAR2(255)) AS TMS_GROUP_PATH_NAME, + CAST(NULL AS VARCHAR2(255)) AS ISSUE_DATE, + CAST(NULL AS VARCHAR2(255)) AS MATURITY_LAST_REDEMPTION_DATE, + CAST(NULL AS VARCHAR2(255)) AS TMS_ACTIVE_SINCE, + CAST(NULL AS VARCHAR2(255)) AS TMS_ACTIVE_UNTIL, + CAST(NULL AS VARCHAR2(255)) AS CURRENCY_2, + CAST(NULL AS VARCHAR2(255)) AS INTEREST_RATE, + CAST(NULL AS VARCHAR2(255)) AS TMS_TYPE, + CAST(NULL AS VARCHAR2(255)) AS PERIOD_ID, + CAST(NULL AS VARCHAR2(255)) AS PRICING_TYPE, + CAST(NULL AS VARCHAR2(255)) AS ASSET_BBG_NAME, + CAST(NULL AS VARCHAR2(255)) AS BBG_BM_IDX, + CAST(NULL AS VARCHAR2(255)) AS BBG_FLT_BM_MULTIPLIER, + CAST(NULL AS VARCHAR2(255)) AS BBG_COLLAT_TYPE, + CAST(NULL AS VARCHAR2(255)) AS BBG_MARKET_ISSUE, + CAST(NULL AS VARCHAR2(255)) AS BBG_MATURITY_TYPE, + CAST(NULL AS VARCHAR2(255)) AS COUPON_TYPE, + CAST(NULL AS VARCHAR2(255)) AS POOL_FACTOR, + CAST(NULL AS VARCHAR2(255)) AS BBG_FLT_SPREAD, + CAST(NULL AS VARCHAR2(255)) AS CSDB_REDEMPTION_TYPE, + CAST(NULL AS VARCHAR2(255)) AS CSDB_REDEMPTION_FREQ, + CAST(NULL AS VARCHAR2(255)) AS CSDB_REDEMPTION_PRICE, + CAST(NULL AS VARCHAR2(255)) AS CSDB_ISSUE_PRICE, + CAST(NULL AS VARCHAR2(255)) AS CSDB_STATUS, + CAST(NULL AS VARCHAR2(255)) AS UOC_INTEREST_PAYMENT_TYPE, + CAST(NULL AS VARCHAR2(255)) AS MATURITY_DATE, + CAST(NULL AS VARCHAR2(255)) AS UOC_BUCKET_SIZE, + CAST(NULL AS VARCHAR2(255)) AS IS_ELIGIBLE_VIA_GUARANTOR, + CAST(NULL AS VARCHAR2(255)) AS UOC_RESIDUAL_MATURITY, + CAST(NULL AS VARCHAR2(255)) AS ACCOUNTING_TYPE, + CAST(NULL AS VARCHAR2(255)) AS GROUPING_VALUE, + CAST(NULL AS VARCHAR2(255)) AS SPREAD_TYPE, + CAST(NULL AS VARCHAR2(255)) AS TMS_ELIGIBILITY_TYPE, + CAST(NULL AS VARCHAR2(255)) AS REF_AMOUNT_BUCKET, + CAST(NULL AS VARCHAR2(255)) AS ISSUANCE_TYPE, + CAST(NULL AS VARCHAR2(255)) AS BRANCH6_ID, + CAST(NULL AS VARCHAR2(255)) AS BRANCH7_ID, + CAST(NULL AS VARCHAR2(255)) AS BRANCH8_ID, + CAST(NULL AS VARCHAR2(255)) AS BRANCH9_ID, + CAST(NULL AS VARCHAR2(255)) AS TMS_NAME, + CAST(NULL AS VARCHAR2(255)) AS CSDB_CSD, + CAST(NULL AS VARCHAR2(255)) AS CSDB_SECURITISATION_TYPE, + CAST(NULL AS VARCHAR2(255)) AS CFI_CLASSIFICATION, + CAST(NULL AS VARCHAR2(255)) AS INSTRUMENT_ESA_95_CLASS, + CAST(NULL AS VARCHAR2(255)) AS INSTR_ESA_95_CLASS_VAL_TYPE, + CAST(NULL AS VARCHAR2(255)) AS INSTRUMENT_ESA_2010_CLASS, + CAST(NULL AS VARCHAR2(255)) AS REDEMPTION_CURRENCY, + CAST(NULL AS VARCHAR2(255)) AS CSDB_SUPPL_INFO, + CAST(NULL AS VARCHAR2(255)) AS UOC_RESET_PERIOD_MORE_ONE_YEAR, + CAST(NULL AS VARCHAR2(255)) AS CAP, + CAST(NULL AS VARCHAR2(255)) AS REFERENCERATE, + CAST(NULL AS VARCHAR2(255)) AS REFERENCERATECOMMENT, + CAST(NULL AS VARCHAR2(255)) AS BBG_TICKER, + CAST(NULL AS VARCHAR2(255)) AS BBG_STRIP_TYPE, + CAST(NULL AS VARCHAR2(255)) AS BBG_MTY_YEARS_TDY, + CAST(NULL AS VARCHAR2(255)) AS BBG_EXCHANGE_CODE, + CAST(NULL AS VARCHAR2(255)) AS BBG_PRVT_PLACE, + CAST(NULL AS VARCHAR2(255)) AS BBG_PUTABLE, + CAST(NULL AS VARCHAR2(255)) AS BBG_ISSUER_INDUSTRY, + CAST(NULL AS VARCHAR2(255)) AS BBG_INFLATION_LINKED_INDICATOR, + CAST(NULL AS VARCHAR2(255)) AS CSDB_IS_PRIVATE_PLACEMENT, + CAST(NULL AS VARCHAR2(255)) AS CLASSIFICATION_TMS, + CAST(NULL AS VARCHAR2(255)) AS ISSUE_DATE_TMS, + CAST(NULL AS VARCHAR2(255)) AS ISSUE_DATE_CSDB, + CAST(NULL AS VARCHAR2(255)) AS CURRENCY_TMS, + CAST(NULL AS VARCHAR2(255)) AS CURRENCY_MDP, + CAST(NULL AS VARCHAR2(255)) AS CURRENCY_CSDB, + CAST(NULL AS VARCHAR2(255)) AS COUPON_TYPE_MDP, + CAST(NULL AS VARCHAR2(255)) AS COUPON_TYPE_CSDB, + CAST(NULL AS VARCHAR2(255)) AS POOL_FACTOR_CEPH, + CAST(NULL AS VARCHAR2(255)) AS POOL_FACTOR_CSDB, + CAST(NULL AS VARCHAR2(255)) AS MATURITY_DATE_TMS, + CAST(NULL AS VARCHAR2(255)) AS MATURITY_DATE_MDP, + CAST(NULL AS VARCHAR2(255)) AS CP_PROGRAMME, + CAST(NULL AS VARCHAR2(255)) AS C2D_ELA_ABS_TYPE, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_MATURITY_DATE, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_ASSET_TYPE, + CAST(NULL AS VARCHAR2(255)) AS C2D_ELA_ASSET_TYPE, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_DENOMINATION, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_OA_ID, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_CONTRACT_ID, + CAST(NULL AS VARCHAR2(255)) AS C2D_UC_INSTRMNT_ID, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_DEFAULTED_DATE, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_DEFAULTED_STANDARD, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_QUOTE_CONVENTION, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_RESTRUCTURING_TYPE, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_MATURITY_DATE, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_TENOR, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_SENIORITY, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_INSTRUMENT_TYPE, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_CURRENCY, + CAST(NULL AS VARCHAR2(255)) AS TMS_FUTURES_MATURITY_BUCKET, + CAST(NULL AS VARCHAR2(255)) AS BBG_FLOATER, + CAST(NULL AS VARCHAR2(255)) AS BBG_INDUSTRY_GROUP, + CAST(NULL AS VARCHAR2(255)) AS CSDB_ESG1, + CAST(NULL AS VARCHAR2(255)) AS CSDB_ESG2, + CAST(NULL AS VARCHAR2(255)) AS CSDB_ESG3, + CAST(NULL AS VARCHAR2(255)) AS REU_SECURITY_DESCRIPTION, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_CATEGORY, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_CATEGORY_DESCRIPTION, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_TYPE, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_TYPE_DESCRIPTION, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_SUBTYPE, + CAST(NULL AS VARCHAR2(255)) AS REU_ASSET_SUBTYPE_DESCRIPTION, + CAST(NULL AS VARCHAR2(255)) AS REU_ISO_CURRENCY, + CAST(NULL AS VARCHAR2(255)) AS BBG_DES_NOTES, + CAST(NULL AS VARCHAR2(255)) AS BBG_INDUSTRY_SUBGROUP, + CAST(NULL AS VARCHAR2(255)) AS BBG_SECURITY_TYP, + CAST(NULL AS VARCHAR2(255)) AS BBG_SECURITY_TYP2, + CAST(NULL AS VARCHAR2(255)) AS INDEP_SUBPROCESS_FK +FROM CHECK_GET_ASSET_KEY diff --git a/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_TRANSACTIONAL_DATA.sql b/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_TRANSACTIONAL_DATA.sql new file mode 100644 index 0000000..c3067a6 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_TRANSACTIONAL_DATA.sql @@ -0,0 +1,190 @@ +-- This model incrementally updates the target table DW_TMP.TE_ASSET_TRANSACTIONAL_DATA_test by adding new rows and updating the already existing ones. +-- Wether the row is new or existing one is determined by ASSET_KEY column. +-- The model is a part of the mapping m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD. +-- Pre-/post-hook log data into the control tables. +-- A_TASK_HISTORY_TARGET_insert tag ensures logging of the target table data into the control tables. +{{ config(materialized='incremental', + incremental_strategy='merge', + unique_key='ASSET_FK', + tags = ['m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='DW_TMP', + alias = 'TE_ASSET_TRANSACTIONAL_DATA_test', + merge_update_columns = ['ASSET_FK', + 'C2D_ABS_LAST_COUPON_DATE', + 'C2D_COUPON_RATE', + 'EA_COUPON_FREQ', + 'EA_NOMINAL_VALUE_OUTSTAND_EUR', + 'EA_ACTUAL_VALUE_OUTSTAND_EUR', + 'STRIPS', + 'EA_NOMINAL_VALUE_OUTSTAND_LC', + 'EA_ACTUAL_VALUE_OUTSTAND_LC'] + ) }} + +-- Set sequence_name: to be used for the generation of the next sequence value. +{% set sequence_name = source("dw_rar_sequences", "AK_NH_ASSET") %} + +-- Select data from Source Qualifier +-- Implement the GET_ASSET_CODE_MAP_CACHE lookup: +-- - join with DW_REF_MAIN.ASSET_CODE_TYPE +-- - join with DW_RAR.MU_ASSET_CODE_MAP +-- - get the lookup ASSET_FK for the corresponding ISIN +-- Perform some data transformation +-- Filter data: keep rows where ASSET_FK is not null +WITH PASS_THROUGH AS ( +SELECT CASE + WHEN DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK IS NOT NULL THEN DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK + ELSE NULL + END AS ASSET_FK, + ABS_DATE_LAST_COUPON, + COUPON_RATE, + COUPON_FREQUENCY, + EA_NOMINAL_VALUE_OUTSTAND_EUR, + EA_ACTUAL_VALUE_OUTSTAND_EUR, + STRIPS, + EA_NOMINAL_VALUE_OUTSTAND_LC, + EA_ACTUAL_VALUE_OUTSTAND_LC, + ROW_NUMBER() OVER (PARTITION BY ASSET_FK ORDER BY ASSET_FK) AS rn +FROM {{ ref("m_DWH_ASSET_TE_ASSET_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_TE_ASSET_SQ") }} SQ +LEFT JOIN (SELECT CODE_TYPE_NO, CODE_TYPE_ID + FROM DW_REF_MAIN.ASSET_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) DW_REF_ASSET_CODE_TYPE + ON DW_REF_ASSET_CODE_TYPE.CODE_TYPE_ID = UPPER(RTRIM(LTRIM(SQ.ISIN_CODE_TYPE))) +LEFT JOIN (SELECT ASSET_FK , CODE_TYPE_NO_ID, CODE_VALUE + FROM DW_RAR.MU_ASSET_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) DW_RAR_MU_ASSET_CODE_MAP + ON DW_RAR_MU_ASSET_CODE_MAP.CODE_TYPE_NO_ID = DW_REF_ASSET_CODE_TYPE.CODE_TYPE_NO + AND DW_RAR_MU_ASSET_CODE_MAP.CODE_VALUE = RTRIM(LTRIM(SQ.ISIN_CODE_VALUE)) +WHERE DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK IS NOT NULL +), + +-- Select data from the previous subquery. +-- Join with the target table DW_TMP.TE_ASSET_test. +-- Update information in columns based on the current target table data and the incoming values. +-- Remove duplicate rows. +CHECK_GET_ASSET_KEY AS ( +SELECT inp.ASSET_FK, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.ABS_DATE_LAST_COUPON + ELSE + CASE + WHEN lkp_trans.C2D_ABS_LAST_COUPON_DATE IS NOT NULL THEN lkp_trans.C2D_ABS_LAST_COUPON_DATE + ELSE inp.ABS_DATE_LAST_COUPON + END + END AS C2D_ABS_LAST_COUPON_DATE, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.COUPON_RATE + ELSE + CASE + WHEN lkp_trans.C2D_COUPON_RATE IS NOT NULL THEN lkp_trans.C2D_COUPON_RATE + ELSE inp.COUPON_RATE + END + END AS C2D_COUPON_RATE, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.COUPON_FREQUENCY + ELSE + CASE + WHEN lkp_trans.EA_COUPON_FREQ IS NOT NULL THEN lkp_trans.EA_COUPON_FREQ + ELSE inp.COUPON_FREQUENCY + END + END AS EA_COUPON_FREQ, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.EA_NOMINAL_VALUE_OUTSTAND_EUR + ELSE + CASE + WHEN lkp_trans.EA_NOMINAL_VALUE_OUTSTAND_EUR IS NOT NULL THEN lkp_trans.EA_NOMINAL_VALUE_OUTSTAND_EUR + ELSE inp.EA_NOMINAL_VALUE_OUTSTAND_EUR + END + END AS EA_NOMINAL_VALUE_OUTSTAND_EUR, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.EA_ACTUAL_VALUE_OUTSTAND_EUR + ELSE + CASE + WHEN lkp_trans.EA_ACTUAL_VALUE_OUTSTAND_EUR IS NOT NULL THEN lkp_trans.EA_ACTUAL_VALUE_OUTSTAND_EUR + ELSE inp.EA_ACTUAL_VALUE_OUTSTAND_EUR + END + END AS EA_ACTUAL_VALUE_OUTSTAND_EUR, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.STRIPS + ELSE + CASE + WHEN lkp_trans.STRIPS IS NOT NULL THEN lkp_trans.STRIPS + ELSE inp.STRIPS + END + END AS STRIPS, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.EA_NOMINAL_VALUE_OUTSTAND_LC + ELSE + CASE + WHEN lkp_trans.EA_NOMINAL_VALUE_OUTSTAND_LC IS NOT NULL THEN lkp_trans.EA_NOMINAL_VALUE_OUTSTAND_LC + ELSE inp.EA_NOMINAL_VALUE_OUTSTAND_LC + END + END AS EA_NOMINAL_VALUE_OUTSTAND_LC, + CASE + WHEN lkp_trans.ASSET_FK IS NULL THEN inp.EA_ACTUAL_VALUE_OUTSTAND_LC + ELSE + CASE + WHEN lkp_trans.EA_ACTUAL_VALUE_OUTSTAND_LC IS NOT NULL THEN lkp_trans.EA_ACTUAL_VALUE_OUTSTAND_LC + ELSE inp.EA_ACTUAL_VALUE_OUTSTAND_LC + END + END AS EA_ACTUAL_VALUE_OUTSTAND_LC +FROM PASS_THROUGH inp +LEFT JOIN DW_TMP.TE_ASSET_TRANSACTIONAL_DATA_test lkp_trans ON lkp_trans.ASSET_FK = inp.ASSET_FK +WHERE rn=1 +) + +-- Select data from previous subquery. +-- Generate a new ASSET_FK value if null. +-- Create columns with null values corresponding to the target table columns which are not updated anymore. +SELECT CASE WHEN ASSET_FK IS NULL THEN {{ sequence_next_value(sequence_name) }} + ELSE ASSET_FK + END AS ASSET_FK, + C2D_ABS_LAST_COUPON_DATE, + C2D_COUPON_RATE, + EA_COUPON_FREQ, + EA_NOMINAL_VALUE_OUTSTAND_EUR, + EA_ACTUAL_VALUE_OUTSTAND_EUR, + STRIPS, + EA_NOMINAL_VALUE_OUTSTAND_LC, + EA_ACTUAL_VALUE_OUTSTAND_LC, + CAST(NULL AS VARCHAR2(255)) AS CSDB_LAST_COUPON_RATE, + CAST(NULL AS VARCHAR2(255)) AS CSDB_LAST_COUPON_DATE, + CAST(NULL AS VARCHAR2(255)) AS COUPON_RATE, + CAST(NULL AS VARCHAR2(255)) AS BBG_ISSUE_AMT, + CAST(NULL AS VARCHAR2(255)) AS COUPON_CURRENCY, + CAST(NULL AS VARCHAR2(255)) AS CSDB_ISSUE_AMT, + CAST(NULL AS VARCHAR2(255)) AS DATE_BASIS, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT_LC, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT_EUR, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT2, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT2_EUR, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT2_TYPE, + CAST(NULL AS VARCHAR2(255)) AS COUPON_FREQ, + CAST(NULL AS VARCHAR2(255)) AS WAL, + CAST(NULL AS VARCHAR2(255)) AS WAL_DATE, + CAST(NULL AS VARCHAR2(255)) AS COUPON_RATE_TMS, + CAST(NULL AS VARCHAR2(255)) AS COUPON_RATE_MDP, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT_LC_MDP, + CAST(NULL AS VARCHAR2(255)) AS OUTSTANDING_AMT_LC_CSDB, + CAST(NULL AS VARCHAR2(255)) AS COUPON_FREQ_TMS, + CAST(NULL AS VARCHAR2(255)) AS COUPON_FREQ_CSDB, + CAST(NULL AS VARCHAR2(255)) AS TMS_BISIP_DURATION, + CAST(NULL AS VARCHAR2(255)) AS BBG_MTY_YEARS_TDY, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_COUPON_RATE, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_PAYMENT_FREQUENCY, + CAST(NULL AS VARCHAR2(255)) AS MDP_CMA_DEFAULT_PROBABILITY, + CAST(NULL AS VARCHAR2(255)) AS BBG_CUR_MKT_CAP, + CAST(NULL AS VARCHAR2(255)) AS BBG_YAS_YLD_SPREAD, + CAST(NULL AS VARCHAR2(255)) AS BBG_Z_SPRD_MID, + CAST(NULL AS VARCHAR2(255)) AS BBG_BLP_I_SPRD_MID, + CAST(NULL AS VARCHAR2(255)) AS BBG_MID_G_SPREAD, + CAST(NULL AS VARCHAR2(255)) AS BBG_IDX_RATIO, + CAST(NULL AS VARCHAR2(255)) AS FIXING_RATE, + CAST(NULL AS VARCHAR2(255)) AS SPREAD, + CAST(NULL AS VARCHAR2(255)) AS FLOOR, + CAST(NULL AS VARCHAR2(255)) AS CAP, + CAST(NULL AS VARCHAR2(255)) AS FACTOR, + CAST(NULL AS VARCHAR2(255)) AS BBG_DVD_CRNCY, + CAST(NULL AS VARCHAR2(255)) AS BBG_DVD_PAY_DT, + CAST(NULL AS VARCHAR2(255)) AS INDEP_SUBPROCESS_FK +FROM CHECK_GET_ASSET_KEY diff --git a/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT.sql b/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT.sql new file mode 100644 index 0000000..500064c --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT.sql @@ -0,0 +1,87 @@ +{{ + config( + materialized="incremental", + incremental_strategy="merge", + unique_key=["asset_fk"], + incremental_predicates=[ + "DBT_INTERNAL_SOURCE.UPD_INS = 'U'" + ], + merge_update_columns=["ea_eligibility_status", "limit_amount", "tms_eligibility_type", "isin_with_larger_limit", "indep_subprocess_fk"], + schema="DW_TMP", + alias="TE_ASSET_ELIGIBILITY_LIMIT", + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + tags=["m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT", "A_TASK_HISTORY_TARGET_insert"], + ) +}} + +WITH EA_SQ AS (SELECT 'ISIN' AS CODE_TYPE_ISIN, + TRIM(ISIN) AS CODE_VALUE_ISIN, + STATUS + FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET")}} + WHERE ACTION_TYPE = 'U'), +UMI_SQ AS (SELECT 'TMS' AS CODE_TYPE_TMS, + TRIM(UMI_ID) AS CODE_VALUE_TMS, + OUTSTANDING_SIZE + FROM {{ source("sf_tms", "UMISECURITYINFO") }} ), +GET_ASSET_CODE_MAP AS (SELECT * FROM {{ ref("LKP_MU_ASSET_CODE_MAP_ASSET_FK")}}), +ASSET_ENRICHMENT AS (SELECT DISTINCT DACM.ASSET_FK AS ASSET_FK, + AE.INSTR_REPORT_TMS_ELIGIBILITY_TYPE AS INSTR_REPORT_TMS_ELIGIBILITY_TYPE + FROM {{ source("sf_tms", "ASSET_ENRICHMENT") }} AE + JOIN (SELECT * FROM GET_ASSET_CODE_MAP WHERE CODE_TYPE_ID in ('ISIN','TMS')) DACM + ON AE.TYPE4_TMS=DACM.CODE_VALUE), +ISIN_WITH_LARGER_LIMITS AS (SELECT DISTINCT DACM.ASSET_FK AS ASSET_FK + FROM {{ source("sf_led", "ISIN_WITH_LARGER_LIMITS")}} IWLL + JOIN (SELECT * FROM GET_ASSET_CODE_MAP WHERE CODE_TYPE_ID in ('ISIN','TMS')) DACM + ON IWLL.IDENTIFIER_TYPE=DACM.CODE_TYPE_ID + AND IWLL.IDENTIFIER =DACM.CODE_VALUE), +UMI_FLTR AS (SELECT code_type_tms, code_value_tms, outstanding_size, asset_fk, a_valid_from, a_valid_to + FROM UMI_SQ sq + LEFT JOIN GET_ASSET_CODE_MAP gacm + ON sq.CODE_TYPE_TMS = gacm.code_type_id AND sq.code_value_tms = gacm.code_value + WHERE ASSET_FK IS NOT NULL), +EA_FLTR AS (SELECT code_type_isin, code_value_isin, status, asset_fk, a_valid_from, a_valid_to + FROM EA_SQ sq + LEFT JOIN GET_ASSET_CODE_MAP gacm + ON sq.CODE_TYPE_ISIN = gacm.code_type_id AND sq.code_value_isin = gacm.code_value + WHERE ASSET_FK IS NOT NULL), +EA_UMI AS (SELECT NVL(EA.ASSET_FK, UMI.ASSET_FK) AS ASSET_FK, + STATUS, + OUTSTANDING_SIZE + FROM EA_FLTR EA + FULL OUTER JOIN UMI_FLTR UMI + ON EA.ASSET_FK = UMI.ASSET_FK), +SOURCES AS (SELECT EU.ASSET_FK, + EU.STATUS AS EA_ELIGIBILITY_STATUS, + AE.INSTR_REPORT_TMS_ELIGIBILITY_TYPE AS TMS_ELIGIBILITY_TYPE, + EU.OUTSTANDING_SIZE AS LIMIT_AMOUNT, + CASE WHEN IWILL.ASSET_FK IS NULL THEN 'N' ELSE 'Y' END AS ISIN_WITH_LARGER_LIMIT + FROM EA_UMI EU + LEFT JOIN ASSET_ENRICHMENT AE + ON EU.ASSET_FK = AE.ASSET_FK + LEFT JOIN ISIN_WITH_LARGER_LIMITS IWILL + ON EU.ASSET_FK = IWILL.ASSET_FK), +FINAL_LOOKUP AS (SELECT SRC.EA_ELIGIBILITY_STATUS, + SRC.TMS_ELIGIBILITY_TYPE, + SRC.LIMIT_AMOUNT, + SRC.ISIN_WITH_LARGER_LIMIT, + CASE WHEN AEL.ASSET_FK IS NULL THEN 'I' + WHEN NVL(AEL.EA_ELIGIBILITY_STATUS, '01') = NVL(SRC.EA_ELIGIBILITY_STATUS, '01') AND NVL(AEL.LIMIT_AMOUNT, -999999999) = NVL(SRC.LIMIT_AMOUNT,-999999999) + AND NVL(AEL.TMS_ELIGIBILITY_TYPE, '>>NULL<<') = NVL(SRC.TMS_ELIGIBILITY_TYPE, '>>NULL<<') AND NVL(AEL.ISIN_WITH_LARGER_LIMIT, '>>NULL<<') = NVL(SRC.ISIN_WITH_LARGER_LIMIT,'>>NULL<<') + THEN 'N' ELSE 'U' END AS UPD_INS, + SRC.ASSET_FK, + CAST(NULL AS VARCHAR2(1 CHAR)) AS TMS_PSPP_BLOCKED, + (SELECT MAX(A_TASK_HISTORY_KEY) FROM CT_MRDS.A_TASK_HISTORY WHERE TASK_RUN_ID = '{{ invocation_id}}') AS INDEP_SUBPROCESS_FK + FROM SOURCES SRC + LEFT JOIN {{ this }} AEL + ON SRC.ASSET_FK = AEL.ASSET_FK) +SELECT ASSET_FK, + EA_ELIGIBILITY_STATUS, + TMS_ELIGIBILITY_TYPE, + LIMIT_AMOUNT, + ISIN_WITH_LARGER_LIMIT, + TMS_PSPP_BLOCKED, + INDEP_SUBPROCESS_FK, + UPD_INS +FROM FINAL_LOOKUP +WHERE UPD_INS IN ('I','U') \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT_ERR_LOG.sql b/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT_ERR_LOG.sql new file mode 100644 index 0000000..de81ccf --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT_ERR_LOG.sql @@ -0,0 +1,33 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + schema='CT_MRDS', + alias='ERR_LOG', + tags=["m_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT"], +) }} +{% set sequence_name = source("ct_rar_sequences", "AK_ERR_LOG") %} + +WITH UMI_SQ AS (SELECT A_WORKFLOW_HISTORY_KEY, + 'TMS' AS CODE_TYPE_TMS, + TRIM(UMI_ID) AS CODE_VALUE_TMS, + OUTSTANDING_SIZE + FROM {{ source("sf_tms", "UMISECURITYINFO") }} ) +SELECT {{ sequence_next_value(sequence_name) }} AS ERR_LOG_KEY, + A_WORKFLOW_HISTORY_KEY, -- will be target key + 22 AS ERR_MESSAGE_FK, + 2 AS ERR_SEVERITY_FK, + '{{ var("orchestration_run_id") }}' AS ERR_WORKFLOW_RUN_ID, + '{{ var("workflow_name") }}' as ERR_WORKFLOW_NAME, + 'm_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT' as ERR_MAPPING_NAME, + 'm_DWH_EL_LIMIT_TE_ASSET_ELIGIBILITY_LIMIT_SF_C2D_EA_SF_TMS_UMI_SF_LED_LLMT' as ERR_LOCATION, + 'SF_TMS_UMISECURITYINFO' AS ERR_SOURCE, + 'UMI_ID' AS ERR_KEY_BUSINESS, + 'TMS_CODE_VALUE' AS ERR_KEY_BUSINESS_VALUE, + 'A_WORKFLOW_HISTORY_KEY' AS ERR_KEY_TECHNICAL, + TO_CHAR(A_WORKFLOW_HISTORY_KEY) AS ERR_KEY_TECHNICAL_VALUE, + 'UMI_ID Code Value not found in DW_RAR_MU_ASSET_CODE_MAP. The source row will not be processed.' AS ERR_DESCRIPTION, + (SELECT ERR_MESSAGE FROM CT_RAR.ERR_MESSAGE WHERE ERR_MESSAGE_KEY = 22) AS ERR_MESSAGE, + A_WORKFLOW_HISTORY_KEY AS INDEP_SUBPROCESS_FK -- will be indep subprocess key +FROM UMI_SQ sq +LEFT JOIN {{ ref("LKP_MU_ASSET_CODE_MAP_ASSET_FK")}} acm +ON sq.CODE_TYPE_TMS = acm.code_type_id AND sq.code_value_tms = acm.code_value +WHERE ASSET_FK IS NULL \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR.sql new file mode 100644 index 0000000..5f1519f --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias = 'EA_ADDITIONAL_GUARANTOR_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ADDITIONAL_GUARANTOR_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET.sql new file mode 100644 index 0000000..1550cfe --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET.sql @@ -0,0 +1,141 @@ +-- This model incrementally updates the target table sf_c2d.EA_ELIGIBLE_ASSET_test by adding new rows and updating the already existing ones. +-- Wether the row is new or existing one is determined by ISIN column. +-- The model is a part of the mapping m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET. +-- Pre-/post-hook log data into the control tables. +-- A_TASK_HISTORY_TARGET_insert tag ensures logging of the target table data into the control tables. +{{ + config( + materialized='incremental', + incremental_strategy='merge', + schema = 'sf_c2d', + unique_key='ISIN', + merge_exclude_columns=['EMERGENCY_COLLATERAL','GUAR_OTHER_NAME','ISS_OTHER_NAME'], + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET', 'A_TASK_HISTORY_TARGET_insert'], + alias='EA_ELIGIBLE_ASSET_test', + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}" + ) +}} + +-- Create lookup: +-- join SQ with the target model +-- get the latest DATE_TIME_OF_ENTRY timestamp for each ISIN from the current target table data. +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset_test sf_c2d_ea_eligible_asset + RIGHT JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +-- Join SQ table with the lookup +-- Filter the incoming data: keep rows where ISIN is new or DATE_TIME_OF_ENTRY is later than the lookup one. +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.* +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY ) + +-- Select the filtered data with some naming adjustments. +-- Create columns with null values corresponding to the target table columns which are not updated anymore. +SELECT A_KEY, + A_WORKFLOW_HISTORY_KEY, + ISIN, + ORN, + ASSET_NAME, + DENOMINATION, + COUNTRY_OF_LOCATION, + ASSET_TYPE, + PRICE_QUOTATION, + COUPON_DEFINITION, + COUPON_RATE, + COUPON_FREQUENCY, + ISSUANCE_DATE, + MATURITY_DATE, + NOMINAL_VAL_OUTSTANDING, + ISS_GROUP, + ISS_LT_RATING_ID, + ISS_LT_RATING_DATE, + ISS_ST_RATING_ID, + ISS_ST_RATING_DATE, + ISS_CODE_TYPE, + ISS_CODE_VALUE, + ISS_NAME, + ISS_RESIDENCE, + ISS_PSE, + GUAR_GROUP, + GUAR_LT_RATING_ID, + GUAR_LT_RATING_DATE, + GUAR_CODE_TYPE, + GUAR_CODE_VALUE, + GUAR_NAME, + GUAR_RESIDENCE, + GUAR_PSE, + NCB_COMMENT, + REFERENCE_MARKET, + LT_RATING_ID, + LT_RATING_DATE, + LT_RATING_PRG_NAME, + LT_RATING_2_ID, + LT_RATING_2_DATE, + LT_RATING_2_PRG_NAME, + ST_RATING_ID, + ST_RATING_DATE, + ST_RATING_PRG_NAME, + ST_RATING_2_ID, + ST_RATING_2_DATE, + ST_RATING_2_PRG_NAME, + ICAS_RATING, + OUTRIGHT_PURCHASE, + CRD_OR_EQUIVALENT, + DATE_TIME_OF_ENTRY, + SENDER_CODE, + STATUS, + DATE_TIME_OF_STATUS, + VALIDATION_STATUS, + LIQUIDITY_CLASS, + ECB_COMMENT, + LOAN_LEVEL_DATA, + LOAN_LEVEL_DATA_LAST_DATE, + ABS_PUBL_LAST_SURVEILLANCE, + ABS_TYPE, + ABS_DATE_LAST_COUPON, + CEPH_INFLATION_LINKED, + CEPH_CALLABLE_PUTABLE, + CEPH_SINKABLE, + CEPH_COMMENT, + A_EA_HEADER_FK, + ABS_PUBL_LAST_SURVEILLANCE2, + ACTION_TYPE, + ACTION_DATE, + COVERED_BOND_OWN_USE, + CREDIT_QUALITY_RANK, + EFFECTIVE_MATURITY_DATE, + EXTENDED_MATURITY_DATE, + EXTENDIBLE, + EXTENSION_OPTION_EXERCISED, + EXTENSION_TRIGGER, + GUAR_LEI, + GUAR_ASSESSMENT, + ISS_LEI, + NON_OWN_USE_HAIRCUT, + OWN_USE_HAIRCUT, + WAL, + WAL_REF_DATE, + ACTUAL_OUTSTANDING_VALUE, + STRIPS, + UPDATED_BY, + O_SNAPSHOT_DATE as SNAPSHOT_DATE, + DBRS_PROGRAMME_ID, + FITCH_PROGRAMME_ID, + MOODY_PROGRAMME_ID, + SNP_PROGRAMME_ID, + REFERENCE_RATE, + GREEN_SUSTAINABLE, + TEST_INDICATOR, + MPEC_RIAD_CROSSCHECK, + SCOPE_PROGRAMME_ID, + CREREF_PROGRAMME_ID, + CAST(NULL AS VARCHAR2(255)) AS EMERGENCY_COLLATERAL, + CAST(NULL AS VARCHAR2(255)) AS GUAR_OTHER_NAME, + CAST(NULL AS VARCHAR2(255)) AS ISS_OTHER_NAME +FROM FILTERED_EA_ELIGIBLE_ASSETS \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT.sql new file mode 100644 index 0000000..3872201 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_FOREIGN_EXCHANGE_SWAP_COUNT_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_FOREIGN_EXCHANGE_SWAP_COUNT_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER.sql new file mode 100644 index 0000000..3debe7d --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_INTEREST_RATE_SWAP_PROVIDER_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_INTEREST_RATE_SWAP_PROVIDER_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK.sql new file mode 100644 index 0000000..1c2cd4f --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_ISSUER_ACCOUNT_BANK_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ISSUER_ACCOUNT_BANK_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER.sql new file mode 100644 index 0000000..dab2977 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_LIQUIDITY_SUPPORT_PROVIDER_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_LIQUIDITY_SUPPORT_PROVIDER_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR.sql new file mode 100644 index 0000000..8f2556d --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_ORIGINATOR_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ORIGINATOR_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER.sql new file mode 100644 index 0000000..3ef3d31 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER.sql @@ -0,0 +1,23 @@ +{{ config(materialized='table', + tags = ['m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + schema='sf_c2d', + alias='EA_SERVICE_test' +) }} + +WITH LKP_SF_C2D_EA_ELIGIBLE_ASSET AS ( + SELECT sf_c2d_ea_eligible_asset.ISIN AS LKP_ISIN ,max(sf_c2d_ea_eligible_asset.DATE_TIME_OF_ENTRY) as LKP_DATE_TIME_OF_ENTRY + FROM sf_c2d.ea_eligible_asset sf_c2d_ea_eligible_asset + JOIN {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS ON SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN = sf_c2d_ea_eligible_asset.ISIN + GROUP BY sf_c2d_ea_eligible_asset.ISIN +), + +FILTERED_EA_ELIGIBLE_ASSETS AS ( +SELECT SQ_OU_C2D_ELIGIBLE_ASSETS.A_KEY +FROM {{ ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_ELIGIBLE_ASSET_SQ") }} SQ_OU_C2D_ELIGIBLE_ASSETS +LEFT JOIN LKP_SF_C2D_EA_ELIGIBLE_ASSET ON LKP_SF_C2D_EA_ELIGIBLE_ASSET.LKP_ISIN = SQ_OU_C2D_ELIGIBLE_ASSETS.ISIN +WHERE LKP_ISIN IS NULL OR DATE_TIME_OF_ENTRY > LKP_DATE_TIME_OF_ENTRY) + +SELECT SQ.* +FROM {{ref("m_DWH_PRELOAD_SF_C2D_EA_OU_C2D_EA_SERVICER_SQ")}} SQ +JOIN FILTERED_EA_ELIGIBLE_ASSETS SF_C2D_EA_ELIGIBLE_ASSET ON SF_C2D_EA_ELIGIBLE_ASSET.A_KEY = SQ.A_EA_FK diff --git a/dbt/models/rar/targets/m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql new file mode 100644 index 0000000..853b893 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = "table", + schema = "SF_RIAD", + alias = "ENTTY_IDNTFRS_FLTTND", + tags = ["m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND", "A_TASK_HISTORY_TARGET_insert"], + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}" +) }} + +SELECT + A_KEY, + {{ get_workflow_history_key() }} AS A_WORKFLOW_HISTORY_KEY, + ENTTY_RIAD_ID, + BSNSS_VLD_FRM, + BSNSS_VLD_T, + CNTRY, + DT_BRTH, + DT_CLS, + ENTTY_RIAD_CD, + NM_ENTTY, + ECNMC_ACTVTY, + INSTTTNL_SCTR, + INSTTTNL_SCTR_DTL, + PSTL_BX_CD, + STRT, + PSTL_CD, + CTY, + IS_INCTV, + IS_UCITS_CMPLNT, + INTRNTNL_ORGNSTN_CD, + BLNC_SHT_TTL_DMSTC, + NT_ASSTS_DMSTC, + COLLATERALGROUP, + INSTTTNL_SCTR_CNTRL, + IS_C2D_RLVNT, + IS_RAR_RLVNT, + ARTCL_123_STTS, + CSPP_ASSSSMNT, + ENTRY_DATE, + RIAD, + AVID, + BIC, + LEI, + NCB, + REU, + UCDB_CD, + VAT, + TMS_CD, + BLMBRG_CD, + C2D_CD, + ULTIMATEHEAD_ENTTY_RIAD_CD, + ULTIMATEHEAD_ENTTY_RIAD_ID, + NMBR_EMPLYS, + ENTRPRS_SZ, + TYP_BNKNG_LCNS, + LGL_FRM, + BLNC_SHT_TTL, + BLNC_SHT_TTL_GRP, + CMMNT +FROM + {{ ref( + "m_DWH_PRELOAD_SF_RIAD_ENTTY_IDNTFRS_FLTTND_OU_RIAD_ENTTY_IDNTFRS_FLTTND_SQ" + ) }} \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD.sql b/dbt/models/rar/targets/m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD.sql new file mode 100644 index 0000000..0dfbdf4 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD.sql @@ -0,0 +1,173 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + unique_key='ASSET_FK', + tags = ['m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD','A_TASK_HISTORY_TARGET_insert'], + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + schema='DW_TMP', + alias = 'TE_ASSET_RATING_test' +) }} + +WITH MAPPLET AS ( +SELECT ASSET_FK, + ISIN_CODE_VALUE AS ISIN, + RTRIM(LTRIM(ST_RATING_ID)) AS ST_RATING_ID, + RTRIM(LTRIM(ST_RATING_PRG_NAME)) AS ST_RATING_PRG_NAME, + RTRIM(LTRIM(ST_RATING_2_ID)) AS ST_RATING_2_ID, + RTRIM(LTRIM(ST_RATING_2_PRG_NAME)) AS ST_RATING_2_PRG_NAME, + RTRIM(LTRIM(LT_RATING_ID)) AS LT_RATING_ID, + RTRIM(LTRIM(LT_RATING_PRG_NAME)) AS LT_RATING_PRG_NAME, + RTRIM(LTRIM(LT_RATING_2_ID)) AS LT_RATING_2_ID, + RTRIM(LTRIM(LT_RATING_2_PRG_NAME)) AS LT_RATING_2_PRG_NAME, + ACTION_TYPE +FROM {{ ref("m_DWH_RAT_TE_ASSET_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LOAD_SQ") }} SQ +LEFT JOIN (SELECT CODE_TYPE_NO, CODE_TYPE_ID + FROM DW_REF_MAIN.ASSET_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) DW_REF_ASSET_CODE_TYPE + ON DW_REF_ASSET_CODE_TYPE.CODE_TYPE_ID = UPPER(RTRIM(LTRIM(SQ.ISIN_CODE_TYPE))) +LEFT JOIN (SELECT ASSET_FK , CODE_TYPE_NO_ID, CODE_VALUE + FROM DW_RAR.MU_ASSET_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) DW_RAR_MU_ASSET_CODE_MAP + ON DW_RAR_MU_ASSET_CODE_MAP.CODE_TYPE_NO_ID = DW_REF_ASSET_CODE_TYPE.CODE_TYPE_NO + AND DW_RAR_MU_ASSET_CODE_MAP.CODE_VALUE = RTRIM(LTRIM(SQ.ISIN_CODE_VALUE)) +WHERE ASSET_FK IS NOT NULL +), + +ST_RATING_ID_NOT_NULL AS ( +SELECT ASSET_FK, + ST_RATING_ID, + ST_RATING_PRG_NAME, + CASE + WHEN SUBSTR(ST_RATING_ID,1,2)='SP' THEN 1 + ELSE 0 + END AS v_RATING_TYPE_SP +FROM MAPPLET +WHERE ST_RATING_ID IS NOT NULL AND SUBSTR(ST_RATING_ID,1,1)='S' + AND SUBSTR(ST_RATING_ID,2,1) IN ('A','P') + AND SUBSTR(ST_RATING_ID,3,2) IN ('MY','SP','DB','IA','SC') +), +ST_RATING_ID_NOT_NULL_2 AS ( +SELECT ASSET_FK, + ST_RATING_ID AS VALUE, + ST_RATING_PRG_NAME AS PROGRAMME_NAME, + CASE + WHEN v_RATING_TYPE_SP=1 THEN 'EADB_I_ST_P_'||SUBSTR(ST_RATING_ID,3,2) + ELSE 'EADB_I_ST_'||SUBSTR(ST_RATING_ID,3,2) + END AS RATING_TYPE_FK, + 'C2D_EA' AS SOURCE +FROM ST_RATING_ID_NOT_NULL +), + +ST_RATING_2_ID_NOT_NULL AS ( +SELECT ASSET_FK, + ST_RATING_2_ID, + ST_RATING_2_PRG_NAME, + CASE + WHEN SUBSTR(ST_RATING_2_ID,1,2)='SP' THEN 1 + ELSE 0 + END AS v_RATING_TYPE_SP +FROM MAPPLET +WHERE ST_RATING_2_ID IS NOT NULL + AND SUBSTR(ST_RATING_2_ID,1,1)='S' + AND SUBSTR(ST_RATING_2_ID,2,1) IN ('A','P') + AND SUBSTR(ST_RATING_2_ID,3,2) IN ('MY','SP','DB','IA','SC') +), + +ST_RATING_2_ID_NOT_NULL_2 AS ( +SELECT ASSET_FK, + ST_RATING_2_ID AS VALUE, + ST_RATING_2_PRG_NAME AS PROGRAMME_NAME, + CASE + WHEN v_RATING_TYPE_SP=1 THEN 'EADB_I_ST_P_'||SUBSTR(ST_RATING_2_ID,3,2) + ELSE 'EADB_I_ST_'||SUBSTR(ST_RATING_2_ID,3,2) + END AS RATING_TYPE_FK, + 'C2D_EA' AS SOURCE +FROM ST_RATING_2_ID_NOT_NULL +), + +LT_RATING_ID_NOT_NULL AS ( +SELECT ASSET_FK, + LT_RATING_ID, + LT_RATING_PRG_NAME, + CASE + WHEN SUBSTR(LT_RATING_ID,1,2)='LP' THEN 1 + ELSE 0 + END AS v_RATING_TYPE_LP +FROM MAPPLET +WHERE LT_RATING_ID IS NOT NULL AND SUBSTR(LT_RATING_ID,1,1)='L' + AND SUBSTR(LT_RATING_ID,2,1) IN ('A','P') + AND SUBSTR(ST_RATING_ID,3,2) IN ('MY','SP','DB','IA','SC') +), + +LT_RATING_ID_NOT_NULL_2 AS( +SELECT ASSET_FK, + LT_RATING_ID AS VALUE, + LT_RATING_PRG_NAME AS PROGRAMME_NAME, + CASE + WHEN v_RATING_TYPE_LP=1 THEN 'EADB_I_LT_P_'||SUBSTR(LT_RATING_ID,3,2) + ELSE 'EADB_I_LT_'||SUBSTR(LT_RATING_ID,3,2) + END AS RATING_TYPE_FK, + 'C2D_EA' AS SOURCE +FROM LT_RATING_ID_NOT_NULL +), + +LT_RATING_2_ID_NOT_NULL AS ( +SELECT ASSET_FK, + LT_RATING_2_ID, + LT_RATING_2_PRG_NAME, + CASE + WHEN SUBSTR(LT_RATING_2_ID,1,2)='LP' THEN 1 + ELSE 0 + END AS v_RATING_TYPE_LP +FROM MAPPLET +WHERE LT_RATING_2_ID IS NOT NULL AND SUBSTR(LT_RATING_2_ID,1,1)='L' + AND SUBSTR(LT_RATING_2_ID,2,1) IN ('A','P') + AND SUBSTR(ST_RATING_2_ID,3,2) IN ('MY','SP','DB','IA','SC') +), + +LT_RATING_2_ID_NOT_NULL_2 AS( +SELECT ASSET_FK, + LT_RATING_2_ID AS VALUE, + LT_RATING_2_PRG_NAME AS PROGRAMME_NAME, + CASE + WHEN v_RATING_TYPE_LP=1 THEN 'EADB_I_LT_P_'||SUBSTR(LT_RATING_2_ID,3,2) + ELSE 'EADB_I_LT_'||SUBSTR(LT_RATING_2_ID,3,2) + END AS RATING_TYPE_FK, + 'C2D_EA' AS SOURCE +FROM LT_RATING_2_ID_NOT_NULL +), + +UNION_SQ AS ( +SELECT * FROM ST_RATING_ID_NOT_NULL_2 +UNION ALL +SELECT * FROM ST_RATING_2_ID_NOT_NULL_2 +UNION ALL +SELECT * FROM LT_RATING_ID_NOT_NULL_2 +UNION ALL +SELECT * FROM LT_RATING_2_ID_NOT_NULL_2), + +DEDUPLICATED AS ( +SELECT ASSET_FK, + VALUE, + PROGRAMME_NAME, + RATING_TYPE_FK, + SOURCE, + ROW_NUMBER() OVER (PARTITION BY ASSET_FK, RATING_TYPE_FK ORDER BY ASSET_FK ASC, RATING_TYPE_FK ASC) AS rn +FROM UNION_SQ) + +SELECT ASSET_FK, + VALUE, + PROGRAMME_NAME, + RATING_TYPE_FK, + SOURCE, + CAST(NULL AS VARCHAR2(255)) AS DATE_, + CAST(NULL AS VARCHAR2(255)) AS OTHER_NOTES, + CAST(NULL AS VARCHAR2(255)) AS IS_PRELIMENARY, + CAST(NULL AS VARCHAR2(255)) AS WATCHLIST_VALUE, + CAST(NULL AS VARCHAR2(255)) AS WATCHLIST_DATE, + CAST(NULL AS VARCHAR2(255)) AS OUTLOOK_VALUE, + CAST(NULL AS VARCHAR2(255)) AS OUTLOOK_DATE, + CAST(NULL AS VARCHAR2(255)) AS RATING_ACTION, + CAST(NULL AS VARCHAR2(255)) AS RATING_ACTION_DATE, + CAST(NULL AS VARCHAR2(255)) AS INDEP_SUBPROCESS_FK -- Added to replicate the original target table. +FROM DEDUPLICATED WHERE rn=1 \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_GUAR.sql b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_GUAR.sql new file mode 100644 index 0000000..c5c06c7 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_GUAR.sql @@ -0,0 +1,201 @@ +{{ config( + materialized = "incremental", + incremental_strategy="append", + schema = "DW_TMP", + alias = "TE_INSTITUTION_RATING_GUAR", + tags = ["m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET", "A_TASK_HISTORY_TARGET_insert"], + pre_hook = "{{ control_model_start() }}", + post_hook = "{{ control_model_end('TRG') }}" +) }} + +WITH EVALUATE_VALUE AS ( + --IIF(NOT ISNULL(ISS_CODE_TYPE),ISS_CODE_TYPE,'NCB') + --decode(true,v_ISS_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_ISS_CODE_TYPE) + SELECT + CASE + WHEN SQ.ISS_CODE_TYPE IS NULL THEN 'NCB' + WHEN SQ.ISS_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + ELSE SQ.ISS_CODE_TYPE + END AS ISS_CODE_TYPE, + SQ.ISS_CODE_VALUE, + --IIF(NOT ISNULL(GUAR_CODE_TYPE),GUAR_CODE_TYPE,'NCB') + --decode(true,v_GUAR_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_GUAR_CODE_TYPE) + CASE + WHEN SQ.GUAR_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + --:UDF.UDF_CLEAN_CODE_TYPE(inp_CODE_TYPE) + --iif(ISNULL(inp_CODE_TYPE) OR LENGTH(:UDF.UDF_TRIM(inp_CODE_TYPE)) = 0, 'NCB', UPPER(:UDF.UDF_TRIM(inp_CODE_TYPE))) + --:UDF.UDF_TRIM(inp_CODE_TYPE) + --LTRIM(RTRIM(input_string)) + WHEN SQ.GUAR_CODE_TYPE IS NULL + OR LENGTH(LTRIM(RTRIM(SQ.GUAR_CODE_TYPE))) = 0 THEN 'NCB' + ELSE UPPER(LTRIM(RTRIM(CAST(SQ.GUAR_CODE_TYPE AS VARCHAR2(255))))) + END AS CODE_TYPE, + SQ.GUAR_CODE_VALUE AS CODE_VALUE, + SQ.ISS_LT_RATING_ID, + SQ.ISS_ST_RATING_ID, + SQ.GUAR_LT_RATING_ID, + SQ.A_WORKFLOW_HISTORY_KEY + FROM + {{ source("sf_c2d", "EA_ELIGIBLE_ASSET") }} SQ + WHERE SQ.ACTION_TYPE = 'U' +), + +LKP_CODE_TYPE_NO AS ( + SELECT + EVALUATE_VALUE.CODE_TYPE AS CODE_TYPE_NO, + EVALUATE_VALUE.CODE_TYPE AS GUAR_CODE_TYPE, + EVALUATE_VALUE.CODE_VALUE AS GUAR_CODE_VALUE, + EVALUATE_VALUE.GUAR_LT_RATING_ID, + DW_RAR_MU_INSTITUTION_CODE_MAP.INSTITUTION_FK, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_TYPE_NO_ID AS CODE_TYPE_NO_ID, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_VALUE + FROM EVALUATE_VALUE + LEFT JOIN (SELECT CODE_TYPE_NO, CODE_TYPE_ID + FROM DW_REF_MAIN.INSTITUTION_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) DW_REF_INSTITUTION_CODE_TYPE + ON DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_ID = UPPER(RTRIM(LTRIM(EVALUATE_VALUE.CODE_TYPE))) + LEFT JOIN (SELECT INSTITUTION_FK , CODE_TYPE_NO_ID, CODE_VALUE + FROM DW_RAR.MU_INSTITUTION_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) DW_RAR_MU_INSTITUTION_CODE_MAP + ON DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_TYPE_NO_ID = DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_NO + AND DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_VALUE = RTRIM(LTRIM(EVALUATE_VALUE.CODE_VALUE)) +), + +EXP_GET_INSTITUTION_KEY AS ( + SELECT + --iif(isnull(CODE_VALUE), null, Inp_INSTITUTION_FK) + CASE + WHEN LKP_CODE_TYPE_NO.INSTITUTION_FK IS NULL THEN NULL + ELSE LKP_CODE_TYPE_NO.INSTITUTION_FK + END AS INSTITUTION_FK, + --iif(isnull(CODE_VALUE), null, inp_CODE_TYPE_NO) + CASE + WHEN LKP_CODE_TYPE_NO.CODE_VALUE IS NULL THEN NULL + ELSE LKP_CODE_TYPE_NO.CODE_TYPE_NO + END AS CODE_TYPE_NO, + LKP_CODE_TYPE_NO.CODE_VALUE, + LKP_CODE_TYPE_NO.CODE_TYPE_NO_ID, + LKP_CODE_TYPE_NO.CODE_VALUE, + LKP_CODE_TYPE_NO.GUAR_CODE_TYPE, + LKP_CODE_TYPE_NO.GUAR_CODE_VALUE, + LKP_CODE_TYPE_NO.GUAR_LT_RATING_ID + FROM + LKP_CODE_TYPE_NO +), + +FILL_NULL_INSTITUTION_CODE_KEY AS ( + SELECT + EXP_GET_INSTITUTION_KEY.INSTITUTION_FK, + --EXP_GET_INSTITUTION_KEY.CODE_TYPE_NO, + --EXP_GET_INSTITUTION_KEY.CODE_VALUE, + --EXP_GET_INSTITUTION_KEY.CODE_TYPE_NO_ID, + --EXP_GET_INSTITUTION_KEY.CODE_VALUE1, + EXP_GET_INSTITUTION_KEY.GUAR_CODE_TYPE, + EXP_GET_INSTITUTION_KEY.GUAR_CODE_VALUE, + EXP_GET_INSTITUTION_KEY.GUAR_LT_RATING_ID + FROM + EXP_GET_INSTITUTION_KEY + --not isnull(INSTITUTION_FK) AND NOT ISNULL(GUAR_LT_RATING_ID) + WHERE + EXP_GET_INSTITUTION_KEY.INSTITUTION_FK IS NOT NULL + AND EXP_GET_INSTITUTION_KEY.GUAR_LT_RATING_ID IS NOT NULL +), + +EXP_MAP_GUAR_TARGET AS ( + SELECT + FILL_NULL_INSTITUTION_CODE_KEY.INSTITUTION_FK, + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID AS RATING, + --IIF(SUBSTR(GUAR_LT_RATING_ID,3,2)='SP','EADB_O_LT_G_SP', + --IIF(SUBSTR(GUAR_LT_RATING_ID,3,2)='DB','EADB_O_LT_G_DB', + --IIF(SUBSTR(GUAR_LT_RATING_ID,3,2)='IA','EADB_O_LT_G_IA', + --IIF(SUBSTR(GUAR_LT_RATING_ID,3,2)='MY','EADB_O_LT_G_MY', + --IIF(SUBSTR(GUAR_LT_RATING_ID,3,2)='SC','EADB_O_LT_G_SC',NULL))))) + --rtrim(ltrim(RATING)) + CASE + WHEN ( + SUBSTR( + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID, + 3, + 2 + ) + ) = 'SP' THEN 'EADB_O_LT_G_SP' + WHEN ( + SUBSTR( + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID, + 3, + 2 + ) + ) = 'DP' THEN 'EADB_O_LT_G_DP' + WHEN ( + SUBSTR( + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID, + 3, + 2 + ) + ) = 'IA' THEN 'EADB_O_LT_G_IA' + WHEN ( + SUBSTR( + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID, + 3, + 2 + ) + ) = 'MY' THEN 'EADB_O_LT_G_MY' + WHEN ( + SUBSTR( + FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID, + 3, + 2 + ) + ) = 'SC' THEN 'EADB_O_LT_G_SC' + ELSE CAST(NULL AS VARCHAR2(255)) + END AS RATING_TYPE_FK, + DW_REF_MAIN_RATING.NUMERICAL_EQUIVALENT + FROM + FILL_NULL_INSTITUTION_CODE_KEY + LEFT JOIN DW_REF_MAIN.RATING DW_REF_MAIN_RATING ON DW_REF_MAIN_RATING.RATING_VALUE_CODE = FILL_NULL_INSTITUTION_CODE_KEY.GUAR_LT_RATING_ID + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ), + AGG_MAX_NUM_EQUIVALENT AS ( + SELECT + EXP_MAP_GUAR_TARGET.INSTITUTION_FK, + EXP_MAP_GUAR_TARGET.RATING_TYPE_FK, + EXP_MAP_GUAR_TARGET.NUMERICAL_EQUIVALENT, + EXP_MAP_GUAR_TARGET.RATING + FROM + EXP_MAP_GUAR_TARGET + GROUP BY + EXP_MAP_GUAR_TARGET.INSTITUTION_FK, + EXP_MAP_GUAR_TARGET.RATING_TYPE_FK, + EXP_MAP_GUAR_TARGET.NUMERICAL_EQUIVALENT, + EXP_MAP_GUAR_TARGET.RATING + ORDER BY + EXP_MAP_GUAR_TARGET.INSTITUTION_FK, + EXP_MAP_GUAR_TARGET.RATING_TYPE_FK, + EXP_MAP_GUAR_TARGET.NUMERICAL_EQUIVALENT + ) + + SELECT + AGG_MAX_NUM_EQUIVALENT.INSTITUTION_FK, + AGG_MAX_NUM_EQUIVALENT.RATING_TYPE_FK, + CAST(NULL as varchar2(255)) AS SPECIFIC_PROVIDER_NAME, + AGG_MAX_NUM_EQUIVALENT.RATING AS VALUE, + CAST(NULL as varchar2(255)) AS DATE_, + CAST(NULL as varchar2(255)) AS OTHER_NOTES, + CAST(NULL as varchar2(255)) AS VALUE_PD, + CAST(NULL as varchar2(255)) AS IS_PRELIMINARY, + AGG_MAX_NUM_EQUIVALENT.NUMERICAL_EQUIVALENT, + CAST(NULL as varchar2(255)) AS OUTLOOK_VALUE, + CAST(NULL as varchar2(255)) AS OUTLOOK_DATE, + CAST(NULL as varchar2(255)) AS WATCHLIST_VALUE, + CAST(NULL as varchar2(255)) AS WATCHLIST_DATE, + --SETVARIABLE($$Var_m_Start_StartTime,NULL) + CAST(NULL as date) AS SNAPSHOT_DATE, + CAST(NULL as varchar2(255)) AS RATING_ACTION, + CAST(NULL as varchar2(255)) AS RATING_ACTION_DATE, + 'C2D_EA' AS SOURCE, + CAST(NULL as varchar2(255)) AS RATING_OTHER_TYPE, + --IIF($$Var_m_main_load='Y',null,setvariable($$Var_m_dwh_subprocess_load_set_key, null)), + CAST(NULL AS number) AS INDEP_SUBPROCESS_FK + FROM + AGG_MAX_NUM_EQUIVALENT diff --git a/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LT.sql b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LT.sql new file mode 100644 index 0000000..9b7d4ba --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_LT.sql @@ -0,0 +1,189 @@ +{{ config( + materialized = "incremental", + incremental_strategy="append", + schema = "DW_TMP", + alias = "TE_INSTITUTION_RATING_LT", + tags = ["m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET", "A_TASK_HISTORY_TARGET_insert"], + pre_hook = "{{ control_model_start() }}", + post_hook = "{{ control_model_end('TRG') }}" +) }} + +WITH SOURCE_QUALIFIER AS ( + --IIF(NOT ISNULL(ISS_CODE_TYPE),ISS_CODE_TYPE,'NCB') + --decode(true,v_ISS_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_ISS_CODE_TYPE) + SELECT + CASE + WHEN SQ.ISS_CODE_TYPE IS NULL THEN 'NCB' + WHEN SQ.ISS_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + ELSE SQ.ISS_CODE_TYPE + END AS ISS_CODE_TYPE, + SQ.ISS_CODE_VALUE, + --IIF(NOT ISNULL(GUAR_CODE_TYPE),GUAR_CODE_TYPE,'NCB') + --decode(true,v_GUAR_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_GUAR_CODE_TYPE) + CASE + WHEN SQ.GUAR_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + WHEN SQ.GUAR_CODE_TYPE IS NULL THEN 'NCB' + ELSE SQ.GUAR_CODE_TYPE + END AS GUAR_CODE_TYPE, + SQ.GUAR_CODE_VALUE, + SQ.ISS_LT_RATING_ID, + SQ.ISS_ST_RATING_ID, + SQ.GUAR_LT_RATING_ID, + SQ.A_WORKFLOW_HISTORY_KEY + FROM + {{ source("sf_c2d", "EA_ELIGIBLE_ASSET") }} SQ +WHERE SQ.ACTION_TYPE = 'U' +), + +EVALUATE_VALUE AS ( + SELECT * FROM SOURCE_QUALIFIER + LEFT JOIN ( + SELECT + DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_NO, + DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_ID --FROM DW_REF_INSTITUTION_CODE_TYPE + FROM + DW_REF_MAIN.INSTITUTION_CODE_TYPE DW_REF_INSTITUTION_CODE_TYPE + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ) LKP_CODE_TYPE_NO ON LKP_CODE_TYPE_NO.CODE_TYPE_ID = UPPER(LTRIM(RTRIM(SOURCE_QUALIFIER.ISS_CODE_TYPE))) + LEFT JOIN ( + SELECT + DW_RAR_MU_INSTITUTION_CODE_MAP.INSTITUTION_FK, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_TYPE_NO_ID, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_VALUE --FROM DW_RAR_MU_INSTITUTION_CODE_MAP + FROM + DW_RAR.MU_INSTITUTION_CODE_MAP DW_RAR_MU_INSTITUTION_CODE_MAP + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ) LKP_INSTITUTION_CODE_KEY ON LKP_INSTITUTION_CODE_KEY.CODE_TYPE_NO_ID = LKP_CODE_TYPE_NO.CODE_TYPE_NO + AND LKP_INSTITUTION_CODE_KEY.CODE_VALUE = SOURCE_QUALIFIER.ISS_CODE_VALUE +), + +MPL_GET_INSTITUTION_CODE_KEY AS ( + SELECT + --iif(isnull(CODE_VALUE),null,Inp_INSTITUTION_FK) + CASE + WHEN EVALUATE_VALUE.CODE_VALUE IS NULL THEN CAST(NULL AS VARCHAR2(255)) + ELSE CAST(EVALUATE_VALUE.INSTITUTION_FK AS VARCHAR2(255)) + END AS INSTITUTION_FK, + --iif(isnull(CODE_VALUE),null,inp_CODE_TYPE_NO) + CASE + WHEN EVALUATE_VALUE.CODE_VALUE IS NULL THEN CAST(NULL AS VARCHAR2(255)) + ELSE EVALUATE_VALUE.CODE_TYPE_NO + END AS CODE_TYPE_NO, + --NSTRING udf_clean_CODE_TYPE( inp_CODE_TYPE as string ) + --:UDF.UDF_CLEAN_CODE_TYPE(inp_CODE_TYPE) + --iif(ISNULL(inp_CODE_TYPE) OR LENGTH(:UDF.UDF_TRIM(inp_CODE_TYPE)) = 0, 'NCB', UPPER(:UDF.UDF_TRIM(inp_CODE_TYPE))) + --:UDF.UDF_TRIM(inp_CODE_TYPE) + --LTRIM(RTRIM(input_string)) + CASE + WHEN EVALUATE_VALUE.ISS_CODE_TYPE IS NULL + OR LENGTH(LTRIM(RTRIM(EVALUATE_VALUE.ISS_CODE_TYPE))) = 0 THEN 'NCB' + ELSE CAST( + UPPER(LTRIM(RTRIM(EVALUATE_VALUE.ISS_CODE_TYPE))) AS varchar2(255)) + END AS CODE_TYPE, + EVALUATE_VALUE.CODE_VALUE, + EVALUATE_VALUE.CODE_TYPE_NO_ID, + EVALUATE_VALUE.CODE_VALUE AS CODE_VALUE1, + EVALUATE_VALUE.ISS_CODE_TYPE, + EVALUATE_VALUE.ISS_LT_RATING_ID, + EVALUATE_VALUE.ISS_ST_RATING_ID + FROM + EVALUATE_VALUE +), + +FIL_NULL_INSTITUTION_CODE_KEY AS ( + SELECT + MPL.INSTITUTION_FK, + MPL.ISS_CODE_TYPE, + MPL.CODE_VALUE, + MPL.ISS_LT_RATING_ID, + MPL.ISS_ST_RATING_ID + FROM + MPL_GET_INSTITUTION_CODE_KEY MPL + WHERE + MPL.INSTITUTION_FK IS NOT NULL + AND MPL.ISS_LT_RATING_ID IS NOT NULL +), + +EXP_MAP_LT_TARGET AS ( + SELECT + FILL_NULL.INSTITUTION_FK, + --IIF(SUBSTR(RATING,3,2)='SP','EADB_O_LT_SP', + --IIF(SUBSTR(RATING,3,2)='DB','EADB_O_LT_DB', + --IIF(SUBSTR(RATING,3,2)='IA','EADB_O_LT_IA', + --IIF(SUBSTR(RATING,3,2)='MY','EADB_O_LT_MY', + --IIF(SUBSTR(RATING,3,2)='SC','EADB_O_LT_SC',NULL))))) + CASE + WHEN (SUBSTR(FILL_NULL.ISS_LT_RATING_ID, 3, 2)) = 'SP' THEN 'EADB_O_LT_SP' + WHEN (SUBSTR(FILL_NULL.ISS_LT_RATING_ID, 3, 2)) = 'DB' THEN 'EADB_O_LT_DB' + WHEN (SUBSTR(FILL_NULL.ISS_LT_RATING_ID, 3, 2)) = 'IA' THEN 'EADB_O_LT_IA' + WHEN (SUBSTR(FILL_NULL.ISS_LT_RATING_ID, 3, 2)) = 'MY' THEN 'EADB_O_LT_MY' + WHEN (SUBSTR(FILL_NULL.ISS_LT_RATING_ID, 3, 2)) = 'SC' THEN 'EADB_O_LT_SC' + ELSE CAST(NULL AS VARCHAR2(255)) + END AS RATING_TYPE_FK, + --rtrim(ltrim(RATING)) + RTRIM(LTRIM(FILL_NULL.ISS_LT_RATING_ID)) AS RATING, + 'A' AS DUMMY + FROM + FIL_NULL_INSTITUTION_CODE_KEY FILL_NULL +), + +LKP_MAIN_RATING AS ( + SELECT + EXP_MAP_LT_TARGET.INSTITUTION_FK, + EXP_MAP_LT_TARGET.RATING_TYPE_FK, + EXP_MAP_LT_TARGET.RATING, + DW_REF_MAIN_RATING.NUMERICAL_EQUIVALENT + FROM EXP_MAP_LT_TARGET + LEFT JOIN DW_REF_MAIN.RATING DW_REF_MAIN_RATING ON EXP_MAP_LT_TARGET.RATING = DW_REF_MAIN_RATING.RATING_VALUE_CODE + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') +), + +EXP_GET_RECORD_COUNT_LT AS ( + SELECT + LKP_MAIN_RATING.INSTITUTION_FK, + LKP_MAIN_RATING.RATING_TYPE_FK, + LKP_MAIN_RATING.NUMERICAL_EQUIVALENT, + LKP_MAIN_RATING.RATING, + ---SETVARIABLE($$Var_m_Start_StartTime,NULL) + CAST(NULL as date) AS SNAPSHOT_DATE, + 'C2D_EA' AS SOURCE, + --IIF($$Var_m_main_load='Y',null,setvariable($$Var_m_dwh_subprocess_load_set_key, null)), + CAST(NULL AS number) AS INDEP_SUBPROCESS_FK + FROM + LKP_MAIN_RATING LKP_MAIN_RATING + GROUP BY + LKP_MAIN_RATING.INSTITUTION_FK, + LKP_MAIN_RATING.RATING_TYPE_FK, + LKP_MAIN_RATING.NUMERICAL_EQUIVALENT, + LKP_MAIN_RATING.RATING + ORDER BY + LKP_MAIN_RATING.INSTITUTION_FK, + LKP_MAIN_RATING.RATING_TYPE_FK, + LKP_MAIN_RATING.NUMERICAL_EQUIVALENT +) + +SELECT + EXP_LT_COUNT.INSTITUTION_FK, + EXP_LT_COUNT.RATING_TYPE_FK, + CAST(NULL as varchar2(255)) AS SPECIFIC_PROVIDER_NAME, + EXP_LT_COUNT.RATING AS VALUE, + CAST(NULL as varchar2(255)) AS DATE_, + CAST(NULL as varchar2(255)) AS OTHER_NOTES, + CAST(NULL as varchar2(255)) AS VALUE_PD, + CAST(NULL as varchar2(255)) AS IS_PRELIMINARY, + EXP_LT_COUNT.NUMERICAL_EQUIVALENT, + CAST(NULL as varchar2(255)) AS OUTLOOK_VALUE, + CAST(NULL as varchar2(255)) AS OUTLOOK_DATE, + CAST(NULL as varchar2(255)) AS WATCHLIST_VALUE, + CAST(NULL as varchar2(255)) AS WATCHLIST_DATE, + EXP_LT_COUNT.SNAPSHOT_DATE, + CAST(NULL as varchar2(255)) AS RATING_ACTION, + CAST(NULL as varchar2(255)) AS RATING_ACTION_DATE, + EXP_LT_COUNT.SOURCE, + CAST(NULL as varchar2(255)) AS RATING_OTHER_TYPE, + EXP_LT_COUNT.INDEP_SUBPROCESS_FK +FROM + EXP_GET_RECORD_COUNT_LT EXP_LT_COUNT \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_ST.sql b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_ST.sql new file mode 100644 index 0000000..f6daea4 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET_ST.sql @@ -0,0 +1,184 @@ +{{ config( + materialized = "incremental", + incremental_strategy="append", + schema = "DW_TMP", + alias = "TE_INSTITUTION_RATING_ST", + tags = ["m_DWH_RAT_TE_INST_RATING_SF_C2D_EA_ELIGIBLE_ASSET", "A_TASK_HISTORY_TARGET_insert"], + pre_hook = "{{ control_model_start() }}", + post_hook = "{{ control_model_end('TRG') }}" +) }} + +WITH SOURCE_QUALIFIER AS ( + --IIF(NOT ISNULL(ISS_CODE_TYPE),ISS_CODE_TYPE,'NCB') + --decode(true,v_ISS_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_ISS_CODE_TYPE) + SELECT + CASE + WHEN SQ.ISS_CODE_TYPE IS NULL THEN 'NCB' + WHEN SQ.ISS_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + ELSE SQ.ISS_CODE_TYPE + END AS ISS_CODE_TYPE, + SQ.ISS_CODE_VALUE, + --IIF(NOT ISNULL(GUAR_CODE_TYPE),GUAR_CODE_TYPE,'NCB') + --decode(true,v_GUAR_CODE_TYPE='RIAD','RIAD_ID_FROM_C2D',v_GUAR_CODE_TYPE) + CASE + WHEN SQ.GUAR_CODE_TYPE = 'RIAD' THEN 'RIAD_ID_FROM_C2D' + WHEN SQ.GUAR_CODE_TYPE IS NULL THEN 'NCB' + ELSE SQ.GUAR_CODE_TYPE + END AS GUAR_CODE_TYPE, + SQ.GUAR_CODE_VALUE, + SQ.ISS_LT_RATING_ID, + SQ.ISS_ST_RATING_ID, + SQ.GUAR_LT_RATING_ID, + SQ.A_WORKFLOW_HISTORY_KEY + FROM + {{ source("sf_c2d", "EA_ELIGIBLE_ASSET") }} SQ + WHERE SQ.ACTION_TYPE = 'U' +), + +EVALUATE_VALUE AS ( + SELECT * FROM SOURCE_QUALIFIER + LEFT JOIN ( + SELECT + DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_NO, + DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_ID --FROM DW_REF_INSTITUTION_CODE_TYPE + FROM + DW_REF_MAIN.INSTITUTION_CODE_TYPE DW_REF_INSTITUTION_CODE_TYPE + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ) LKP_CODE_TYPE_NO ON LKP_CODE_TYPE_NO.CODE_TYPE_ID = UPPER(LTRIM(RTRIM(SOURCE_QUALIFIER.ISS_CODE_TYPE))) + LEFT JOIN ( + SELECT + DW_RAR_MU_INSTITUTION_CODE_MAP.INSTITUTION_FK, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_TYPE_NO_ID, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_VALUE --FROM DW_RAR_MU_INSTITUTION_CODE_MAP + FROM + DW_RAR.MU_INSTITUTION_CODE_MAP DW_RAR_MU_INSTITUTION_CODE_MAP + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') + ) LKP_INSTITUTION_CODE_KEY ON LKP_INSTITUTION_CODE_KEY.CODE_TYPE_NO_ID = LKP_CODE_TYPE_NO.CODE_TYPE_NO + AND LKP_INSTITUTION_CODE_KEY.CODE_VALUE = SOURCE_QUALIFIER.ISS_CODE_VALUE +), + +MPL_GET_INSTITUTION_CODE_KEY AS ( + SELECT + --iif(isnull(CODE_VALUE),null,Inp_INSTITUTION_FK) + CASE + WHEN EVALUATE_VALUE.CODE_VALUE IS NULL THEN CAST(NULL AS VARCHAR2(255)) + ELSE CAST(EVALUATE_VALUE.INSTITUTION_FK AS VARCHAR2(255)) + END AS INSTITUTION_FK, + --iif(isnull(CODE_VALUE),null,inp_CODE_TYPE_NO) + CASE + WHEN EVALUATE_VALUE.CODE_VALUE IS NULL THEN CAST(NULL AS VARCHAR2(255)) + ELSE EVALUATE_VALUE.CODE_TYPE_NO + END AS CODE_TYPE_NO, + --NSTRING udf_clean_CODE_TYPE( inp_CODE_TYPE as string ) + --:UDF.UDF_CLEAN_CODE_TYPE(inp_CODE_TYPE) + --iif(ISNULL(inp_CODE_TYPE) OR LENGTH(:UDF.UDF_TRIM(inp_CODE_TYPE)) = 0, 'NCB', UPPER(:UDF.UDF_TRIM(inp_CODE_TYPE))) + --:UDF.UDF_TRIM(inp_CODE_TYPE) + --LTRIM(RTRIM(input_string)) + CASE + WHEN EVALUATE_VALUE.ISS_CODE_TYPE IS NULL + OR LENGTH(LTRIM(RTRIM(EVALUATE_VALUE.ISS_CODE_TYPE))) = 0 THEN 'NCB' + ELSE CAST( + UPPER(LTRIM(RTRIM(EVALUATE_VALUE.ISS_CODE_TYPE))) AS varchar2(255)) + END AS GUAR_CODE_TYPE, + EVALUATE_VALUE.CODE_VALUE, + EVALUATE_VALUE.CODE_TYPE_NO_ID, + EVALUATE_VALUE.CODE_VALUE AS CODE_VALUE1, + EVALUATE_VALUE.ISS_CODE_TYPE, + EVALUATE_VALUE.ISS_LT_RATING_ID, + EVALUATE_VALUE.ISS_ST_RATING_ID + FROM + EVALUATE_VALUE +), + +FIL_NULL_INSTITUTION_CODE_KEY AS ( + SELECT + MPL.INSTITUTION_FK, + MPL.ISS_CODE_TYPE, + MPL.CODE_VALUE, + MPL.ISS_LT_RATING_ID, + MPL.ISS_ST_RATING_ID + FROM + MPL_GET_INSTITUTION_CODE_KEY MPL + WHERE + MPL.INSTITUTION_FK IS NOT NULL + AND MPL.ISS_ST_RATING_ID IS NOT NULL +), + +EXP_MAP_ST_TARGET AS ( + SELECT + FILL_NULL.INSTITUTION_FK, + --IIF(SUBSTR(ISS_ST_RATING_ID3,3,2)='SP','EADB_O_ST_SP', + --IIF(SUBSTR(ISS_ST_RATING_ID3,3,2)='DB','EADB_O_ST_DB', + --IIF(SUBSTR(ISS_ST_RATING_ID3,3,2)='IA','EADB_O_ST_IA', + --IIF(SUBSTR(ISS_ST_RATING_ID3,3,2)='MY','EADB_O_ST_MY', + --IIF(SUBSTR(ISS_ST_RATING_ID3,3,2)='SC','EADB_O_ST_SC',NULL))))) + CASE + WHEN (SUBSTR(FILL_NULL.ISS_ST_RATING_ID, 3, 2)) = 'SP' THEN 'EADB_O_ST_SP' + WHEN (SUBSTR(FILL_NULL.ISS_ST_RATING_ID, 3, 2)) = 'DB' THEN 'EADB_O_ST_DB' + WHEN (SUBSTR(FILL_NULL.ISS_ST_RATING_ID, 3, 2)) = 'IA' THEN 'EADB_O_ST_IA' + WHEN (SUBSTR(FILL_NULL.ISS_ST_RATING_ID, 3, 2)) = 'MY' THEN 'EADB_O_ST_MY' + WHEN (SUBSTR(FILL_NULL.ISS_ST_RATING_ID, 3, 2)) = 'SC' THEN 'EADB_O_ST_SC' + ELSE CAST(NULL AS VARCHAR2(255)) + END AS RATING_TYPE_FK, + --rtrim(ltrim(RATING)) + RTRIM(LTRIM(FILL_NULL.ISS_ST_RATING_ID)) AS RATING, + 'A' AS DUMMY + FROM + FIL_NULL_INSTITUTION_CODE_KEY FILL_NULL +), + +LOOKUP AS ( + SELECT * FROM EXP_MAP_ST_TARGET + LEFT JOIN DW_REF_MAIN.RATING DW_REF_MAIN_RATING ON EXP_MAP_ST_TARGET.RATING = DW_REF_MAIN_RATING.RATING_VALUE_CODE + WHERE + A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS') +), + +EXP_GET_RECORD_COUNT_ST AS ( + SELECT + LOOKUP.INSTITUTION_FK, + LOOKUP.RATING_TYPE_FK, + LOOKUP.NUMERICAL_EQUIVALENT, + LOOKUP.RATING, + ---SETVARIABLE($$Var_m_Start_StartTime,NULL) + CAST(NULL as date) AS SNAPSHOT_DATE, + 'C2D_EA' AS SOURCE, + --IIF($$Var_m_main_load='Y',null,setvariable($$Var_m_dwh_subprocess_load_set_key, null)), + CAST(NULL AS number) AS INDEP_SUBPROCESS_FK + FROM + LOOKUP + GROUP BY + LOOKUP.INSTITUTION_FK, + LOOKUP.RATING_TYPE_FK, + LOOKUP.NUMERICAL_EQUIVALENT, + LOOKUP.RATING + ORDER BY + LOOKUP.INSTITUTION_FK, + LOOKUP.RATING_TYPE_FK, + LOOKUP.NUMERICAL_EQUIVALENT +) + +SELECT + EXP_ST_COUNT.INSTITUTION_FK, + EXP_ST_COUNT.RATING_TYPE_FK, + CAST(NULL as varchar2(255)) AS SPECIFIC_PROVIDER_NAME, + EXP_ST_COUNT.RATING AS VALUE, + CAST(NULL as varchar2(255)) AS DATE_, + CAST(NULL as varchar2(255)) AS OTHER_NOTES, + CAST(NULL as varchar2(255)) AS VALUE_PD, + CAST(NULL as varchar2(255)) AS IS_PRELIMINARY, + EXP_ST_COUNT.NUMERICAL_EQUIVALENT, + CAST(NULL as varchar2(255)) AS OUTLOOK_VALUE, + CAST(NULL as varchar2(255)) AS OUTLOOK_DATE, + CAST(NULL as varchar2(255)) AS WATCHLIST_VALUE, + CAST(NULL as varchar2(255)) AS WATCHLIST_DATE, + EXP_ST_COUNT.SNAPSHOT_DATE, + CAST(NULL as varchar2(255)) AS RATING_ACTION, + CAST(NULL as varchar2(255)) AS RATING_ACTION_DATE, + CAST(NULL as varchar2(255)) AS SOURCE, + CAST(NULL as varchar2(255)) AS RATING_OTHER_TYPE, + EXP_ST_COUNT.INDEP_SUBPROCESS_FK +FROM + EXP_GET_RECORD_COUNT_ST EXP_ST_COUNT \ No newline at end of file diff --git a/dbt/models/rar/targets/m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD.sql b/dbt/models/rar/targets/m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD.sql new file mode 100644 index 0000000..98aa095 --- /dev/null +++ b/dbt/models/rar/targets/m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD.sql @@ -0,0 +1,78 @@ +{{ config(materialized='incremental', + incremental_strategy='append', + unique_key='ASSET_FK', + pre_hook="{{ control_model_start() }}", + post_hook="{{ control_model_end('TRG') }}", + alias = 'TE_ASSET_ROLE_INSTITUTION_test', + tags = ['m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD', 'A_TASK_HISTORY_TARGET_insert'], + schema = 'DW_TMP') }} + +WITH ASSET_VALUES AS ( +SELECT CASE + WHEN SQ.CODE_VALUE IS NULL THEN NULL + ELSE LKP_INSTITUTION_CODE_KEY.INSTITUTION_FK + END AS INSTITUTION_FK, + CASE + WHEN ROLE_FK='GUA_N' THEN 'GUA' + ELSE ROLE_FK + END AS ROLE_FK, + 'ISIN' AS CODE_TYPE_ASSET, + ISIN, + RTRIM(LTRIM(ISIN)) AS CODE_VALUE_ASSET, + ROLE, + GUAR_ASSESSMENT, + RESERVE_FUND_THRESHOLDS_EXCEED, + 'C2D_EA' AS SOURCE +FROM {{ ref("m_DWH_ROLE_TE_ASSET_SF_C2D_EA_INSTROLES_LOAD_SQ") }} SQ +LEFT JOIN (SELECT DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_NO AS CODE_TYPE_NO, + DW_REF_INSTITUTION_CODE_TYPE.CODE_TYPE_ID AS CODE_TYPE_ID + FROM DW_REF_MAIN.INSTITUTION_CODE_TYPE DW_REF_INSTITUTION_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) LKP_CODE_TYPE_NO ON LKP_CODE_TYPE_NO.CODE_TYPE_ID=UPPER(LTRIM(RTRIM(SQ.CODE_TYPE))) +LEFT JOIN (SELECT DW_RAR_MU_INSTITUTION_CODE_MAP.INSTITUTION_FK AS INSTITUTION_FK, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_TYPE_NO_ID AS CODE_TYPE_NO_ID, + DW_RAR_MU_INSTITUTION_CODE_MAP.CODE_VALUE AS CODE_VALUE + FROM DW_RAR.MU_INSTITUTION_CODE_MAP DW_RAR_MU_INSTITUTION_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) LKP_INSTITUTION_CODE_KEY ON LKP_INSTITUTION_CODE_KEY.CODE_TYPE_NO_ID=LKP_CODE_TYPE_NO.CODE_TYPE_NO + AND LKP_INSTITUTION_CODE_KEY.CODE_VALUE=SQ.CODE_VALUE +WHERE INSTITUTION_FK IS NOT NULL +), + + +TRANS AS ( +SELECT DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK AS ASSET_FK, + AV.INSTITUTION_FK, + AV.ROLE_FK, + AV.ROLE, + AV.GUAR_ASSESSMENT, + AV.RESERVE_FUND_THRESHOLDS_EXCEED, + AV.SOURCE, + ROW_NUMBER() OVER (PARTITION BY DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK, AV.INSTITUTION_FK, AV.ROLE_FK, AV.SOURCE ORDER BY DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK ASC ,AV.INSTITUTION_FK ASC ,AV.ROLE_FK ASC ,AV.ROLE ASC ,AV.SOURCE ASC) AS rn +FROM ASSET_VALUES AV +LEFT JOIN (SELECT CODE_TYPE_NO, CODE_TYPE_ID + FROM DW_REF_MAIN.ASSET_CODE_TYPE + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00','mm/dd/yyyy HH24:MI:SS')) DW_REF_ASSET_CODE_TYPE + ON DW_REF_ASSET_CODE_TYPE.CODE_TYPE_ID = UPPER(RTRIM(LTRIM(AV.CODE_TYPE_ASSET))) +LEFT JOIN (SELECT ASSET_FK , CODE_TYPE_NO_ID, CODE_VALUE + FROM DW_RAR.MU_ASSET_CODE_MAP + WHERE A_VALID_TO = To_Date('12/31/9999 00:00:00', 'mm/dd/yyyy HH24:MI:SS')) DW_RAR_MU_ASSET_CODE_MAP + ON DW_RAR_MU_ASSET_CODE_MAP.CODE_TYPE_NO_ID = DW_REF_ASSET_CODE_TYPE.CODE_TYPE_NO + AND DW_RAR_MU_ASSET_CODE_MAP.CODE_VALUE = RTRIM(LTRIM(AV.CODE_VALUE_ASSET)) +WHERE DW_RAR_MU_ASSET_CODE_MAP.ASSET_FK IS NOT NULL +) + + +SELECT TRANS.ASSET_FK, + TRANS.INSTITUTION_FK, + TRANS.ROLE_FK, + TRANS.ROLE AS SUBROLE_ID, + TRANS.GUAR_ASSESSMENT, + TRANS.RESERVE_FUND_THRESHOLDS_EXCEED, + TRANS.SOURCE, + CAST(NULL AS VARCHAR2(255)) AS COVER_PERCENT, + CAST(NULL AS VARCHAR2(255)) AS INDEP_SUBPROCESS_FK +FROM DW_TMP.TE_ASSET_ROLE_INSTITUTION_test DW_TMP_TE_ASSET_ROLE_INSTITUTITION +RIGHT JOIN TRANS ON DW_TMP_TE_ASSET_ROLE_INSTITUTITION.ASSET_FK = TRANS.ASSET_FK + AND DW_TMP_TE_ASSET_ROLE_INSTITUTITION.INSTITUTION_FK = TRANS.INSTITUTION_FK + AND DW_TMP_TE_ASSET_ROLE_INSTITUTITION.ROLE_FK = TRANS.ROLE_FK + AND DW_TMP_TE_ASSET_ROLE_INSTITUTITION.SOURCE = TRANS.SOURCE +WHERE DW_TMP_TE_ASSET_ROLE_INSTITUTITION.ASSET_FK IS NULL AND rn=1 diff --git a/dbt/models/sources/c2d.yml b/dbt/models/sources/c2d.yml new file mode 100644 index 0000000..91fc5a7 --- /dev/null +++ b/dbt/models/sources/c2d.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: c2d + schema: c2d + tables: + - name: V_EA_SNAPSHOT_DATES diff --git a/dbt/models/sources/dw_rar.yml b/dbt/models/sources/dw_rar.yml new file mode 100644 index 0000000..494367c --- /dev/null +++ b/dbt/models/sources/dw_rar.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: dw_rar + schema: dw_rar + tables: + - name: NH_ASSET_updated_test \ No newline at end of file diff --git a/dbt/models/sources/dw_ref_main.yml b/dbt/models/sources/dw_ref_main.yml new file mode 100644 index 0000000..78f8d0b --- /dev/null +++ b/dbt/models/sources/dw_ref_main.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: dw_ref_main + schema: dw_ref_main + tables: + - name: ASSET_CODE_TYPE \ No newline at end of file diff --git a/dbt/models/sources/dw_tmp.yml b/dbt/models/sources/dw_tmp.yml new file mode 100644 index 0000000..37c2ed3 --- /dev/null +++ b/dbt/models/sources/dw_tmp.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: dw_tmp + schema: dw_tmp + tables: + - name: TE_ASSET \ No newline at end of file diff --git a/dbt/models/sources/mpec.yaml b/dbt/models/sources/mpec.yaml new file mode 100644 index 0000000..eac44d2 --- /dev/null +++ b/dbt/models/sources/mpec.yaml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: mpec + schema: mpec + tables: + - name: T_MPEC diff --git a/dbt/models/sources/ods.yaml b/dbt/models/sources/ods.yaml new file mode 100644 index 0000000..be56611 --- /dev/null +++ b/dbt/models/sources/ods.yaml @@ -0,0 +1,186 @@ +version: 2 + +sources: + - name: ods + schema: ods + tables: + - name: DEBT + identifier: CSDB_DEBT_ODS + - name: DEBT_DAILY + identifier: CSDB_DEBT_DAILY_ODS + - name: INSTR_DESC_FULL + identifier: CSDB_INSTR_DESC_FULL_ODS + - name: INSTR_RAT_FULL + identifier: CSDB_INSTR_RAT_FULL_ODS + - name: ISSUER_DESC_FULL + identifier: CSDB_ISSUER_DESC_FULL_ODS + - name: ISSUER_RAT_FULL + identifier: CSDB_ISSUER_RAT_FULL_ODS + - name: RATINGS_RT_INSTRUMENT_RATING + identifier: CSDB_RT_INSTRUMENT_RATING_ODS + - name: STANDING_FACILITIES_HEADER + identifier: LM_STANDING_FACILITIES_HEADER_ODS + - name: STANDING_FACILITIES + identifier: LM_STANDING_FACILITIES_ODS + - name: FORECAST_HEADER + identifier: LM_FORECAST_HEADER_ODS + - name: FORECAST_ITEM + identifier: LM_FORECAST_ITEM_ODS + - name: BALANCESHEET_HEADER + identifier: LM_BALANCESHEET_HEADER_ODS + - name: BALANCESHEET_ITEM + identifier: LM_BALANCESHEET_ITEM_ODS + - name: QR_ADJ_HEADER + identifier: LM_QRE_ADJUSTMENTS_HEADER_ODS + - name: QR_ADJ_ITEM_HEADER + identifier: LM_QRE_ADJUSTMENTS_ITEM_HEADER_ODS + - name: QR_ADJ_ITEM + identifier: LM_QRE_ADJUSTMENTS_ITEM_ODS + - name: ADHOC_ADJ_HEADER + identifier: LM_ADHOC_ADJUSTMENTS_HEADER_ODS + - name: ADHOC_ADJ_ITEM_HEADER + identifier: LM_ADHOC_ADJUSTMENTS_ITEM_HEADER_ODS + - name: ADHOC_ADJ_ITEM + identifier: LM_ADHOC_ADJUSTMENTS_ITEM_ODS + - name: CSM_ADJ_HEADER + identifier: LM_CSM_ADJUSTMENTS_HEADER_ODS + - name: CSM_ADJ_ITEM_HEADER + identifier: LM_CSM_ADJUSTMENTS_ITEM_HEADER_ODS + - name: CSM_ADJ_ITEM + identifier: LM_CSM_ADJUSTMENTS_ITEM_ODS + - name: TTS_HEADER + identifier: LM_TTS_HEADER_ODS + - name: MRR_IND_CURRENT_ACCOUNT_HEADER + identifier: LM_CURRENT_ACCOUNTS_HEADER_ODS + - name: MRR_IND_CURRENT_ACCOUNT_ITEM + identifier: LM_CURRENT_ACCOUNTS_ITEM_ODS + - name: AGGREGATED_ALLOTMENT + identifier: TOP_AGGREGATED_ALLOTMENT_ODS + - name: ALLOTMENT + identifier: TOP_ALLOTMENT_ODS + - name: ALLOTMENT_MODIFICATION_HEADER + identifier: TOP_ALLOTMENT_MODIFICATION_HEADER_ODS + - name: ALLOTMENT_MODIFICATION_ITEM + identifier: TOP_ALLOTMENT_MODIFICATION_ITEM_ODS + - name: ANNOUNCEMENT + identifier: TOP_ANNOUNCEMENT_ODS + - name: FBL_ITEM + identifier: TOP_FULLBIDLIST_ITEM_ODS + - name: FULLBID_ARRAY_COMPILED + identifier: TOP_FULLBID_ARRAY_COMPILED_ODS + - name: ALLOTMENT_MODIFICATION_AGGREGATED + identifier: TOP_ALLOTMENT_MODIFICATION_AGGREGATED_ODS + - name: MPEC_ADMIN + identifier: C2D_MPEC_ADMIN_ODS + - name: MPEC_CONTENT + identifier: C2D_MPEC_CONTENT_ODS + - name: MPEC_CONTENT_CRITERION + identifier: C2D_MPEC_CONTENT_CRITERION_ODS + - name: MPEC_CONTENT_CRITERION_FULL + identifier: C2D_MPEC_CONTENT_CRITERION_FULL_ODS + - name: MPEC_CONTENT_FULL + identifier: C2D_MPEC_CONTENT_FULL_ODS + - name: MPEC_MID_FULL + identifier: C2D_MPEC_MID_FULL_ODS + - name: ELA_INFO_REPLICATION + identifier: C2D_ELA_INFO_REPLICATION_ODS + - name: PRICING + identifier: CEPH_PRICING_ODS + - name: BRANCH + identifier: TMS_BRANCH # Current table name in the database + - name: CALENDAR + identifier: TMS_CALENDAR + - name: CLIENT_ACCOUNT # This is the alias used in dbt + identifier: TMS_CLIENT_ACCOUNT # Current table name in the database + - name: CLIENT_INSTRUCTION + identifier: TMS_CLIENT_INSTRUCTION + - name: HISTORY_LOG + identifier: TMS_HISTORY_LOG + - name: PARAMETER + identifier: TMS_PARAMETER + - name: LIMIT_ACCESS + identifier: TMS_LIMIT_ACCESS + - name: PORTFOLIO_ACCESS + identifier: TMS_PORTFOLIO_ACCESS + - name: PORTFOLIO_TREE + identifier: TMS_PORTFOLIO_TREE + - name: RULES + identifier: TMS_RULES + - name: SDM_ENTITY_STATE + identifier: TMS_SDM_ENTITY_STATE + - name: ACMCURRENCYFLOW # This is the alias used in dbt + identifier: TMS_ACM_CURRENCY_FLOW # Current table name in the database + - name: ACMENTRYSTATELEDGERGROUP # This is the alias used in dbt + identifier: TMS_ACM_ENTRY_STATE_LEDGER_GROUP # Current table name in the database + - name: ACTIVITY + identifier: TMS_ACTIVITY # Current table name in the database + - name: CLIENT + identifier: TMS_CLIENT + - name: CUSTODYBALANCE + identifier: TMS_CUSTODY_BALANCE + - name: EFFECTIVEROLEPROFILE + identifier: TMS_EFFECTIVE_ROLE_PROFILE + - name: ACTIVITYINTERVAL + identifier: TMS_ACTIVITY_INTERVAL + - name: CLIENTGROUPMAP # This is the alias used in dbt + identifier: TMS_CLIENT_GROUP_MAP # Current table name in the database + - name: EFFECTIVEPERMISSIONS + identifier: TMS_EFFECTIVE_PERMISSIONS + - name: LIMITLOG + identifier: TMS_LIMIT_LOG_ODS + - name: ROLE_PORTFOLIO_PROFILE + identifier: TMS_ROLE_PORTFOLIO_PROFILE + - name: TMS_SECURITY_POSITION + indentifier: TMS_SECURITY_POSITION + - name: F_BRANCH + identifier: FXCD_F_BRANCH_ODS + - name: F_CLEARER + identifier: FXCD_F_CLEARER_ODS + - name: F_CLEARER_NCB_LIMIT + identifier: FXCD_F_CLEARER_NCB_LIMIT_ODS + - name: F_CONSTANT + identifier: FXCD_F_CONSTANT_ODS + - name: F_COUNTERPARTY + identifier: FXCD_F_COUNTERPARTY_ODS + - name: F_COUNTRY + identifier: FXCD_F_COUNTRY_ODS + - name: F_COUNTRY_NCB_LIMIT + identifier: FXCD_F_COUNTRY_NCB_LIMIT_ODS + - name: F_CTP_GROUP + identifier: FXCD_F_CTP_GROUP_ODS + - name: F_CTP_GROUP_NCB_LIMIT + identifier: FXCD_F_CTP_GROUP_NCB_LIMIT_ODS + - name: F_CTP_NCB_LIMIT + identifier: FXCD_F_CTP_NCB_LIMIT_ODS + - name: F_ELIGIBILITY + identifier: FXCD_F_ELIGIBILITY_ODS + - name: F_ELIGIBILITY_GROUP_MAP + identifier: FXCD_F_ELIGIBILITY_GROUP_MAP_ODS + - name: F_ELIGIBILITY_ISSUER + identifier: FXCD_F_ELIGIBILITY_ISSUER_ODS + - name: F_ELIGIBILITY_ISSUER_MAP + identifier: FXCD_F_ELIGIBILITY_ISSUER_MAP_ODS + - name: F_ELIGIBILITY_MAP + identifier: FXCD_F_ELIGIBILITY_MAP_ODS + - name: F_ENTITY + identifier: FXCD_F_ENTITY_ODS + - name: F_EQUIVALENCE_RULE + identifier: FXCD_F_EQUIVALENCE_RULE_ODS + - name: F_ISSUER + identifier: FXCD_F_ISSUER_ODS + - name: F_ISSUER_NCB_LIMIT + identifier: FXCD_F_ISSUER_NCB_LIMIT_ODS + - name: F_LIQUIDITY_NCB_LIMIT + identifier: FXCD_F_LIQUIDITY_NCB_LIMIT_ODS + - name: F_NCB_PORTFOLIO_SHARE + identifier: FXCD_F_NCB_PORTFOLIO_SHARE_ODS + - name: F_RATING + identifier: FXCD_F_RATING_ODS + - name: F_RATING_AGENCY + identifier: FXCD_F_RATING_AGENCY_ODS + - name: T2_MAXIMA_CREDIT_LINE_AMOUNT + identifier: T2_MAXIMA_CREDIT_LINE_AMOUNT_ODS + - name: PEAK_INTRADAY_CREDIT_USE + identifier: T2_PEAK_INTRADAY_CREDIT_USE_ODS + - name: PEAK_LIQUIDITY_NEED + identifier: T2_PEAK_LIQUIDITY_NEED_ODS \ No newline at end of file diff --git a/dbt/models/sources/ou_c2d.yml b/dbt/models/sources/ou_c2d.yml new file mode 100644 index 0000000..068cef1 --- /dev/null +++ b/dbt/models/sources/ou_c2d.yml @@ -0,0 +1,25 @@ +version: 2 + +sources: + - name: ou_c2d + schema: ou_c2d + tables: + - name: UC_NMA_ACC + - name: UC_NMA_DISSEM + - name: UC_NMA_DECC_DISSEM + - name: UC_MA_DISSEM + - name: EA_ELIGIBLE_ASSET + - name: EA_ADDITIONAL_GUARANTOR + - name: EA_FOREIGN_EXCHANGE_SWAP_COUNT + - name: EA_INTEREST_RATE_SWAP_PROVIDER + - name: EA_ISSUER_ACCOUNT_BANK + - name: EA_LIQUIDITY_SUPPORT_PROVIDER + - name: EA_ORIGINATOR + - name: EA_SERVICER + - name: EA_RIAD_MFI_ISIN + - name: MPEC_ADMIN + - name: MPEC_CONTENT + - name: MPEC_CONTENT_CRITERION + - name: MPEC_CONTENT_CRITERION_FULL + - name: MPEC_CONTENT_FULL + - name: MPEC_MID_FULL diff --git a/dbt/models/sources/ou_casdb.yml b/dbt/models/sources/ou_casdb.yml new file mode 100644 index 0000000..cfd3595 --- /dev/null +++ b/dbt/models/sources/ou_casdb.yml @@ -0,0 +1,8 @@ + +version: 2 + +sources: + - name: ou_csdb + schema: ou_csdb + tables: + - name: RATINGS_RT_PARTY_RATING diff --git a/dbt/models/sources/ou_lm.yml b/dbt/models/sources/ou_lm.yml new file mode 100644 index 0000000..98017d2 --- /dev/null +++ b/dbt/models/sources/ou_lm.yml @@ -0,0 +1,41 @@ +version: 2 + +sources: + - name: OU_LM + schema: ODS + tables: + - name: STANDING_FACILITIES_HEADER + identifier: LM_STANDING_FACILITIES_HEADER_ODS + - name: STANDING_FACILITIES + identifier: LM_STANDING_FACILITIES_ODS + - name: FORECAST_HEADER + identifier: LM_FORECAST_HEADER_ODS + - name: FORECAST_ITEM + identifier: LM_FORECAST_ITEM_ODS + - name: BALANCESHEET_HEADER + identifier: LM_BALANCESHEET_HEADER_ODS + - name: BALANCESHEET_ITEM + identifier: LM_BALANCESHEET_ITEM_ODS + - name: QR_ADJ_HEADER + identifier: LM_QRE_ADJUSTMENTS_HEADER_ODS + - name: QR_ADJ_ITEM_HEADER + identifier: LM_QRE_ADJUSTMENTS_ITEM_HEADER_ODS + - name: QR_ADJ_ITEM + identifier: LM_QRE_ADJUSTMENTS_ITEM_ODS + - name: ADHOC_ADJ_HEADER + identifier: LM_ADHOC_ADJUSTMENTS_HEADER_ODS + - name: ADHOC_ADJ_ITEM_HEADER + identifier: LM_ADHOC_ADJUSTMENTS_ITEM_HEADER_ODS + - name: ADHOC_ADJ_ITEM + identifier: LM_ADHOC_ADJUSTMENTS_ITEM_ODS + - name: CSM_ADJ_HEADER + identifier: LM_CSM_ADJUSTMENTS_HEADER_ODS + - name: CSM_ADJ_ITEM_HEADER + identifier: LM_CSM_ADJUSTMENTS_ITEM_HEADER_ODS + - name: CSM_ADJ_ITEM + identifier: LM_CSM_ADJUSTMENTS_ITEM_ODS + - name: TTS_HEADER + identifier: LM_TTS_HEADER_ODS + - name: TTS_ITEM + identifier: LM_TTS_ITEM_ODS + diff --git a/dbt/models/sources/ou_mrr.yml b/dbt/models/sources/ou_mrr.yml new file mode 100644 index 0000000..b563980 --- /dev/null +++ b/dbt/models/sources/ou_mrr.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: OU_MRR + schema: ODS + tables: + - name: MRR_IND_CURRENT_ACCOUNT_HEADER + identifier: LM_CURRENT_ACCOUNTS_HEADER_ODS + - name: MRR_IND_CURRENT_ACCOUNT_ITEM + identifier: LM_CURRENT_ACCOUNTS_ITEM_ODS \ No newline at end of file diff --git a/dbt/models/sources/ou_riad.yml b/dbt/models/sources/ou_riad.yml new file mode 100644 index 0000000..34d5a71 --- /dev/null +++ b/dbt/models/sources/ou_riad.yml @@ -0,0 +1,16 @@ +version: 2 + +sources: + - name: ou_riad + schema: ou_riad + tables: + - name: ENTTY_IDNTFRS_FLTTND + - name: RIAD_CLOSE_LINK + - name: RIAD_CLOSE_LINK_ADHOC + - name: ENTTY_FLTTND_C2D + - name: ENTTY_RIAD_CD_ALS + - name: ENTTY_FLTTND_ECMS + - name: GRP_FTR_STRCTR + - name: RLTNSHP + + diff --git a/dbt/models/sources/ou_rqsd.yml b/dbt/models/sources/ou_rqsd.yml new file mode 100644 index 0000000..9fff23c --- /dev/null +++ b/dbt/models/sources/ou_rqsd.yml @@ -0,0 +1,9 @@ +version: 2 + +sources: + - name: ou_rqsd + schema: ou_rqsd + tables: + - name: RQSD_OBSERVATIONS + - name: RQSD_SUBA_DEVO + - name: RQSD_FX \ No newline at end of file diff --git a/dbt/models/sources/ou_tms.yml b/dbt/models/sources/ou_tms.yml new file mode 100644 index 0000000..3ae2675 --- /dev/null +++ b/dbt/models/sources/ou_tms.yml @@ -0,0 +1,37 @@ +version: 2 + +sources: + - name: ou_tms + schema: ou_tms + tables: + - name: ACMCURRENCYFLOW + - name: ACTIVITYLOGDUE + - name: ACMENTRYSTATELEDGERGROUP + - name: ACTIVITY_LOG + - name: BALANCE + - name: BLACKOUT_LOG + - name: BRANCH + - name: CALENDAR + - name: CASHFLOW + - name: CLIENT + - name: CUSTODYBALANCE + - name: ECBINSTRUMENTBONDCASHFLOW + - name: EFFECTIVEROLEPROFILE + - name: FINMESSAGELOG + - name: HISTORY_LOG + - name: INSTRUMENTBONDCASHFLOW + - name: INSTRUMENT_REPORT + - name: MARKETINFO + - name: PARAMETER + - name: PORTFOLIOTREE + - name: PRICES + - name: PROPERTY + - name: RECONCILIATION + - name: ROLEPORTFOLIOPROFILE + - name: RULES + - name: SDM_ENTITY_STATE + - name: SECURITYPOSITION + - name: SETTLEMENTCASHFLOW + - name: SETTLEMENTLOG + - name: TRANSACTION + - name: USERINFORMATION diff --git a/dbt/models/sources/ou_top.yml b/dbt/models/sources/ou_top.yml new file mode 100644 index 0000000..f6973d6 --- /dev/null +++ b/dbt/models/sources/ou_top.yml @@ -0,0 +1,11 @@ +sources: + - name: ou_top + schema: ou_top + tables: + - name: AGGREGATED_ALLOTMENT + - name: ALLOTMENT + - name: allotment_modification_header + - name: allotment_modification_item + - name: ANNOUNCEMENT + - name: FBL_ITEM + - name: FULLBID_ARRAY_COMPILED \ No newline at end of file diff --git a/dbt/models/sources/riad.yml b/dbt/models/sources/riad.yml new file mode 100644 index 0000000..3de0f67 --- /dev/null +++ b/dbt/models/sources/riad.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: riad + schema: riad + tables: + - name: T_ENTTY_FLTTND_ESSNTL_CURRENT \ No newline at end of file diff --git a/dbt/models/sources/sf_c2d.yml b/dbt/models/sources/sf_c2d.yml new file mode 100644 index 0000000..10db620 --- /dev/null +++ b/dbt/models/sources/sf_c2d.yml @@ -0,0 +1,15 @@ +version: 2 + +sources: + - name: sf_c2d + schema: sf_c2d + tables: + - name: EA_ELIGIBLE_ASSET + - name: EA_SERVICER + - name: EA_ISSUER_ACCOUNT_BANK + - name: EA_ADDITIONAL_GUARANTOR + - name: EA_FOREIGN_EXCHANGE_SWAP_COUNT + - name: EA_INTEREST_RATE_SWAP_PROVIDER + - name: EA_ORIGINATOR + - name: EA_LIQUIDITY_SUPPORT_PROVIDER + diff --git a/dbt/models/sources/sf_led.yml b/dbt/models/sources/sf_led.yml new file mode 100644 index 0000000..bd96f7c --- /dev/null +++ b/dbt/models/sources/sf_led.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: sf_led + schema: sf_led + tables: + - name: ISIN_WITH_LARGER_LIMITS + diff --git a/dbt/models/sources/sf_tms.yml b/dbt/models/sources/sf_tms.yml new file mode 100644 index 0000000..605c7e9 --- /dev/null +++ b/dbt/models/sources/sf_tms.yml @@ -0,0 +1,9 @@ +version: 2 + +sources: + - name: sf_tms + schema: sf_tms + tables: + - name: UMISECURITYINFO + - name: ASSET_ENRICHMENT + diff --git a/dbt/models/views/.gitkeep b/dbt/models/views/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql new file mode 100644 index 0000000..b79f273 --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql @@ -0,0 +1,63 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_1_1_FIN_CUR', + schema='RQSD' +) }} + +select +t.a_workflow_history_key, +t.obligationmodulereferencedate, +t.reportingentitycollectionuniqueid, +t.receivedfileversionnumber, +t.receivedfilereceiveddate, +t.revalidationdate, +t.ann_1_1_ref_date, +t.ctp_mfi_id, +t.ctp_name, +t.ctp_country, +t.ctp_legal_status, +t.ctp_lprt, +t.ctp_measure_start_date, +t.ann_1_1_etwdr_mfi_id, +t.ann_1_1_etwdr_lei, +t.ann_1_1_submitter_comment, +t.datacollectioncode, +t.reportingcyclename, +t.reportingcyclestatus, +t.modulecode, +t.moduleversionnumber, +t.reportingentityname, +t.entityattributecountry, +t.entitygroupentityname, +t.obligationmoduleremittancedate, +t.obligationmoduleexpected, +t.revalidationversionnumber, +t.receivedfilesystemfilename, +t.obligationstatusstatus, +t.filestatussetsubmissionstatus, +t.filestatussetvalidationstatus, +t.numberoferrors, +t.numberofwarnings, +t.delayindays, +t.failedattempts, +t.tablename +from +( +select a.a_workflow_history_key, a.obligationmodulereferencedate, a.reportingentitycollectionuniqueid, a.receivedfileversionnumber, a.receivedfilereceiveddate, a.revalidationdate, a.ann_1_1_ref_date, +a.ctp_mfi_id, a.ctp_name, a.ctp_country, a.ctp_legal_status, a.ctp_lprt, a.ctp_measure_start_date, a.ann_1_1_etwdr_mfi_id, a.ann_1_1_etwdr_lei, a.ann_1_1_submitter_comment, +a.datacollectioncode, a.reportingcyclename, a.reportingcyclestatus, a.modulecode, a.moduleversionnumber, a.reportingentityname, a.entityattributecountry, +a.entitygroupentityname, a.obligationmoduleremittancedate, a.obligationmoduleexpected, a.revalidationversionnumber, a.receivedfilesystemfilename, a.obligationstatusstatus, +a.filestatussetsubmissionstatus, a.filestatussetvalidationstatus, a.numberoferrors, a.numberofwarnings, a.delayindays, a.failedattempts, a.tablename, +row_number() over (partition by a.reportingentitycollectionuniqueid, a.obligationmodulereferencedate, a.ctp_mfi_id +order by greatest(a.receivedfilereceiveddate, nvl(a.revalidationdate, '0')) desc) as rn +from {{ ref('m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS') }} a, {{ ref('m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW') }} v +where a.reportingentitycollectionuniqueid = v.reportingentitycollectionuniqueid +and a.obligationmodulereferencedate = v.obligationmodulereferencedate +and a.receivedfilereceiveddate = v.max_version +and a.A_WORKFLOW_HISTORY_KEY IN ( + SELECT DISTINCT A_WORKFLOW_HISTORY_KEY + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME = 'w_MOPDB_RQSD_PROCESS' + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y') +) t +where t.rn = 1 \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql new file mode 100644 index 0000000..ca26c18 --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql @@ -0,0 +1,9 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_1_1_FIN_MAX_VERSION', + schema='RQSD' +) }} + +SELECT MAX(RECEIVEDFILERECEIVEDDATE) AS MAX_VERSION, REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_1_FIN_ALL_ODS_RQSD_OBSERVATIONS') }} +GROUP BY REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql new file mode 100644 index 0000000..58828b0 --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql @@ -0,0 +1,64 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_1_2_FIN_CUR', + schema='RQSD' +) }} + +SELECT +t.a_workflow_history_key, +t.obligationmodulereferencedate, +t.reportingentitycollectionuniqueid, +t.receivedfileversionnumber, +t.receivedfilereceiveddate, +t.revalidationdate, +t.ann_1_2_ref_date, +t.etwdr_comp_key, +t.etwdr_mfi_id, +t.etwdr_lei, +t.etwdr_name, +t.etwdr_country, +t.etwdr_in_eu, +t.etwdr_source, +t.etwdr_submitter, +t.ann_1_2_submitter_comment, +t.datacollectioncode, +t.reportingcyclename, +t.reportingcyclestatus, +t.modulecode, +t.moduleversionnumber, +t.reportingentityname, +t.entityattributecountry, +t.entitygroupentityname, +t.obligationmoduleremittancedate, +t.obligationmoduleexpected, +t.revalidationversionnumber, +t.receivedfilesystemfilename, +t.obligationstatusstatus, +t.filestatussetsubmissionstatus, +t.filestatussetvalidationstatus, +t.numberoferrors, +t.numberofwarnings, +t.delayindays, +t.failedattempts, +t.tablename +from +( +select +a.a_workflow_history_key, a.obligationmodulereferencedate, a.reportingentitycollectionuniqueid, a.receivedfileversionnumber, a.receivedfilereceiveddate, a.revalidationdate, +a.ann_1_2_ref_date, a.etwdr_comp_key, a.etwdr_mfi_id, a.etwdr_lei, a.etwdr_name, a.etwdr_country, a.etwdr_in_eu, a.etwdr_source, a.etwdr_submitter, +a.ann_1_2_submitter_comment, a.datacollectioncode, a.reportingcyclename, a.reportingcyclestatus, a.modulecode, a.moduleversionnumber, a.reportingentityname, +a.entityattributecountry, a.entitygroupentityname, a.obligationmoduleremittancedate, a.obligationmoduleexpected, a.revalidationversionnumber, a.receivedfilesystemfilename, +a.obligationstatusstatus, a.filestatussetsubmissionstatus, a.filestatussetvalidationstatus, a.numberoferrors, a.numberofwarnings, a.delayindays, a.failedattempts, a.tablename, +row_number() over (partition by a.reportingentitycollectionuniqueid, a.obligationmodulereferencedate, a.etwdr_comp_key +order by greatest(a.receivedfilereceiveddate, nvl(a.revalidationdate, '0')) desc) as rn +from {{ ref('m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS') }} a, {{ ref('m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW') }} v +where a.reportingentitycollectionuniqueid = v.reportingentitycollectionuniqueid +and a.obligationmodulereferencedate = v.obligationmodulereferencedate +and a.receivedfilereceiveddate = v.max_version +and a.A_WORKFLOW_HISTORY_KEY IN ( + SELECT DISTINCT A_WORKFLOW_HISTORY_KEY + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME = 'w_MOPDB_RQSD_PROCESS' + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y') +) t +where t.rn = 1 \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql new file mode 100644 index 0000000..15f9cae --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql @@ -0,0 +1,9 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_1_2_FIN_MAX_VERSION', + schema='RQSD' +) }} + +SELECT MAX(RECEIVEDFILERECEIVEDDATE) AS MAX_VERSION, REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE +FROM {{ ref('m_MOPDB_RQSD_ANNEX_1_2_FIN_ALL_ODS_RQSD_OBSERVATIONS') }} +GROUP BY REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW.sql new file mode 100644 index 0000000..41c5ea3 --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW.sql @@ -0,0 +1,107 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_2_CUR_NCB', + schema='RQSD' +) }} + +select t.a_workflow_history_key, + t.obligationmodulereferencedate, + t.reportingentitycollectionuniqueid, + t.receivedfileversionnumber, + t.receivedfilereceiveddate, + t.revalidationdate, + t.ref_date, + t.inst_comp_key, + t.mfi_id, + t.legal_entity_id, + t.inst_name, + t.currency, + t.reported_by_supervisor, + t.confirmed_by_supervisor, + t.exp_corep_cons, + t.exp_corep_solo, + t.exp_finrep_cons, + t.exp_finrep_solo, + t.exp_lev_cons, + t.exp_lev_solo, + t.exp_lcr_cons, + t.exp_lcr_solo, + t.exp_nsfr_cons, + t.exp_nsfr_solo, + t.cons_cet1_amt, + t.cons_tier1_amt, + t.cons_tot_cap_amt, + t.cons_cet1_ratio, + t.cons_tier1_ratio, + t.cons_tot_cap_ratio, + t.cons_risk_wght_assets, + t.solo_cet1_amt, + t.solo_tier1_amt, + t.solo_tot_cap_amt, + t.solo_cet1_ratio, + t.solo_tier1_ratio, + t.solo_tot_cap_ratio, + t.solo_risk_wght_assets, + t.cons_tot_assets, + t.solo_tot_assets, + t.cons_lev_ratio_full, + t.cons_lev_ratio_trans, + t.cons_lev_ratio_req, + CASE WHEN t.cons_lev_ratio_req IS NULL THEN NULL + WHEN t.cons_lev_ratio_req=0.0300000000 THEN 'N' + ELSE 'Y' END AS cons_lev_ratio_adj, + t.solo_lev_ratio_full, + t.solo_lev_ratio_trans, + t.solo_lev_ratio_req, + CASE WHEN t.solo_lev_ratio_req IS NULL THEN NULL + WHEN t.solo_lev_ratio_req=0.0300000000 THEN 'N' + ELSE 'Y' END AS solo_lev_ratio_adj, + t.cons_lc_ratio, + t.solo_lc_ratio, + t.cons_nsfr_ratio, + t.solo_nsfr_ratio, + t.submitter_comment, + t.datacollectioncode, + t.reportingcyclename, + t.reportingcyclestatus, + t.modulecode, + t.moduleversionnumber, + t.reportingentityname, + t.entityattributecountry, + t.entitygroupentityname, + t.obligationmoduleremittancedate, + t.obligationmoduleexpected, + t.revalidationversionnumber, + t.receivedfilesystemfilename, + t.obligationstatusstatus, + t.filestatussetsubmissionstatus, + t.filestatussetvalidationstatus, + t.numberoferrors, + t.numberofwarnings, + t.delayindays, + t.failedattempts, + t.tablename + from + ( + select + a.a_workflow_history_key, a.obligationmodulereferencedate, a.reportingentitycollectionuniqueid, a.receivedfileversionnumber, a.receivedfilereceiveddate, a.revalidationdate, a.ref_date, + a.inst_comp_key, a.mfi_id, a.legal_entity_id, a.inst_name, a.currency, a.reported_by_supervisor, a.confirmed_by_supervisor, a.exp_corep_cons, a.exp_corep_solo, + a.exp_finrep_cons, a.exp_finrep_solo, a.exp_lev_cons, a.exp_lev_solo, a.exp_lcr_cons, a.exp_lcr_solo, a.exp_nsfr_cons, a.exp_nsfr_solo, a.cons_cet1_amt, a.cons_tier1_amt, + a.cons_tot_cap_amt, a.cons_cet1_ratio, a.cons_tier1_ratio, a.cons_tot_cap_ratio, a.cons_risk_wght_assets, a.solo_cet1_amt, a.solo_tier1_amt, a.solo_tot_cap_amt, + a.solo_cet1_ratio, a.solo_tier1_ratio, a.solo_tot_cap_ratio, a.solo_risk_wght_assets, a.cons_tot_assets, a.solo_tot_assets, a.cons_lev_ratio_full, a.cons_lev_ratio_trans, + a.cons_lev_ratio_req, a.solo_lev_ratio_full, a.solo_lev_ratio_trans, a.solo_lev_ratio_req, a.cons_lc_ratio, a.solo_lc_ratio, a.cons_nsfr_ratio, a.solo_nsfr_ratio, + a.submitter_comment, a.datacollectioncode, a.reportingcyclename, a.reportingcyclestatus, a.modulecode, a.moduleversionnumber, a.reportingentityname, a.entityattributecountry, + a.entitygroupentityname, a.obligationmoduleremittancedate, a.obligationmoduleexpected, a.revalidationversionnumber, a.receivedfilesystemfilename, a.obligationstatusstatus, + a.filestatussetsubmissionstatus, a.filestatussetvalidationstatus, a.numberoferrors, a.numberofwarnings, a.delayindays, a.failedattempts, a.tablename, + row_number() over (partition by a.reportingentitycollectionuniqueid, a.obligationmodulereferencedate, a.inst_comp_key order by greatest(a.receivedfilereceiveddate, nvl(a.revalidationdate, '0')) desc) as rn + from {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS') }} a, {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW') }} v + where a.reportingentitycollectionuniqueid = v.reportingentitycollectionuniqueid + and a.obligationmodulereferencedate = v.obligationmodulereferencedate + and a.receivedfilereceiveddate = v.max_version + and a.A_WORKFLOW_HISTORY_KEY IN ( + SELECT DISTINCT A_WORKFLOW_HISTORY_KEY + FROM {{ source('control_tables', 'A_WORKFLOW_HISTORY') }} A_WORKFLOW_HISTORY + WHERE A_WORKFLOW_HISTORY.WORKFLOW_NAME = 'w_MOPDB_RQSD_PROCESS' + AND A_WORKFLOW_HISTORY.WORKFLOW_SUCCESSFUL = 'Y') + ) t + where t.rn = 1 and t.entitygroupentityname='NCB' \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql new file mode 100644 index 0000000..3b3753b --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_VIEW.sql @@ -0,0 +1,155 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_2_CUR', + schema='RQSD' +) }} + +SELECT a_workflow_history_key, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(REF_DATE AS VARCHAR2(200)) AS REF_DATE, + CAST(INST_COMP_KEY AS VARCHAR2(200)) AS INST_COMP_KEY, + CAST(MFI_ID AS VARCHAR2(200)) AS MFI_ID, + CAST(LEGAL_ENTITY_ID AS VARCHAR2(200)) AS LEGAL_ENTITY_ID, + CAST(INST_NAME AS VARCHAR2(200)) AS INST_NAME, + CAST(CURRENCY AS VARCHAR2(200)) AS CURRENCY, + CAST(REPORTED_BY_SUPERVISOR AS VARCHAR2(200)) AS REPORTED_BY_SUPERVISOR, + CAST(CONFIRMED_BY_SUPERVISOR AS VARCHAR2(200)) AS CONFIRMED_BY_SUPERVISOR, + CAST(EXP_COREP_CONS AS VARCHAR2(200)) AS EXP_COREP_CONS, + CAST(EXP_COREP_SOLO AS VARCHAR2(200)) AS EXP_COREP_SOLO, + CAST(EXP_FINREP_CONS AS VARCHAR2(200)) AS EXP_FINREP_CONS, + CAST(EXP_FINREP_SOLO AS VARCHAR2(200)) AS EXP_FINREP_SOLO, + CAST(EXP_LEV_CONS AS VARCHAR2(200)) AS EXP_LEV_CONS, + CAST(EXP_LEV_SOLO AS VARCHAR2(200)) AS EXP_LEV_SOLO, + CAST(EXP_LCR_CONS AS VARCHAR2(200)) AS EXP_LCR_CONS, + CAST(EXP_LCR_SOLO AS VARCHAR2(200)) AS EXP_LCR_SOLO, + CAST(EXP_NSFR_CONS AS VARCHAR2(200)) AS EXP_NSFR_CONS, + CAST(EXP_NSFR_SOLO AS VARCHAR2(200)) AS EXP_NSFR_SOLO, + CAST(CONS_CET1_AMT AS VARCHAR2(200)) AS CONS_CET1_AMT, + CAST(CONS_TIER1_AMT AS VARCHAR2(200)) AS CONS_TIER1_AMT, + CAST(CONS_TOT_CAP_AMT AS VARCHAR2(200)) AS CONS_TOT_CAP_AMT, + CAST(CONS_CET1_RATIO AS VARCHAR2(200)) AS CONS_CET1_RATIO, + CAST(CONS_TIER1_RATIO AS VARCHAR2(200)) AS CONS_TIER1_RATIO, + CAST(CONS_TOT_CAP_RATIO AS VARCHAR2(200)) AS CONS_TOT_CAP_RATIO, + CAST(CONS_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS CONS_RISK_WGHT_ASSETS, + CAST(SOLO_CET1_AMT AS VARCHAR2(200)) AS SOLO_CET1_AMT, + CAST(SOLO_TIER1_AMT AS VARCHAR2(200)) AS SOLO_TIER1_AMT, + CAST(SOLO_TOT_CAP_AMT AS VARCHAR2(200)) AS SOLO_TOT_CAP_AMT, + CAST(SOLO_CET1_RATIO AS VARCHAR2(200)) AS SOLO_CET1_RATIO, + CAST(SOLO_TIER1_RATIO AS VARCHAR2(200)) AS SOLO_TIER1_RATIO, + CAST(SOLO_TOT_CAP_RATIO AS VARCHAR2(200)) AS SOLO_TOT_CAP_RATIO, + CAST(SOLO_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS SOLO_RISK_WGHT_ASSETS, + CAST(CONS_TOT_ASSETS AS VARCHAR2(200)) AS CONS_TOT_ASSETS, + CAST(SOLO_TOT_ASSETS AS VARCHAR2(200)) AS SOLO_TOT_ASSETS, + CAST(CONS_LEV_RATIO_FULL AS VARCHAR2(200)) AS CONS_LEV_RATIO_FULL, + CAST(CONS_LEV_RATIO_TRANS AS VARCHAR2(200)) AS CONS_LEV_RATIO_TRANS, + CAST(CONS_LEV_RATIO_REQ AS VARCHAR2(200)) AS CONS_LEV_RATIO_REQ, + CAST(CONS_LEV_RATIO_ADJ AS CHAR(1)) AS CONS_LEV_RATIO_ADJ, + CAST(SOLO_LEV_RATIO_FULL AS VARCHAR2(200)) AS SOLO_LEV_RATIO_FULL, + CAST(SOLO_LEV_RATIO_TRANS AS VARCHAR2(200)) AS SOLO_LEV_RATIO_TRANS, + CAST(SOLO_LEV_RATIO_REQ AS VARCHAR2(200)) AS SOLO_LEV_RATIO_REQ, + CAST(SOLO_LEV_RATIO_ADJ AS CHAR(1)) AS SOLO_LEV_RATIO_ADJ, + CAST(CONS_LC_RATIO AS VARCHAR2(200)) AS CONS_LC_RATIO, + CAST(SOLO_LC_RATIO AS VARCHAR2(200)) AS SOLO_LC_RATIO, + CAST(CONS_NSFR_RATIO AS VARCHAR2(200)) AS CONS_NSFR_RATIO, + CAST(SOLO_NSFR_RATIO AS VARCHAR2(200)) AS SOLO_NSFR_RATIO, + CAST(SUBMITTER_COMMENT AS VARCHAR2(200)) AS SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_CUR_NCB_VIEW') }} +UNION ALL +SELECT {{ get_workflow_history_key() }} AS a_workflow_history_key, + CAST(OBLIGATIONMODULEREFERENCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREFERENCEDATE, + CAST(REPORTINGENTITYCOLLECTIONUNIQUEID AS VARCHAR2(200)) AS REPORTINGENTITYCOLLECTIONUNIQUEID, + CAST(RECEIVEDFILEVERSIONNUMBER AS NUMBER(38,10)) AS RECEIVEDFILEVERSIONNUMBER, + CAST(RECEIVEDFILERECEIVEDDATE AS VARCHAR2(30)) AS RECEIVEDFILERECEIVEDDATE, + CAST(REVALIDATIONDATE AS VARCHAR2(30)) AS REVALIDATIONDATE, + CAST(REF_DATE AS VARCHAR2(200)) AS REF_DATE, + CAST(INST_COMP_KEY AS VARCHAR2(200)) AS INST_COMP_KEY, + CAST(MFI_ID AS VARCHAR2(200)) AS MFI_ID, + CAST(LEGAL_ENTITY_ID AS VARCHAR2(200)) AS LEGAL_ENTITY_ID, + CAST(INST_NAME AS VARCHAR2(200)) AS INST_NAME, + CAST(CURRENCY AS VARCHAR2(200)) AS CURRENCY, + CAST(REPORTED_BY_SUPERVISOR AS VARCHAR2(200)) AS REPORTED_BY_SUPERVISOR, + CAST(CONFIRMED_BY_SUPERVISOR AS VARCHAR2(200)) AS CONFIRMED_BY_SUPERVISOR, + CAST(EXP_COREP_CONS AS VARCHAR2(200)) AS EXP_COREP_CONS, + CAST(EXP_COREP_SOLO AS VARCHAR2(200)) AS EXP_COREP_SOLO, + CAST(EXP_FINREP_CONS AS VARCHAR2(200)) AS EXP_FINREP_CONS, + CAST(EXP_FINREP_SOLO AS VARCHAR2(200)) AS EXP_FINREP_SOLO, + CAST(EXP_LEV_CONS AS VARCHAR2(200)) AS EXP_LEV_CONS, + CAST(EXP_LEV_SOLO AS VARCHAR2(200)) AS EXP_LEV_SOLO, + CAST(EXP_LCR_CONS AS VARCHAR2(200)) AS EXP_LCR_CONS, + CAST(EXP_LCR_SOLO AS VARCHAR2(200)) AS EXP_LCR_SOLO, + CAST(EXP_NSFR_CONS AS VARCHAR2(200)) AS EXP_NSFR_CONS, + CAST(EXP_NSFR_SOLO AS VARCHAR2(200)) AS EXP_NSFR_SOLO, + CAST(CONS_CET1_AMT AS VARCHAR2(200)) AS CONS_CET1_AMT, + CAST(CONS_TIER1_AMT AS VARCHAR2(200)) AS CONS_TIER1_AMT, + CAST(CONS_TOT_CAP_AMT AS VARCHAR2(200)) AS CONS_TOT_CAP_AMT, + CAST(CONS_CET1_RATIO AS VARCHAR2(200)) AS CONS_CET1_RATIO, + CAST(CONS_TIER1_RATIO AS VARCHAR2(200)) AS CONS_TIER1_RATIO, + CAST(CONS_TOT_CAP_RATIO AS VARCHAR2(200)) AS CONS_TOT_CAP_RATIO, + CAST(CONS_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS CONS_RISK_WGHT_ASSETS, + CAST(SOLO_CET1_AMT AS VARCHAR2(200)) AS SOLO_CET1_AMT, + CAST(SOLO_TIER1_AMT AS VARCHAR2(200)) AS SOLO_TIER1_AMT, + CAST(SOLO_TOT_CAP_AMT AS VARCHAR2(200)) AS SOLO_TOT_CAP_AMT, + CAST(SOLO_CET1_RATIO AS VARCHAR2(200)) AS SOLO_CET1_RATIO, + CAST(SOLO_TIER1_RATIO AS VARCHAR2(200)) AS SOLO_TIER1_RATIO, + CAST(SOLO_TOT_CAP_RATIO AS VARCHAR2(200)) AS SOLO_TOT_CAP_RATIO, + CAST(SOLO_RISK_WGHT_ASSETS AS VARCHAR2(200)) AS SOLO_RISK_WGHT_ASSETS, + CAST(CONS_TOT_ASSETS AS VARCHAR2(200)) AS CONS_TOT_ASSETS, + CAST(SOLO_TOT_ASSETS AS VARCHAR2(200)) AS SOLO_TOT_ASSETS, + CAST(CONS_LEV_RATIO_FULL AS VARCHAR2(200)) AS CONS_LEV_RATIO_FULL, + CAST(CONS_LEV_RATIO_TRANS AS VARCHAR2(200)) AS CONS_LEV_RATIO_TRANS, + CAST(CONS_LEV_RATIO_REQ AS VARCHAR2(200)) AS CONS_LEV_RATIO_REQ, + CAST(CONS_LEV_RATIO_ADJ AS CHAR(1)) AS CONS_LEV_RATIO_ADJ, + CAST(SOLO_LEV_RATIO_FULL AS VARCHAR2(200)) AS SOLO_LEV_RATIO_FULL, + CAST(SOLO_LEV_RATIO_TRANS AS VARCHAR2(200)) AS SOLO_LEV_RATIO_TRANS, + CAST(SOLO_LEV_RATIO_REQ AS VARCHAR2(200)) AS SOLO_LEV_RATIO_REQ, + CAST(SOLO_LEV_RATIO_ADJ AS CHAR(1)) AS SOLO_LEV_RATIO_ADJ, + CAST(CONS_LC_RATIO AS VARCHAR2(200)) AS CONS_LC_RATIO, + CAST(SOLO_LC_RATIO AS VARCHAR2(200)) AS SOLO_LC_RATIO, + CAST(CONS_NSFR_RATIO AS VARCHAR2(200)) AS CONS_NSFR_RATIO, + CAST(SOLO_NSFR_RATIO AS VARCHAR2(200)) AS SOLO_NSFR_RATIO, + CAST(SUBMITTER_COMMENT AS VARCHAR2(200)) AS SUBMITTER_COMMENT, + CAST(DATACOLLECTIONCODE AS VARCHAR2(200)) AS DATACOLLECTIONCODE, + CAST(REPORTINGCYCLENAME AS VARCHAR2(200)) AS REPORTINGCYCLENAME, + CAST(REPORTINGCYCLESTATUS AS VARCHAR2(200)) AS REPORTINGCYCLESTATUS, + CAST(MODULECODE AS VARCHAR2(200)) AS MODULECODE, + CAST(MODULEVERSIONNUMBER AS NUMBER(38,10)) AS MODULEVERSIONNUMBER, + CAST(REPORTINGENTITYNAME AS VARCHAR2(200)) AS REPORTINGENTITYNAME, + CAST(ENTITYATTRIBUTECOUNTRY AS VARCHAR2(200)) AS ENTITYATTRIBUTECOUNTRY, + CAST(ENTITYGROUPENTITYNAME AS VARCHAR2(200)) AS ENTITYGROUPENTITYNAME, + CAST(OBLIGATIONMODULEREMITTANCEDATE AS VARCHAR2(30)) AS OBLIGATIONMODULEREMITTANCEDATE, + CAST(OBLIGATIONMODULEEXPECTED AS VARCHAR2(200)) AS OBLIGATIONMODULEEXPECTED, + CAST(REVALIDATIONVERSIONNUMBER AS NUMBER(38,10)) AS REVALIDATIONVERSIONNUMBER, + CAST(RECEIVEDFILESYSTEMFILENAME AS VARCHAR2(200)) AS RECEIVEDFILESYSTEMFILENAME, + CAST(OBLIGATIONSTATUSSTATUS AS VARCHAR2(200)) AS OBLIGATIONSTATUSSTATUS, + CAST(FILESTATUSSETSUBMISSIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETSUBMISSIONSTATUS, + CAST(FILESTATUSSETVALIDATIONSTATUS AS VARCHAR2(200)) AS FILESTATUSSETVALIDATIONSTATUS, + CAST(NUMBEROFERRORS AS NUMBER(38,10)) AS NUMBEROFERRORS, + CAST(NUMBEROFWARNINGS AS NUMBER(38,10)) AS NUMBEROFWARNINGS, + CAST(DELAYINDAYS AS NUMBER(38,10)) AS DELAYINDAYS, + CAST(FAILEDATTEMPTS AS NUMBER(38,10)) AS FAILEDATTEMPTS, + CAST(TABLENAME AS VARCHAR2(200)) AS TABLENAME +FROM {{ source("ou_rqsd", "RQSD_SUBA_DEVO") }} \ No newline at end of file diff --git a/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql new file mode 100644 index 0000000..6ebbbda --- /dev/null +++ b/dbt/models/views/m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS_VIEW.sql @@ -0,0 +1,9 @@ +{{ config(materialized='view', + tags=["m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS"], + alias='V_ANNEX_2_MAX_VERSION', + schema='RQSD' +) }} + +SELECT MAX(RECEIVEDFILERECEIVEDDATE) AS MAX_VERSION, REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE +FROM {{ ref('m_MOPDB_RQSD_ANNEX_2_ALL_ODS_RQSD_OBSERVATIONS') }} +GROUP BY REPORTINGENTITYCOLLECTIONUNIQUEID, OBLIGATIONMODULEREFERENCEDATE \ No newline at end of file diff --git a/dbt/profiles.yml b/dbt/profiles.yml new file mode 100644 index 0000000..9879a6e --- /dev/null +++ b/dbt/profiles.yml @@ -0,0 +1,62 @@ + +# Run following script: .\dxc\set_dbt_env_variables.ps1 +# to populate env with variables and then restart VSCode to load new variables + +mrds: + target: dev + outputs: + poc: + type: "{{ env_var('DBT_PROFILE_TARGET_TYPE_POC') }}" + host: "{{ env_var('DBT_PROFILE_TARGET_HOST_POC') }}" + user: "{{ env_var('DBT_PROFILE_TARGET_USER_POC') }}" + password: "{{ env_var('DBT_PROFILE_TARGET_PASSWORD_POC') }}" + port: "{{ env_var('DBT_PROFILE_TARGET_PORT_POC') }}" + database: "{{ env_var('DBT_DATABASE_NAME_POC') }}" + service: "{{ env_var('DBT_PROFILE_TARGET_SERVICE_NAME_POC') }}" + protocol: tcps + schema: public + threads: 4 + dev: + type: "{{ env_var('DBT_PROFILE_TARGET_TYPE_DEV') }}" + host: "{{ env_var('DBT_PROFILE_TARGET_HOST_DEV') }}" + user: "{{ env_var('DBT_PROFILE_TARGET_USER_DEV') }}" + password: "{{ env_var('DBT_PROFILE_TARGET_PASSWORD_DEV') }}" + port: "{{ env_var('DBT_PROFILE_TARGET_PORT_DEV') }}" + database: "{{ env_var('DBT_DATABASE_NAME_DEV') }}" + service: "{{ env_var('DBT_PROFILE_TARGET_SERVICE_NAME_DEV') }}" + protocol: tcps + schema: public + threads: 4 + tst: + type: "{{ env_var('DBT_PROFILE_TARGET_TYPE_TST') }}" + host: "{{ env_var('DBT_PROFILE_TARGET_HOST_TST') }}" + user: "{{ env_var('DBT_PROFILE_TARGET_USER_TST') }}" + password: "{{ env_var('DBT_PROFILE_TARGET_PASSWORD_TST') }}" + port: "{{ env_var('DBT_PROFILE_TARGET_PORT_TST') }}" + database: "{{ env_var('DBT_DATABASE_NAME_TST') }}" + service: "{{ env_var('DBT_PROFILE_TARGET_SERVICE_NAME_TST') }}" + protocol: tcps + schema: public + threads: 4 + acc: + type: "{{ env_var('DBT_PROFILE_TARGET_TYPE_ACC') }}" + host: "{{ env_var('DBT_PROFILE_TARGET_HOST_ACC') }}" + user: "{{ env_var('DBT_PROFILE_TARGET_USER_ACC') }}" + password: "{{ env_var('DBT_PROFILE_TARGET_PASSWORD_ACC') }}" + port: "{{ env_var('DBT_PROFILE_TARGET_PORT_ACC') }}" + database: "{{ env_var('DBT_DATABASE_NAME_ACC') }}" + service: "{{ env_var('DBT_PROFILE_TARGET_SERVICE_NAME_ACC') }}" + protocol: tcps + schema: public + threads: 4 + prd: + type: "{{ env_var('DBT_PROFILE_TARGET_TYPE_PRD') }}" + host: "{{ env_var('DBT_PROFILE_TARGET_HOST_PRD') }}" + user: "{{ env_var('DBT_PROFILE_TARGET_USER_PRD') }}" + password: "{{ env_var('DBT_PROFILE_TARGET_PASSWORD_PRD') }}" + port: "{{ env_var('DBT_PROFILE_TARGET_PORT_PRD') }}" + database: "{{ env_var('DBT_DATABASE_NAME_PRD') }}" + service: "{{ env_var('DBT_PROFILE_TARGET_SERVICE_NAME_PRD') }}" + protocol: tcps + schema: public + threads: 4 \ No newline at end of file diff --git a/dbt/seeds/.gitkeep b/dbt/seeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt/snapshots/.gitkeep b/dbt/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt/snapshots/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot.sql b/dbt/snapshots/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot.sql new file mode 100644 index 0000000..7266cbb --- /dev/null +++ b/dbt/snapshots/m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot.sql @@ -0,0 +1,346 @@ +-- This model updates the target table DW_RAR.NH_ASSET_snapshot by adding new and the updated rows. +-- If the modified version of the already existing row was added the column TO_DATE is getting updated with the current date. +-- Wether the row is new or existing one is determined by ASSET_KEY and FROM_DATE columns. +-- In addition to that 4 more columns are updated in the target table: DBT_SCD_ID, DBT_UPDATED_AT, DBT_VALID_FROM,DBT_VALID TO +-- The model is a part of the mapping m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD. +-- Pre-/post-hook log data into the control tables. +-- A_TASK_HISTORY_TARGET_insert tag ensures logging of the target table data into the control tables. + +{% snapshot NH_ASSET_snapshot %} + +-- Note: the following config part for the renaming of the dbt default columns is only available from dbt-oracle version 1.9 and higher +-- and therefore excluded here: +-- snapshot_meta_column_names={ +-- 'dbt_valid_from': 'FROM_DATE', +-- 'dbt_valid_to': 'TO_DATE'}, + + {{config( + target_schema='DW_RAR', + unique_key='ASSET_KEY', + strategy='timestamp', + updated_at='FROM_DATE', + tags = ['m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_snapshot'], + ) + }} + +-- Select the required for the target model columns. +-- Join with the target model NH_ASSET_test. +-- Compare the incoming and the existing data in the target model: keep only new and modified rows. +-- update the TO_DATE and FROM_DATE columns. + SELECT SQ.ASSET_KEY, + SQ.IS_SECURED, + SQ.CLASSIFICATION, + SQ.IS_EXTENDIBLE, + SQ.EA_IS_EMERGENCY_COLLATERAL, + SQ.TMS_GROUP_PATH, + SQ.TMS_GROUP_PATH_NAME, + SQ.ISSUE_DATE, + SQ.MATURITY_LAST_REDEMPTION_DATE, + SQ.TMS_ACTIVE_SINCE, + SQ.TMS_ACTIVE_UNTIL, + SQ.CURRENCY, + SQ.CURRENCY_2, + SQ.INTEREST_RATE, + SQ.TMS_TYPE, + SQ.PERIOD_ID, + SQ.PRICING_TYPE, + SQ.ASSET_BBG_NAME, + SQ.BBG_BM_IDX, + SQ.BBG_FLT_BM_MULTIPLIER, + SQ.BBG_COLLAT_TYPE, + SQ.BBG_MARKET_ISSUE, + SQ.BBG_MATURITY_TYPE, + SQ.COUPON_TYPE, + SQ.POOL_FACTOR, + SQ.BBG_FLT_SPREAD, + SQ.CSDB_REDEMPTION_TYPE, + SQ.CSDB_REDEMPTION_FREQ, + SQ.CSDB_REDEMPTION_PRICE, + SQ.CSDB_ISSUE_PRICE, + SQ.CSDB_STATUS, + SQ.UOC_INTEREST_PAYMENT_TYPE, + SQ.EA_CSD, + SQ.C2D_ASSET_TYPE, + SQ.EA_COUPON_TYPE, + SQ.EA_REF_MARKET, + SQ.EA_CRD_OR_EQUIV, + SQ.EA_IS_ABS_LLD, + SQ.EA_IS_ABS_LLD_DATE, + SQ.EA_ABS_SURVEIL_DATE, + SQ.EA_ABS_TYPE, + SQ.EA_CEPH_IS_INFLAT, + SQ.EA_CEPH_IS_CALL_PUT, + SQ.EA_CEPH_IS_SINKABLE, + SQ.EA_CEPH_COMMENT, + SQ.EA_IS_OUTRIGHT_PURCHASE, + SQ.EA_LIQ_CLASS, + SQ.EA_DENOMINATION, + SQ.EA_NAME, + SQ.EA_VALIDATION_STATUS, + SQ.EA_NCB_COMMENT, + SQ.EA_ISSUANCE_DATE, + SQ.C2D_MATURITY_DATE, + SQ.MATURITY_DATE, + SQ.UOC_BUCKET_SIZE, + SQ.IS_ELIGIBLE_VIA_GUARANTOR, + SQ.UOC_RESIDUAL_MATURITY, + SQ.EA_SENDER_CODE, + SQ.EA_NON_OWN_USE_HAIRCUT, + SQ.ACCOUNTING_TYPE, + SQ.GROUPING_VALUE, + SQ.SPREAD_TYPE, + SQ.TMS_ELIGIBILITY_TYPE, + SQ.REF_AMOUNT_BUCKET, + SQ.ISSUANCE_TYPE, + SQ.BRANCH6_ID, + SQ.BRANCH7_ID, + SQ.BRANCH8_ID, + SQ.BRANCH9_ID, + SQ.TMS_NAME, + SQ.CSDB_CSD, + SQ.CSDB_SECURITISATION_TYPE, + SQ.CFI_CLASSIFICATION, + SQ.INSTRUMENT_ESA_95_CLASS, + SQ.INSTR_ESA_95_CLASS_VAL_TYPE, + SQ.INSTRUMENT_ESA_2010_CLASS, + SQ.REDEMPTION_CURRENCY, + SQ.CSDB_SUPPL_INFO, + SQ.CAP, + SQ.REFERENCERATE, + SQ.REFERENCERATECOMMENT, + SQ.BBG_FLOATER, + SQ.BBG_INDUSTRY_GROUP, + SQ.CSDB_ESG1, + SQ.CSDB_ESG2, + SQ.CSDB_ESG3, + SQ.REU_SECURITY_DESCRIPTION, + SQ.REU_ASSET_CATEGORY, + SQ.REU_ASSET_CATEGORY_DESCRIPTION, + SQ.REU_ASSET_TYPE, + SQ.REU_ASSET_TYPE_DESCRIPTION, + SQ.REU_ASSET_SUBTYPE, + SQ.REU_ASSET_SUBTYPE_DESCRIPTION, + SQ.REU_ISO_CURRENCY, + SQ.BBG_DES_NOTES, + SQ.BBG_INDUSTRY_SUBGROUP, + SQ.BBG_SECURITY_TYP, + SQ.BBG_SECURITY_TYP2, + SQ.UOC_RESET_PERIOD_MORE_ONE_YEAR, + SQ.EA_EXTENSION_OPTION_EXERCISED, + SQ.EA_EFFECTIVE_MATURITY_DATE, + SQ.EA_COVERED_BOND_OU, + SQ.EA_OWN_USE_HAIRCUT, + SQ.EA_CQS, + SQ.EA_EXTENDIBLE, + SQ.EA_EXTENSION_TRIGGER, + SQ.EA_EXTENDED_MATURITY_DATE, + SQ.EA_ABS_PUBL_SURVEIL_DATE2, + SQ.BBG_TICKER, + SQ.BBG_STRIP_TYPE, + SQ.BBG_EXCHANGE_CODE, + SQ.BBG_PRVT_PLACE, + SQ.BBG_PUTABLE, + SQ.BBG_ISSUER_INDUSTRY, + SQ.BBG_INFLATION_LINKED_INDICATOR, + SQ.CSDB_IS_PRIVATE_PLACEMENT, + SQ.CLASSIFICATION_C2D, + SQ.CLASSIFICATION_TMS, + SQ.ISSUE_DATE_TMS, + SQ.ISSUE_DATE_CSDB, + SQ.CURRENCY_TMS, + SQ.CURRENCY_MDP, + SQ.CURRENCY_CSDB, + SQ.COUPON_TYPE_MDP, + SQ.COUPON_TYPE_CSDB, + SQ.POOL_FACTOR_CEPH, + SQ.POOL_FACTOR_CSDB, + SQ.MATURITY_DATE_TMS, + SQ.MATURITY_DATE_MDP, + SQ.CP_PROGRAMME, + SQ.C2D_ELA_ABS_TYPE, + SQ.C2D_EA_ABS_TYPE, + SQ.C2D_EA_MATURITY_DATE, + SQ.C2D_UC_MATURITY_DATE, + SQ.C2D_UC_ASSET_TYPE, + SQ.C2D_EA_ASSET_TYPE, + SQ.C2D_ELA_ASSET_TYPE, + SQ.C2D_UC_DENOMINATION, + SQ.C2D_EA_DENOMINATION, + SQ.C2D_UC_OA_ID, + SQ.C2D_UC_CONTRACT_ID, + SQ.C2D_UC_INSTRMNT_ID, + SQ.MDP_CMA_DEFAULTED_DATE, + SQ.MDP_CMA_DEFAULTED_STANDARD, + SQ.MDP_CMA_QUOTE_CONVENTION, + SQ.MDP_CMA_RESTRUCTURING_TYPE, + SQ.MDP_CMA_MATURITY_DATE, + SQ.MDP_CMA_TENOR, + SQ.MDP_CMA_SENIORITY, + SQ.MDP_CMA_INSTRUMENT_TYPE, + SQ.MDP_CMA_CURRENCY, + SQ.C2D_EA_REFERENCE_RATE, + SQ.C2D_EA_GREEN_SUSTAINABLE, + SQ.TMS_FUTURES_MATURITY_BUCKET, + SQ.BBG_MTY_YEARS_TDY, + CURRENT_DATE AS FROM_DATE +FROM {{ ref("m_DWH_ASSET_NH_ASSET_TE_ASSET_LOAD_SQ") }} SQ +LEFT JOIN DW_RAR.NH_ASSET_snapshot LKP ON LKP.ASSET_KEY=SQ.ASSET_KEY AND LKP.dbt_valid_to IS NULL +WHERE LKP.ASSET_KEY IS NULL OR + (LKP.ASSET_KEY IS NOT NULL AND + COALESCE(SQ.IS_SECURED, '>>NULL<<') != COALESCE(LKP.IS_SECURED, '>>NULL<<') OR + COALESCE(SQ.CLASSIFICATION, '>>NULL<<') != COALESCE(LKP.CLASSIFICATION, '>>NULL<<') OR + COALESCE(SQ.IS_EXTENDIBLE, '>>NULL<<') != COALESCE(LKP.IS_EXTENDIBLE, '>>NULL<<') OR + COALESCE(SQ.EA_IS_EMERGENCY_COLLATERAL, '>>NULL<<') != COALESCE(LKP.EA_IS_EMERGENCY_COLLATERAL, '>>NULL<<') OR + COALESCE(SQ.TMS_GROUP_PATH, '>>NULL<<') != COALESCE(LKP.TMS_GROUP_PATH, '>>NULL<<') OR + COALESCE(SQ.TMS_GROUP_PATH_NAME, '>>NULL<<') != COALESCE(LKP.TMS_GROUP_PATH_NAME, '>>NULL<<') OR + COALESCE(SQ.ISSUE_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.ISSUE_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.MATURITY_LAST_REDEMPTION_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.MATURITY_LAST_REDEMPTION_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.TMS_ACTIVE_SINCE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.TMS_ACTIVE_SINCE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.TMS_ACTIVE_UNTIL, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.TMS_ACTIVE_UNTIL, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.CURRENCY, '>>NULL<<') != COALESCE(LKP.CURRENCY, '>>NULL<<') OR + COALESCE(SQ.CURRENCY_2, '>>NULL<<') != COALESCE(LKP.CURRENCY_2, '>>NULL<<') OR + COALESCE(SQ.INTEREST_RATE, '>>NULL<<') != COALESCE(LKP.INTEREST_RATE, '>>NULL<<') OR + COALESCE(SQ.TMS_TYPE, '>>NULL<<') != COALESCE(LKP.TMS_TYPE, '>>NULL<<') OR + COALESCE(SQ.PERIOD_ID, '>>NULL<<') != COALESCE(LKP.PERIOD_ID, '>>NULL<<') OR + COALESCE(SQ.PRICING_TYPE, '>>NULL<<') != COALESCE(LKP.PRICING_TYPE, '>>NULL<<') OR + COALESCE(SQ.ASSET_BBG_NAME, '>>NULL<<') != COALESCE(LKP.ASSET_BBG_NAME, '>>NULL<<') OR + COALESCE(SQ.BBG_BM_IDX, '>>NULL<<') != COALESCE(LKP.BBG_BM_IDX, '>>NULL<<') OR + COALESCE(SQ.BBG_FLT_BM_MULTIPLIER, 0) != COALESCE(LKP.BBG_FLT_BM_MULTIPLIER, 0) OR + COALESCE(SQ.BBG_COLLAT_TYPE, '>>NULL<<') != COALESCE(LKP.BBG_COLLAT_TYPE, '>>NULL<<') OR + COALESCE(SQ.BBG_MARKET_ISSUE, '>>NULL<<') != COALESCE(LKP.BBG_MARKET_ISSUE, '>>NULL<<') OR + COALESCE(SQ.BBG_MATURITY_TYPE, '>>NULL<<') != COALESCE(LKP.BBG_MATURITY_TYPE, '>>NULL<<') OR + COALESCE(SQ.COUPON_TYPE, '>>NULL<<') != COALESCE(LKP.COUPON_TYPE, '>>NULL<<') OR + COALESCE(SQ.POOL_FACTOR, 0) != COALESCE(LKP.POOL_FACTOR, 0) OR + COALESCE(SQ.BBG_FLT_SPREAD, 0) != COALESCE(LKP.BBG_FLT_SPREAD, 0) OR + COALESCE(SQ.CSDB_REDEMPTION_TYPE, '>>NULL<<') != COALESCE(LKP.CSDB_REDEMPTION_TYPE, '>>NULL<<') OR + COALESCE(SQ.CSDB_REDEMPTION_FREQ, '>>NULL<<') != COALESCE(LKP.CSDB_REDEMPTION_FREQ, '>>NULL<<') OR + COALESCE(SQ.CSDB_REDEMPTION_PRICE, 0) != COALESCE(LKP.CSDB_REDEMPTION_PRICE, 0) OR + COALESCE(SQ.CSDB_ISSUE_PRICE, 0) != COALESCE(LKP.CSDB_ISSUE_PRICE, 0) OR + COALESCE(SQ.CSDB_STATUS, '>>NULL<<') != COALESCE(LKP.CSDB_STATUS, '>>NULL<<') OR + COALESCE(SQ.UOC_INTEREST_PAYMENT_TYPE, '>>NULL<<') != COALESCE(LKP.UOC_INTEREST_PAYMENT_TYPE, '>>NULL<<') OR + COALESCE(SQ.EA_CSD, '>>NULL<<') != COALESCE(LKP.EA_CSD, '>>NULL<<') OR + COALESCE(SQ.C2D_ASSET_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_ASSET_TYPE, '>>NULL<<') OR + COALESCE(SQ.EA_COUPON_TYPE, '>>NULL<<') != COALESCE(LKP.EA_COUPON_TYPE, '>>NULL<<') OR + COALESCE(SQ.EA_REF_MARKET, '>>NULL<<') != COALESCE(LKP.EA_REF_MARKET, '>>NULL<<') OR + COALESCE(SQ.EA_CRD_OR_EQUIV, '>>NULL<<') != COALESCE(LKP.EA_CRD_OR_EQUIV, '>>NULL<<') OR + COALESCE(SQ.EA_IS_ABS_LLD, '>>NULL<<') != COALESCE(LKP.EA_IS_ABS_LLD, '>>NULL<<') OR + COALESCE(SQ.EA_IS_ABS_LLD_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.EA_IS_ABS_LLD_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.EA_ABS_SURVEIL_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.EA_ABS_SURVEIL_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.EA_ABS_TYPE, '>>NULL<<') != COALESCE(LKP.EA_ABS_TYPE, '>>NULL<<') OR + COALESCE(SQ.EA_CEPH_IS_INFLAT, '>>NULL<<') != COALESCE(LKP.EA_CEPH_IS_INFLAT, '>>NULL<<') OR + COALESCE(SQ.EA_CEPH_IS_CALL_PUT, '>>NULL<<') != COALESCE(LKP.EA_CEPH_IS_CALL_PUT, '>>NULL<<') OR + COALESCE(SQ.EA_CEPH_IS_SINKABLE, '>>NULL<<') != COALESCE(LKP.EA_CEPH_IS_SINKABLE, '>>NULL<<') OR + COALESCE(SQ.EA_CEPH_COMMENT, '>>NULL<<') != COALESCE(LKP.EA_CEPH_COMMENT, '>>NULL<<') OR + COALESCE(SQ.EA_IS_OUTRIGHT_PURCHASE, '>>NULL<<') != COALESCE(LKP.EA_IS_OUTRIGHT_PURCHASE, '>>NULL<<') OR + COALESCE(SQ.EA_LIQ_CLASS, '>>NULL<<') != COALESCE(LKP.EA_LIQ_CLASS, '>>NULL<<') OR + COALESCE(SQ.EA_DENOMINATION, '>>NULL<<') != COALESCE(LKP.EA_DENOMINATION, '>>NULL<<') OR + COALESCE(SQ.EA_NAME, '>>NULL<<') != COALESCE(LKP.EA_NAME, '>>NULL<<') OR + COALESCE(SQ.EA_VALIDATION_STATUS, '>>NULL<<') != COALESCE(LKP.EA_VALIDATION_STATUS, '>>NULL<<') OR + COALESCE(SQ.EA_NCB_COMMENT, '>>NULL<<') != COALESCE(LKP.EA_NCB_COMMENT, '>>NULL<<') OR + COALESCE(SQ.EA_ISSUANCE_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.EA_ISSUANCE_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.C2D_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.C2D_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.UOC_BUCKET_SIZE, '>>NULL<<') != COALESCE(LKP.UOC_BUCKET_SIZE, '>>NULL<<') OR + COALESCE(SQ.IS_ELIGIBLE_VIA_GUARANTOR, '>>NULL<<') != COALESCE(LKP.IS_ELIGIBLE_VIA_GUARANTOR, '>>NULL<<') OR + COALESCE(SQ.UOC_RESIDUAL_MATURITY, '>>NULL<<') != COALESCE(LKP.UOC_RESIDUAL_MATURITY, '>>NULL<<') OR + COALESCE(SQ.EA_SENDER_CODE, '>>NULL<<') != COALESCE(LKP.EA_SENDER_CODE, '>>NULL<<') OR + COALESCE(SQ.EA_NON_OWN_USE_HAIRCUT, 0) != COALESCE(LKP.EA_NON_OWN_USE_HAIRCUT, 0) OR + COALESCE(SQ.ACCOUNTING_TYPE, '>>NULL<<') != COALESCE(LKP.ACCOUNTING_TYPE, '>>NULL<<') OR + COALESCE(SQ.GROUPING_VALUE, '>>NULL<<') != COALESCE(LKP.GROUPING_VALUE, '>>NULL<<') OR + COALESCE(SQ.SPREAD_TYPE, '>>NULL<<') != COALESCE(LKP.SPREAD_TYPE, '>>NULL<<') OR + COALESCE(SQ.TMS_ELIGIBILITY_TYPE, '>>NULL<<') != COALESCE(LKP.TMS_ELIGIBILITY_TYPE, '>>NULL<<') OR + COALESCE(SQ.REF_AMOUNT_BUCKET, '>>NULL<<') != COALESCE(LKP.REF_AMOUNT_BUCKET, '>>NULL<<') OR + COALESCE(SQ.ISSUANCE_TYPE, '>>NULL<<') != COALESCE(LKP.ISSUANCE_TYPE, '>>NULL<<') OR + COALESCE(SQ.BRANCH6_ID, '>>NULL<<') != COALESCE(LKP.BRANCH6_ID, '>>NULL<<') OR + COALESCE(SQ.BRANCH7_ID, '>>NULL<<') != COALESCE(LKP.BRANCH7_ID, '>>NULL<<') OR + COALESCE(SQ.BRANCH8_ID, '>>NULL<<') != COALESCE(LKP.BRANCH8_ID, '>>NULL<<') OR + COALESCE(SQ.BRANCH9_ID, '>>NULL<<') != COALESCE(LKP.BRANCH9_ID, '>>NULL<<') OR + COALESCE(SQ.TMS_NAME, '>>NULL<<') != COALESCE(LKP.TMS_NAME, '>>NULL<<') OR + COALESCE(SQ.CSDB_CSD, '>>NULL<<') != COALESCE(LKP.CSDB_CSD, '>>NULL<<') OR + COALESCE(SQ.CSDB_SECURITISATION_TYPE, '>>NULL<<') != COALESCE(LKP.CSDB_SECURITISATION_TYPE, '>>NULL<<') OR + COALESCE(SQ.CFI_CLASSIFICATION, '>>NULL<<') != COALESCE(LKP.CFI_CLASSIFICATION, '>>NULL<<') OR + COALESCE(SQ.INSTRUMENT_ESA_95_CLASS, '>>NULL<<') != COALESCE(LKP.INSTRUMENT_ESA_95_CLASS, '>>NULL<<') OR + COALESCE(SQ.INSTR_ESA_95_CLASS_VAL_TYPE, '>>NULL<<') != COALESCE(LKP.INSTR_ESA_95_CLASS_VAL_TYPE, '>>NULL<<') OR + COALESCE(SQ.INSTRUMENT_ESA_2010_CLASS, '>>NULL<<') != COALESCE(LKP.INSTRUMENT_ESA_2010_CLASS, '>>NULL<<') OR + COALESCE(SQ.REDEMPTION_CURRENCY, '>>NULL<<') != COALESCE(LKP.REDEMPTION_CURRENCY, '>>NULL<<') OR + COALESCE(SQ.CSDB_SUPPL_INFO, '>>NULL<<') != COALESCE(LKP.CSDB_SUPPL_INFO, '>>NULL<<') OR + COALESCE(SQ.CAP, '>>NULL<<') != COALESCE(LKP.CAP, '>>NULL<<') OR + COALESCE(SQ.REFERENCERATE, '>>NULL<<') != COALESCE(LKP.REFERENCERATE, '>>NULL<<') OR + COALESCE(SQ.REFERENCERATECOMMENT, '>>NULL<<') != COALESCE(LKP.REFERENCERATECOMMENT, '>>NULL<<') OR + COALESCE(SQ.BBG_FLOATER, '>>NULL<<') != COALESCE(LKP.BBG_FLOATER, '>>NULL<<') OR + COALESCE(SQ.BBG_INDUSTRY_GROUP, '>>NULL<<') != COALESCE(LKP.BBG_INDUSTRY_GROUP, '>>NULL<<') OR + COALESCE(SQ.CSDB_ESG1, '>>NULL<<') != COALESCE(LKP.CSDB_ESG1, '>>NULL<<') OR + COALESCE(SQ.CSDB_ESG2, '>>NULL<<') != COALESCE(LKP.CSDB_ESG2, '>>NULL<<') OR + COALESCE(SQ.CSDB_ESG3, '>>NULL<<') != COALESCE(LKP.CSDB_ESG3, '>>NULL<<') OR + COALESCE(SQ.REU_SECURITY_DESCRIPTION, '>>NULL<<') != COALESCE(LKP.REU_SECURITY_DESCRIPTION, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_CATEGORY, '>>NULL<<') != COALESCE(LKP.REU_ASSET_CATEGORY, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_CATEGORY_DESCRIPTION, '>>NULL<<') != COALESCE(LKP.REU_ASSET_CATEGORY_DESCRIPTION, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_TYPE, '>>NULL<<') != COALESCE(LKP.REU_ASSET_TYPE, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_TYPE_DESCRIPTION, '>>NULL<<') != COALESCE(LKP.REU_ASSET_TYPE_DESCRIPTION, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_SUBTYPE, '>>NULL<<') != COALESCE(LKP.REU_ASSET_SUBTYPE, '>>NULL<<') OR + COALESCE(SQ.REU_ASSET_SUBTYPE_DESCRIPTION, '>>NULL<<') != COALESCE(LKP.REU_ASSET_SUBTYPE_DESCRIPTION, '>>NULL<<') OR + COALESCE(SQ.REU_ISO_CURRENCY, '>>NULL<<') != COALESCE(LKP.REU_ISO_CURRENCY, '>>NULL<<') OR + COALESCE(SQ.BBG_DES_NOTES, '>>NULL<<') != COALESCE(LKP.BBG_DES_NOTES, '>>NULL<<') OR + COALESCE(SQ.BBG_INDUSTRY_SUBGROUP, '>>NULL<<') != COALESCE(LKP.BBG_INDUSTRY_SUBGROUP, '>>NULL<<') OR + COALESCE(SQ.BBG_SECURITY_TYP, '>>NULL<<') != COALESCE(LKP.BBG_SECURITY_TYP, '>>NULL<<') OR + COALESCE(SQ.BBG_SECURITY_TYP2, '>>NULL<<') != COALESCE(LKP.BBG_SECURITY_TYP2, '>>NULL<<') OR + COALESCE(SQ.UOC_RESET_PERIOD_MORE_ONE_YEAR, '>>NULL<<') != COALESCE(LKP.UOC_RESET_PERIOD_MORE_ONE_YEAR, '>>NULL<<') OR + COALESCE(SQ.EA_EXTENSION_OPTION_EXERCISED, '>>NULL<<') != COALESCE(LKP.EA_EXTENSION_OPTION_EXERCISED, '>>NULL<<') OR + COALESCE(SQ.EA_EFFECTIVE_MATURITY_DATE, TO_DATE('01/01/0001','mm/dd/yyyy')) != COALESCE(LKP.EA_EFFECTIVE_MATURITY_DATE, TO_DATE('01/01/0001','mm/dd/yyyy')) OR + COALESCE(SQ.EA_COVERED_BOND_OU, '>>NULL<<') != COALESCE(LKP.EA_COVERED_BOND_OU, '>>NULL<<') OR + COALESCE(SQ.EA_OWN_USE_HAIRCUT, 0) != COALESCE(LKP.EA_OWN_USE_HAIRCUT, 0) OR + COALESCE(SQ.EA_CQS, 0) != COALESCE(LKP.EA_CQS, 0) OR + COALESCE(SQ.EA_EXTENDIBLE, '>>NULL<<') != COALESCE(LKP.EA_EXTENDIBLE, '>>NULL<<') OR + COALESCE(SQ.EA_EXTENSION_TRIGGER, '>>NULL<<') != COALESCE(LKP.EA_EXTENSION_TRIGGER, '>>NULL<<') OR + COALESCE(SQ.EA_EXTENDED_MATURITY_DATE, TO_DATE('01/01/0001','mm/dd/yyyy')) != COALESCE(LKP.EA_EXTENDED_MATURITY_DATE, TO_DATE('01/01/0001','mm/dd/yyyy')) OR + COALESCE(SQ.EA_ABS_PUBL_SURVEIL_DATE2, TO_DATE('01/01/0001','mm/dd/yyyy')) != COALESCE(LKP.EA_ABS_PUBL_SURVEIL_DATE2, TO_DATE('01/01/0001','mm/dd/yyyy')) OR + COALESCE(SQ.BBG_TICKER, '>>NULL<<') != COALESCE(LKP.BBG_TICKER, '>>NULL<<') OR + COALESCE(SQ.BBG_STRIP_TYPE, '>>NULL<<') != COALESCE(LKP.BBG_STRIP_TYPE, '>>NULL<<') OR + COALESCE(SQ.BBG_EXCHANGE_CODE, '>>NULL<<') != COALESCE(LKP.BBG_EXCHANGE_CODE, '>>NULL<<') OR + COALESCE(SQ.BBG_PRVT_PLACE, '>>NULL<<') != COALESCE(LKP.BBG_PRVT_PLACE, '>>NULL<<') OR + COALESCE(SQ.BBG_PUTABLE, '>>NULL<<') != COALESCE(LKP.BBG_PUTABLE, '>>NULL<<') OR + COALESCE(SQ.BBG_ISSUER_INDUSTRY, '>>NULL<<') != COALESCE(LKP.BBG_ISSUER_INDUSTRY, '>>NULL<<') OR + COALESCE(SQ.BBG_INFLATION_LINKED_INDICATOR, '>>NULL<<') != COALESCE(LKP.BBG_INFLATION_LINKED_INDICATOR, '>>NULL<<') OR + COALESCE(SQ.CSDB_IS_PRIVATE_PLACEMENT, '>>NULL<<') != COALESCE(LKP.CSDB_IS_PRIVATE_PLACEMENT, '>>NULL<<') OR + COALESCE(SQ.CLASSIFICATION_C2D, '>>NULL<<') != COALESCE(LKP.CLASSIFICATION_C2D, '>>NULL<<') OR + COALESCE(SQ.CLASSIFICATION_TMS, '>>NULL<<') != COALESCE(LKP.CLASSIFICATION_TMS, '>>NULL<<') OR + COALESCE(SQ.ISSUE_DATE_TMS, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.ISSUE_DATE_TMS, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.ISSUE_DATE_CSDB, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.ISSUE_DATE_CSDB, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.CURRENCY_TMS, '>>NULL<<') != COALESCE(LKP.CURRENCY_TMS, '>>NULL<<') OR + COALESCE(SQ.CURRENCY_MDP, '>>NULL<<') != COALESCE(LKP.CURRENCY_MDP, '>>NULL<<') OR + COALESCE(SQ.CURRENCY_CSDB, '>>NULL<<') != COALESCE(LKP.CURRENCY_CSDB, '>>NULL<<') OR + COALESCE(SQ.COUPON_TYPE_MDP, '>>NULL<<') != COALESCE(LKP.COUPON_TYPE_MDP, '>>NULL<<') OR + COALESCE(SQ.COUPON_TYPE_CSDB, '>>NULL<<') != COALESCE(LKP.COUPON_TYPE_CSDB, '>>NULL<<') OR + COALESCE(SQ.POOL_FACTOR_CEPH, 0) != COALESCE(LKP.POOL_FACTOR_CEPH, 0) OR + COALESCE(SQ.POOL_FACTOR_CSDB, 0) != COALESCE(LKP.POOL_FACTOR_CSDB, 0) OR + COALESCE(SQ.MATURITY_DATE_TMS, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.MATURITY_DATE_TMS, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.MATURITY_DATE_MDP, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.MATURITY_DATE_MDP, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.CP_PROGRAMME, '>>NULL<<') != COALESCE(LKP.CP_PROGRAMME, '>>NULL<<') OR + COALESCE(SQ.C2D_ELA_ABS_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_ELA_ABS_TYPE, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_ABS_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_EA_ABS_TYPE, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.C2D_EA_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.C2D_UC_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) != COALESCE(LKP.C2D_UC_MATURITY_DATE, TO_DATE('01/01/2001','mm/dd/yyyy')) OR + COALESCE(SQ.C2D_UC_ASSET_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_UC_ASSET_TYPE, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_ASSET_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_EA_ASSET_TYPE, '>>NULL<<') OR + COALESCE(SQ.C2D_ELA_ASSET_TYPE, '>>NULL<<') != COALESCE(LKP.C2D_ELA_ASSET_TYPE, '>>NULL<<') OR + COALESCE(SQ.C2D_UC_DENOMINATION, '>>NULL<<') != COALESCE(LKP.C2D_UC_DENOMINATION, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_DENOMINATION, '>>NULL<<') != COALESCE(LKP.C2D_EA_DENOMINATION, '>>NULL<<') OR + COALESCE(SQ.C2D_UC_OA_ID, '>>NULL<<') != COALESCE(LKP.C2D_UC_OA_ID, '>>NULL<<') OR + COALESCE(SQ.C2D_UC_CONTRACT_ID, '>>NULL<<') != COALESCE(LKP.C2D_UC_CONTRACT_ID, '>>NULL<<') OR + COALESCE(SQ.C2D_UC_INSTRMNT_ID, '>>NULL<<') != COALESCE(LKP.C2D_UC_INSTRMNT_ID, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_DEFAULTED_DATE, TO_DATE('01/01/2001' ,'mm/dd/yyyy')) != COALESCE(LKP.MDP_CMA_DEFAULTED_DATE, TO_DATE('01/01/2001' ,'mm/dd/yyyy')) OR + COALESCE(SQ.MDP_CMA_DEFAULTED_STANDARD, '>>NULL<<') != COALESCE(LKP.MDP_CMA_DEFAULTED_STANDARD, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_QUOTE_CONVENTION, '>>NULL<<') != COALESCE(LKP.MDP_CMA_QUOTE_CONVENTION, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_RESTRUCTURING_TYPE, '>>NULL<<') != COALESCE(LKP.MDP_CMA_RESTRUCTURING_TYPE, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_MATURITY_DATE, TO_DATE('01/01/2001' ,'mm/dd/yyyy')) != COALESCE(LKP.MDP_CMA_MATURITY_DATE, TO_DATE('01/01/2001' ,'mm/dd/yyyy')) OR + COALESCE(SQ.MDP_CMA_TENOR, -999999999) != COALESCE(LKP.MDP_CMA_TENOR, -999999999) OR + COALESCE(SQ.BBG_MTY_YEARS_TDY, -999999999) != COALESCE(LKP.BBG_MTY_YEARS_TDY, -999999999) OR + COALESCE(SQ.MDP_CMA_SENIORITY, '>>NULL<<') != COALESCE(LKP.MDP_CMA_SENIORITY, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_INSTRUMENT_TYPE, '>>NULL<<') != COALESCE(LKP.MDP_CMA_INSTRUMENT_TYPE, '>>NULL<<') OR + COALESCE(SQ.MDP_CMA_CURRENCY, '>>NULL<<') != COALESCE(LKP.MDP_CMA_CURRENCY, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_REFERENCE_RATE, '>>NULL<<') != COALESCE(LKP.C2D_EA_REFERENCE_RATE, '>>NULL<<') OR + COALESCE(SQ.C2D_EA_GREEN_SUSTAINABLE, '>>NULL<<') != COALESCE(LKP.C2D_EA_GREEN_SUSTAINABLE, '>>NULL<<') OR + COALESCE(SQ.TMS_FUTURES_MATURITY_BUCKET, '>>NULL<<') != COALESCE(LKP.TMS_FUTURES_MATURITY_BUCKET, '>>NULL<<') + ) + +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_BRANCH.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_BRANCH.sql new file mode 100644 index 0000000..05e0ef4 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_BRANCH.sql @@ -0,0 +1,35 @@ +{% snapshot m_DWH_FXCD_NH_F_BRANCH %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_BRANCH_MARS', + unique_key=['BRANCH_ID'], + strategy='check', + check_cols=['ENTITY_ID', + 'CTP_ID'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_BRANCH"] + ) +}} +SELECT + ((SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1)) AS A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CAST(BRANCH_ID AS NUMBER(28,0)) AS BRANCH_ID, + CAST(ENTITY_ID AS NUMBER(28,0)) AS ENTITY_ID, + CAST(CTP_ID AS NUMBER(28,0)) AS CTP_ID +FROM {{ source('ods','F_BRANCH') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_BRANCH",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER.sql new file mode 100644 index 0000000..23ec069 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER.sql @@ -0,0 +1,51 @@ +{% snapshot m_DWH_FXCD_NH_F_CLEARER %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CLEARER_MARS', + unique_key=['CLEARER_ID'], + strategy='check', + check_cols=['ENTITY_ID', + 'ELIGIBILITY_OF_FLAG', + 'ELIGIBILITY_FR_FLAG', + 'ACTIVE_FLAG', + 'OVERALL_OF_LIMIT_AMT', + 'CASH_OF_LIMIT_AMT', + 'SECURITIES_OF_LIMIT_AMT', + 'OVERALL_FR_LIMIT_AMT', + 'CASH_FR_LIMIT_AMT', + 'SECURITIES_FR_LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CLEARER"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CLEARER_ID, + ENTITY_ID, + {{clean_boolean_fxcd('ELIGIBILITY_OF_FLAG')}} AS ELIGIBILITY_OF_FLAG, + {{clean_boolean_fxcd('ELIGIBILITY_FR_FLAG')}} AS ELIGIBILITY_FR_FLAG, + {{clean_boolean_fxcd('ACTIVE_FLAG')}} AS ACTIVE_FLAG, + OVERALL_OF_LIMIT_AMT, + CASH_OF_LIMIT_AMT, + SECURITIES_OF_LIMIT_AMT, + OVERALL_FR_LIMIT_AMT, + CASH_FR_LIMIT_AMT, + SECURITIES_FR_LIMIT_AMT +FROM {{ source('ods', 'F_CLEARER') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CLEARER",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT.sql new file mode 100644 index 0000000..098b03f --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT.sql @@ -0,0 +1,39 @@ +{% snapshot m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CLEARER_NCB_LIMIT_MARS', + unique_key=['CLEARER_ID', + 'COUNTRY_ID'], + strategy='check', + check_cols=['OVERALL_LIMIT_AMT', + 'CASH_LIMIT_AMT', + 'SECURITIES_LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CLEARER_NCB_LIMIT"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CLEARER_ID, + COUNTRY_ID, + OVERALL_LIMIT_AMT, + CASH_LIMIT_AMT, + SECURITIES_LIMIT_AMT +FROM {{ source('ods', 'F_CLEARER_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CLEARER_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CONSTANT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CONSTANT.sql new file mode 100644 index 0000000..c43944b --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CONSTANT.sql @@ -0,0 +1,35 @@ +{% snapshot m_DWH_FXCD_NH_F_CONSTANT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CONSTANT_MARS', + unique_key=['CONSTANT_NAME'], + strategy='check', + check_cols=['CONSTANT_VALUE', + 'CONSTANT_TYPE'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CONSTANT"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CONSTANT_NAME, + CONSTANT_VALUE, + CONSTANT_TYPE +FROM {{ source('ods', 'F_CONSTANT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CONSTANT",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTERPARTY.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTERPARTY.sql new file mode 100644 index 0000000..ee5e005 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTERPARTY.sql @@ -0,0 +1,62 @@ +{% snapshot m_DWH_FXCD_NH_F_COUNTERPARTY %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_COUNTERPARTY_MARS', + unique_key=['CTP_ID'], + strategy='check', + check_cols=['ENTITY_ID', + 'CTP_GROUP_FKIT_CODE', + 'ACTIVE_FLAG', + 'CTP_COMMENT', + 'GUARANTOR_ID', + 'OF_OVERALL_LMT_AMT', + 'OF_MANUAL_FLAG', + 'FR_OVERALL_LMT_AMT', + 'FR_MANUAL_FLAG', + 'MP_OVERALL_LMT_AMT', + 'MP_MANUAL_FLAG', + 'EOI_CTP_GROUP', + 'ART_101_FLAG', + 'MEDIAN_CAPITAL_FLAG', + 'CHANGE_DESCRIPTION'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_COUNTERPARTY"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CTP_ID, + ENTITY_ID, + CTP_GROUP_FKIT_CODE, + {{clean_boolean_fxcd('ACTIVE_FLAG')}} AS ACTIVE_FLAG, + CTP_COMMENT, + GUARANTOR_ID, + OF_OVERALL_LMT_AMT, + {{clean_boolean_fxcd('OF_MANUAL_FLAG')}} AS OF_MANUAL_FLAG, + FR_OVERALL_LMT_AMT, + {{clean_boolean_fxcd('FR_MANUAL_FLAG')}} AS FR_MANUAL_FLAG, + MP_OVERALL_LMT_AMT, + {{clean_boolean_fxcd('MP_MANUAL_FLAG')}} AS MP_MANUAL_FLAG, + EOI_CTP_GROUP, + {{clean_boolean_fxcd('ART_101_FLAG')}} AS ART_101_FLAG, + {{clean_boolean_fxcd('MEDIAN_CAPITAL_FLAG')}} AS MEDIAN_CAPITAL_FLAG, + CHANGE_DESCRIPTION +FROM {{ source('ods', 'F_COUNTERPARTY') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_COUNTERPARTY",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY.sql new file mode 100644 index 0000000..9f4c90d --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY.sql @@ -0,0 +1,71 @@ +{% snapshot m_DWH_FXCD_NH_F_COUNTRY %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_COUNTRY_MARS', + unique_key=['COUNTRY_ID'], + strategy='check', + check_cols=['ENTITY_ID', + 'COUNTRY_GDP', + 'NCB_USD_LAMBDA', + 'OF_FLAG', + 'FR_FLAG', + 'EU_FLAG', + 'EUROSYSTEM_FLAG', + 'FR_PORTF_SHARE_OPTOUT', + 'FR_LIMIT_CALC_OPTOUT', + 'COUNTRY_COMMENT', + 'OF_LMT_AMNT', + 'OF_MANUAL_FLAG', + 'FR_LMT_AMNT', + 'FR_MANUAL_FLAG', + 'USD_PORTFOLIO_EUR_SIZE', + 'JPY_PORTFOLIO_EUR_SIZE', + 'CAPITAL_KEY_AMNT', + 'LAMBDA_MANUAL_FLAG', + 'CNY_PORTFOLIO_EUR_SIZE', + 'CHANGE_DESCRIPTION'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_COUNTRY"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + COUNTRY_ID, + ENTITY_ID, + COUNTRY_GDP, + NCB_USD_LAMBDA, + {{clean_boolean_fxcd('OF_FLAG')}} AS OF_FLAG, + {{clean_boolean_fxcd('FR_FLAG')}} AS FR_FLAG, + {{clean_boolean_fxcd('EU_FLAG')}} AS EU_FLAG, + {{clean_boolean_fxcd('EUROSYSTEM_FLAG')}} AS EUROSYSTEM_FLAG, + {{clean_boolean_fxcd('FR_PORTF_SHARE_OPTOUT')}} AS FR_PORTF_SHARE_OPTOUT, + {{clean_boolean_fxcd('FR_LIMIT_CALC_OPTOUT')}} AS FR_LIMIT_CALC_OPTOUT, + COUNTRY_COMMENT, + OF_LMT_AMNT, + {{clean_boolean_fxcd('OF_MANUAL_FLAG')}} AS OF_MANUAL_FLAG, + FR_LMT_AMNT, + {{clean_boolean_fxcd('FR_MANUAL_FLAG')}} AS FR_MANUAL_FLAG, + USD_PORTFOLIO_EUR_SIZE, + JPY_PORTFOLIO_EUR_SIZE, + CAPITAL_KEY_AMNT, + {{clean_boolean_fxcd('LAMBDA_MANUAL_FLAG')}} AS LAMBDA_MANUAL_FLAG, + CNY_PORTFOLIO_EUR_SIZE, + CHANGE_DESCRIPTION +FROM {{ source('ods', 'F_COUNTRY') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_COUNTRY",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT.sql new file mode 100644 index 0000000..804465f --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT.sql @@ -0,0 +1,39 @@ +{% snapshot m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_COUNTRY_NCB_LIMIT_MARS', + unique_key=['COUNTRY_ID', + 'NCB_COUNTRY_ID'], + strategy='check', + check_cols=['USD_LIMIT_AMT', + 'JPY_LIMIT_AMT', + 'TOTAL_LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_COUNTRY_NCB_LIMIT"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + COUNTRY_ID, + NCB_COUNTRY_ID, + USD_LIMIT_AMT, + JPY_LIMIT_AMT, + TOTAL_LIMIT_AMT +FROM {{ source('ods', 'F_COUNTRY_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_COUNTRY_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP.sql new file mode 100644 index 0000000..fcb57f1 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP.sql @@ -0,0 +1,39 @@ +{% snapshot m_DWH_FXCD_NH_F_CTP_GROUP %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CTP_GROUP_MARS', + unique_key=['CTP_GROUP_FKIT_CODE'], + strategy='check', + check_cols=['CTP_GROUP_NAME', + 'OF_MAX_LIMIT_AMT', + 'FR_MP_MAX_LIMIT_AMT', + 'ACTIVE'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CTP_GROUP"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + CTP_GROUP_FKIT_CODE, + CTP_GROUP_NAME, + OF_MAX_LIMIT_AMT, + FR_MP_MAX_LIMIT_AMT, + {{clean_boolean_fxcd('ACTIVE')}} AS ACTIVE +FROM {{ source('ods', 'F_CTP_GROUP') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CTP_GROUP",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT.sql new file mode 100644 index 0000000..c925f6c --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT.sql @@ -0,0 +1,37 @@ +{% snapshot m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CTP_GROUP_NCB_LIMIT_MARS', + unique_key=['COUNTRY_ID', + 'CTP_GROUP_FKIT_CODE', + 'ELIGIBILITY_ID'], + strategy='check', + check_cols=['LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CTP_GROUP_NCB_LIMIT"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + COUNTRY_ID, + CTP_GROUP_FKIT_CODE, + ELIGIBILITY_ID, + LIMIT_AMT +FROM {{ source('ods', 'F_CTP_GROUP_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_NCB_LIMIT.sql new file mode 100644 index 0000000..19a0552 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_CTP_NCB_LIMIT.sql @@ -0,0 +1,37 @@ +{% snapshot m_DWH_FXCD_NH_F_CTP_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_CTP_NCB_LIMIT_MARS', + unique_key=['COUNTRY_ID', + 'ELIGIBILITY_ID', + 'CTP_ID'], + strategy='check', + check_cols=['LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_CTP_NCB_LIMIT"] + ) +}} +SELECT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + {{ get_workflow_history_key() }} AS A_DWH_LOAD_SET_FK, + COUNTRY_ID, + ELIGIBILITY_ID, + CTP_ID, + LIMIT_AMT +FROM {{ source('ods', 'F_CTP_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_CTP_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} + diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY.sql new file mode 100644 index 0000000..bcae83d --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY.sql @@ -0,0 +1,70 @@ +{% snapshot m_DWH_FXCD_NH_F_ELIGIBILITY %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ELIGIBILITY_MARS', + unique_key=['ELIGIBILITY_ID'], + strategy='check', + check_cols=[ + 'ELIGIBILITY_TYPE', + 'ELIGIBILITY_NAME', + 'LIMIT_PERCENTAGE', + 'LIMIT_AMT_MANUAL_FLAG', + 'LIMIT_AMT', + 'NCB_LMT_AMT_FLOOR', + 'RATING_THRESHOLD', + 'FKIT_SUBLIMIT_NAME', + 'ELIGIBILITY_ORDER', + 'LIMIT_DISTRIBUTION', + 'CTP_MIN_REQ_RATINGS', + 'LIMITS_CALCULATION', + 'ART_101_FLAG', + 'DEFINITION', + 'FOR_CP_FLAG', + 'FOR_CPG_FLAG', + 'OVERALL_LIMIT_FLAG', + 'IDENTIFIER' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ELIGIBILITY"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ELIGIBILITY_ID AS NUMBER(28,0)) AS ELIGIBILITY_ID, + CAST(ELIGIBILITY_TYPE AS VARCHAR2(2 CHAR)) AS ELIGIBILITY_TYPE, + CAST(ELIGIBILITY_NAME AS VARCHAR2(200 CHAR)) AS ELIGIBILITY_NAME, + CAST(LIMIT_PERCENTAGE AS NUMBER(28,10)) AS LIMIT_PERCENTAGE, + CAST({{ clean_boolean_fxcd('LIMIT_AMT_MANUAL_FLAG') }} AS CHAR(1 CHAR)) AS LIMIT_AMT_MANUAL_FLAG, + CAST(LIMIT_AMT AS NUMBER(28,10)) AS LIMIT_AMT, + CAST(NCB_LMT_AMT_FLOOR AS NUMBER(28,10)) AS NCB_LMT_AMT_FLOOR, + CAST(RATING_THRESHOLD AS VARCHAR2(4 CHAR)) AS RATING_THRESHOLD, + CAST(FKIT_SUBLIMIT_NAME AS VARCHAR2(30 CHAR)) AS FKIT_SUBLIMIT_NAME, + CAST(ELIGIBILITY_ORDER AS NUMBER(28,0)) AS ELIGIBILITY_ORDER, + CAST(LIMIT_DISTRIBUTION AS VARCHAR2(3 CHAR)) AS LIMIT_DISTRIBUTION, + CAST(CTP_MIN_REQ_RATINGS AS NUMBER(28,0)) AS CTP_MIN_REQ_RATINGS, + CAST({{ clean_boolean_fxcd('LIMITS_CALCULATION') }} AS CHAR(1 CHAR)) AS LIMITS_CALCULATION, + CAST({{ clean_boolean_fxcd('ART_101_FLAG') }} AS CHAR(1 CHAR)) AS ART_101_FLAG, + CAST(DEFINITION AS VARCHAR2(50 CHAR)) AS DEFINITION, + CAST(FOR_CP_FLAG AS VARCHAR2(1 CHAR)) AS FOR_CP_FLAG, + CAST(FOR_CPG_FLAG AS VARCHAR2(1 CHAR)) AS FOR_CPG_FLAG, + CAST(OVERALL_LIMIT_FLAG AS VARCHAR2(1 CHAR)) AS OVERALL_LIMIT_FLAG, + CAST(IDENTIFIER AS VARCHAR2(6 CHAR)) AS IDENTIFIER +FROM + {{ source('ods', 'F_ELIGIBILITY') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ELIGIBILITY",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP.sql new file mode 100644 index 0000000..c675137 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP.sql @@ -0,0 +1,37 @@ +{% snapshot m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ELIGIBILITY_GROUP_MAP_MARS', + unique_key=['CTP_GROUP_FKIT_CODE', 'ELIGIBILITY_ID'], + strategy='check', + check_cols=['LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ELIGIBILITY_GROUP_MAP"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(CTP_GROUP_FKIT_CODE AS VARCHAR2(4 CHAR)) AS CTP_GROUP_FKIT_CODE, + CAST(ELIGIBILITY_ID AS NUMBER(28,0)) AS ELIGIBILITY_ID, + CAST(NULL AS NUMBER(28,10)) AS OF_LIMIT_AMT, -- Not mapped in Informatica target, so excluded from check_cols + CAST(NULL AS NUMBER(28,10)) AS FR_MP_LIMIT_AMT, -- Not mapped in Informatica target, so excluded from check_cols + CAST(LIMIT_AMT AS NUMBER(28,10)) AS LIMIT_AMT +FROM + {{ source('ods', 'F_ELIGIBILITY_GROUP_MAP') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ELIGIBILITY_GROUP_MA",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER.sql new file mode 100644 index 0000000..31cd07b --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER.sql @@ -0,0 +1,47 @@ +{% snapshot m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ELIGIBILITY_ISSUER_MARS', + unique_key=['ELIGIBILITY_ISSUER_ID'], + strategy='check', + check_cols=[ + 'ELIGIBILITY_ISSUER_NAME', + 'ART_101_FLAG', + 'RATING_THRESHOLD', + 'DEFINITION', + 'RATING_UPLIFT_FACTOR', + 'ELIG_FLAG' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ELIGIBILITY_ISSUER_ID AS NUMBER(28,0)) AS ELIGIBILITY_ISSUER_ID, + CAST(ELIGIBILITY_ISSUER_NAME AS VARCHAR2(200 CHAR)) AS ELIGIBILITY_ISSUER_NAME, + CAST({{ clean_boolean_fxcd('ART_101_FLAG') }} AS CHAR(1 CHAR)) AS ART_101_FLAG, + CAST(RATING_THRESHOLD AS VARCHAR2(4 CHAR)) AS RATING_THRESHOLD, + CAST(DEFINITION AS VARCHAR2(50 CHAR)) AS DEFINITION, + CAST(NULL AS CHAR(1 BYTE)) AS RATING_TERM_TYPE, -- Not mapped in Informatica target, so excluded from check_cols + CAST(RATING_UPLIFT_FACTOR AS NUMBER(28,0)) AS RATING_UPLIFT_FACTOR, + CAST({{ clean_boolean_fxcd('ELIG_FLAG') }} AS VARCHAR2(1 CHAR)) AS ELIG_FLAG +FROM + {{ source('ods', 'F_ELIGIBILITY_ISSUER') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ELIGIBILITY_ISSUER",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP.sql new file mode 100644 index 0000000..a71ed93 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP.sql @@ -0,0 +1,36 @@ +{% snapshot m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ELIGIBILITY_ISS_MAP_MARS', + unique_key=['ISSUER_ID', 'ELIGIBILITY_ISSUER_ID'], + strategy='check', + check_cols=['ELEGIBLE_FLAG', 'MANUAL_ELIGIBLE_FLAG'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ELIGIBILITY_ISSUER_MAP"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ISSUER_ID AS NUMBER(28,0)) AS ISSUER_ID, + CAST(ELIGIBILITY_ISSUER_ID AS NUMBER(28,0)) AS ELIGIBILITY_ISSUER_ID, + CAST(ELEGIBLE_FLAG AS CHAR(1 CHAR)) AS ELEGIBLE_FLAG, + CAST(MANUAL_ELIGIBLE_FLAG AS CHAR(1 BYTE)) AS MANUAL_ELIGIBLE_FLAG +FROM + {{ source('ods', 'F_ELIGIBILITY_ISSUER_MAP') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ELIGIBILITY_ISSUER_M",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_MAP.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_MAP.sql new file mode 100644 index 0000000..85946b2 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ELIGIBILITY_MAP.sql @@ -0,0 +1,39 @@ +{% snapshot m_DWH_FXCD_NH_F_ELIGIBILITY_MAP %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ELIGIBILITY_MAP_MARS', + unique_key=['CTP_ID', 'ELIGIBILITY_ID'], + strategy='check', + check_cols=['LIMIT_AMT'], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ELIGIBILITY_MAP"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(CTP_ID AS NUMBER(28,0)) AS CTP_ID, + CAST(ELIGIBILITY_ID AS NUMBER(28,0)) AS ELIGIBILITY_ID, + CAST(NULL AS NUMBER(28,10)) AS OF_LIMIT_AMT, -- Not mapped in Informatica target, so excluded from check_cols + CAST(NULL AS CHAR(1 CHAR)) AS OF_FLAG, -- Not mapped in Informatica target, so excluded from check_cols + CAST(NULL AS NUMBER(28,10)) AS FR_LIMIT_AMT, -- Not mapped in Informatica target, so excluded from check_cols + CAST(NULL AS CHAR(1 CHAR)) AS FR_MP_FLAG, -- Not mapped in Informatica target, so excluded from check_cols + CAST(LIMIT_AMT AS NUMBER(28,10)) AS LIMIT_AMT +FROM + {{ source('ods', 'F_ELIGIBILITY_MAP') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ELIGIBILITY_MAP",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ENTITY.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ENTITY.sql new file mode 100644 index 0000000..782cfe1 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ENTITY.sql @@ -0,0 +1,54 @@ +{% snapshot m_DWH_FXCD_NH_F_ENTITY %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ENTITY_MARS', + unique_key=['ENTITY_ID'], + strategy='check', + check_cols=[ + 'FKIT_CODE', + 'ENTITY_NAME', + 'BVD_BANK_INDEX_NUMBER', + 'RISK_COUNTRY_ID', + 'LT_MANUAL_RATING_FLAG', + 'LT_SECOND_BEST_RATING', + 'CAP_TIER1_AMT', + 'CAP_EQUITY_AMT', + 'BLOOMBERG_TICKER' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ENTITY"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ENTITY_ID AS NUMBER(28,0)) AS ENTITY_ID, + CAST(FKIT_CODE AS VARCHAR2(20 CHAR)) AS FKIT_CODE, + CAST(ENTITY_NAME AS VARCHAR2(200 CHAR)) AS ENTITY_NAME, + CAST(BVD_BANK_INDEX_NUMBER AS VARCHAR2(100 CHAR)) AS BVD_BANK_INDEX_NUMBER, + CAST(RISK_COUNTRY_ID AS CHAR(2 BYTE)) AS RISK_COUNTRY_ID, + CAST(NULL AS CHAR(1 CHAR)) AS ST_MANUAL_RATING_FLAG, -- Not mapped in Informatica target, so excluded from check_cols + CAST({{ clean_boolean_fxcd('LT_MANUAL_RATING_FLAG') }} AS CHAR(1 CHAR)) AS LT_MANUAL_RATING_FLAG, + CAST(NULL AS VARCHAR2(4 CHAR)) AS ST_SECOND_BEST_RATING, -- Not mapped in Informatica target, so excluded from check_cols + CAST(LT_SECOND_BEST_RATING AS VARCHAR2(4 CHAR)) AS LT_SECOND_BEST_RATING, + CAST(CAP_TIER1_AMT AS NUMBER(28,10)) AS CAP_TIER1_AMT, + CAST(CAP_EQUITY_AMT AS NUMBER(28,10)) AS CAP_EQUITY_AMT, + CAST(BLOOMBERG_TICKER AS VARCHAR2(20 CHAR)) AS BLOOMBERG_TICKER +FROM + {{ source('ods', 'F_ENTITY') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ENTITY",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_EQUIVALENCE_RULE.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_EQUIVALENCE_RULE.sql new file mode 100644 index 0000000..1a9e4ab --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_EQUIVALENCE_RULE.sql @@ -0,0 +1,37 @@ +{% snapshot m_DWH_FXCD_NH_F_EQUIVALENCE_RULE %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_EQUIVALENCE_RULE_MARS', + unique_key=['AGENCY_ID', 'AGENCY_RATING', 'NUMERICAL_EQUIVALENCE', 'RATING_FACTOR'], + strategy='check', + check_cols='all', + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_EQUIVALENCE_RULE"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RATING_TERM_TYPE, -- Not mapped in Informatica target, so excluded from check_cols + CAST(AGENCY_ID AS VARCHAR2(2 CHAR)) AS AGENCY_ID, + CAST(AGENCY_RATING AS VARCHAR2(4 CHAR)) AS AGENCY_RATING, + CAST(NUMERICAL_EQUIVALENCE AS NUMBER(28,0)) AS NUMERICAL_EQUIVALENCE, + CAST(RATING_FACTOR AS NUMBER(28,10)) AS RATING_FACTOR +FROM + {{ source('ods', 'F_EQUIVALENCE_RULE') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_EQUIVALENCE_RULE",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER.sql new file mode 100644 index 0000000..2457eac --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER.sql @@ -0,0 +1,70 @@ +{% snapshot m_DWH_FXCD_NH_F_ISSUER %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ISSUER_MARS', + unique_key=['ISSUER_ID'], + strategy='check', + check_cols=[ + 'ENTITY_ID', + 'ACTIVE_FLAG', + 'ISSUER_COMMENT', + 'OFM_ISSUER_LMT', + 'OF_MANUAL_FLAG', + 'GLOBAL_LMT_AMNT', + 'MANUAL_GLOBAL_LMT_FLAG', + 'OF_ISSUER_CATEGORY', + 'FR_ISSUER_CATEGORY', + 'ISSUER_LMT', + 'FR_MANUAL_FLAG', + 'NCB_USD_LAMBDA', + 'ART_101_FLAG', + 'UPLIFT_IMPLIED_RATING', + 'LAMBDA_MANUAL_FLAG', + 'CHANGE_DESCRIPTION', + 'LT_FRM_LIMIT_RATIO', + 'LT_FRM_LIMIT_RATIO_MANUAL_FLAG' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + COLUMN_TYPES={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ISSUER"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ISSUER_ID AS NUMBER(28,0)) AS ISSUER_ID, + CAST(ENTITY_ID AS NUMBER(28,0)) AS ENTITY_ID, + CAST({{ clean_boolean_fxcd('ACTIVE_FLAG') }} AS CHAR(1 CHAR)) AS ACTIVE_FLAG, + CAST(ISSUER_COMMENT AS VARCHAR2(4000 CHAR)) AS ISSUER_COMMENT, + CAST(OFM_ISSUER_LMT AS NUMBER(28,10)) AS OFM_ISSUER_LMT, + CAST({{ clean_boolean_fxcd('OF_MANUAL_FLAG') }} AS VARCHAR2(1 CHAR)) AS OF_MANUAL_FLAG, + CAST(GLOBAL_LMT_AMNT AS NUMBER(28,10)) AS GLOBAL_LMT_AMNT, + CAST({{ clean_boolean_fxcd('MANUAL_GLOBAL_LMT_FLAG') }} AS CHAR(1 CHAR)) AS MANUAL_GLOBAL_LMT_FLAG, + CAST(OF_ISSUER_CATEGORY AS VARCHAR2(5 CHAR)) AS OF_ISSUER_CATEGORY, + CAST(FR_ISSUER_CATEGORY AS VARCHAR2(5 CHAR)) AS FR_ISSUER_CATEGORY, + CAST(ISSUER_LMT AS NUMBER(28,10)) AS ISSUER_LMT, + CAST({{ clean_boolean_fxcd('FR_MANUAL_FLAG') }} AS CHAR(1 CHAR)) AS FR_MANUAL_FLAG, + CAST(NCB_USD_LAMBDA AS NUMBER(28,10)) AS NCB_USD_LAMBDA, + CAST(ART_101_FLAG AS CHAR(1 CHAR)) AS ART_101_FLAG, + CAST(UPLIFT_IMPLIED_RATING AS VARCHAR2(4 CHAR)) AS UPLIFT_IMPLIED_RATING, + CAST({{ clean_boolean_fxcd('LAMBDA_MANUAL_FLAG') }} AS VARCHAR2(1 CHAR)) AS LAMBDA_MANUAL_FLAG, + CAST(CHANGE_DESCRIPTION AS VARCHAR2(256 CHAR)) AS CHANGE_DESCRIPTION, + CAST(LT_FRM_LIMIT_RATIO AS NUMBER(28,10)) AS LT_FRM_LIMIT_RATIO, + CAST({{ clean_boolean_fxcd('LT_FRM_LIMIT_RATIO_MANUAL_FLAG') }} AS VARCHAR2(1 CHAR)) AS LT_FRM_LIMIT_RATIO_MANUAL_FLAG +FROM + {{ source('ods', 'F_ISSUER') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ISSUER",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT.sql new file mode 100644 index 0000000..eb6c884 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT.sql @@ -0,0 +1,38 @@ +{% snapshot m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_ISSUER_NCB_LIMIT_MARS', + unique_key=['COUNTRY_ID', 'ISSUER_ID', 'USD_LIMIT_AMT', 'JPY_LIMIT_AMT', 'TOTAL_LIMIT_AMT'], + strategy='check', + check_cols='all', + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_ISSUER_NCB_LIMIT"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(COUNTRY_ID AS CHAR(2 BYTE)) AS COUNTRY_ID, + CAST(ISSUER_ID AS NUMBER(28,0)) AS ISSUER_ID, + CAST(USD_LIMIT_AMT AS NUMBER(28,10)) AS USD_LIMIT_AMT, + CAST(JPY_LIMIT_AMT AS NUMBER(28,10)) AS JPY_LIMIT_AMT, + CAST(TOTAL_LIMIT_AMT AS NUMBER(28,10)) AS TOTAL_LIMIT_AMT, + CAST(NULL AS VARCHAR2(1 CHAR)) AS LIMIT_TYPE -- Not mapped in Informatica target, so excluded from check_cols +FROM + {{ source('ods', 'F_ISSUER_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_ISSUER_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT.sql new file mode 100644 index 0000000..fa2d73f --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT.sql @@ -0,0 +1,38 @@ +{% snapshot m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_LIQUIDITY_NCB_LIMIT_MARS', + unique_key=['COUNTRY_ID'], + strategy='check', + check_cols=[ + 'USD_NCB_MIN_LIMIT_AMT', + 'USD_NCB_MAX_LIMIT_AMT' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_LIQUIDITY_NCB_LIMIT"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(COUNTRY_ID AS CHAR(2 BYTE)) AS COUNTRY_ID, + CAST(USD_NCB_MIN_LIMIT_AMT AS NUMBER(28,10)) AS USD_NCB_MIN_LIMIT_AMT, + CAST(USD_NCB_MAX_LIMIT_AMT AS NUMBER(28,10)) AS USD_NCB_MAX_LIMIT_AMT +FROM + {{ source('ods', 'F_LIQUIDITY_NCB_LIMIT') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE.sql new file mode 100644 index 0000000..cbeb8e9 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE.sql @@ -0,0 +1,50 @@ +{% snapshot m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_NCB_PORTFOLIO_SHARE_MARS', + unique_key=['COUNTRY_ID'], + strategy='check', + check_cols=[ + 'COUNTRY_SHARE', + 'USD_COUNTRY_SHARE', + 'JPY_COUNTRY_SHARE', + 'SCALED_COUNTRY_SHARE', + 'SCALED_USD_COUNTRY_SHARE', + 'SCALED_JPY_COUNTRY_SHARE', + 'CNY_COUNTRY_SHARE', + 'SCALED_CNY_COUNTRY_SHARE' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_NCB_PORTFOLIO_SHARE"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(COUNTRY_ID AS CHAR(2 CHAR)) AS COUNTRY_ID, + CAST(COUNTRY_SHARE AS NUMBER(28,25)) AS COUNTRY_SHARE, + CAST(USD_COUNTRY_SHARE AS NUMBER(28,25)) AS USD_COUNTRY_SHARE, + CAST(JPY_COUNTRY_SHARE AS NUMBER(28,25)) AS JPY_COUNTRY_SHARE, + CAST(SCALED_COUNTRY_SHARE AS NUMBER(28,25)) AS SCALED_COUNTRY_SHARE, + CAST(SCALED_USD_COUNTRY_SHARE AS NUMBER(28,25)) AS SCALED_USD_COUNTRY_SHARE, + CAST(SCALED_JPY_COUNTRY_SHARE AS NUMBER(28,25)) AS SCALED_JPY_COUNTRY_SHARE, + CAST(CNY_COUNTRY_SHARE AS NUMBER(28,25)) AS CNY_COUNTRY_SHARE, + CAST(SCALED_CNY_COUNTRY_SHARE AS NUMBER(28,25)) AS SCALED_CNY_COUNTRY_SHARE +FROM + {{ source('ods', 'F_NCB_PORTFOLIO_SHARE') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_NCB_PORTFOLIO_SHARE",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_RATING.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_RATING.sql new file mode 100644 index 0000000..1d23676 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_RATING.sql @@ -0,0 +1,36 @@ +{% snapshot m_DWH_FXCD_NH_F_RATING %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_RATING_MARS', + unique_key=['ENTITY_ID', 'AGENCY_ID', 'AGENCY_RATING'], + strategy='check', + check_cols='all', + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_RATING"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(ENTITY_ID AS NUMBER(28,0)) AS ENTITY_ID, + CAST(NULL AS VARCHAR2(1 CHAR)) AS RATING_TERM_TYPE, + CAST(AGENCY_ID AS VARCHAR2(2 CHAR)) AS AGENCY_ID, + CAST(AGENCY_RATING AS VARCHAR2(4 CHAR)) AS AGENCY_RATING +FROM + {{ source('ods', 'F_RATING') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_RATING",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/snapshots/m_DWH_FXCD_NH_F_RATING_AGENCY.sql b/dbt/snapshots/m_DWH_FXCD_NH_F_RATING_AGENCY.sql new file mode 100644 index 0000000..2f0b2e9 --- /dev/null +++ b/dbt/snapshots/m_DWH_FXCD_NH_F_RATING_AGENCY.sql @@ -0,0 +1,38 @@ +{% snapshot m_DWH_FXCD_NH_F_RATING_AGENCY %} +{{ + config( + target_schema='dw_rar', + alias = 'NH_F_RATING_AGENCY_MARS', + unique_key=['AGENCY_ID'], + strategy='check', + check_cols=[ + 'AGENCY_NAME', + 'ENABLED_FLAG' + ], + hard_deletes='invalidate', + dbt_valid_to_current="TIMESTAMP '9999-12-31 00:00:00'", + snapshot_meta_column_names={ + 'dbt_valid_from': 'A_VALID_FROM', + 'dbt_valid_to': 'A_VALID_TO', + 'dbt_updated_at': 'UPDATED_AT', + 'dbt_scd_id': 'SCD_ID' + }, + column_types={ + 'A_VALID_FROM': 'DATE', + 'A_VALID_TO': 'DATE' + }, + tags=["m_DWH_FXCD_NH_F_RATING_AGENCY"] + ) +}} + +SELECT DISTINCT + (SELECT COALESCE(MAX(A_KEY), 0) FROM {{ this }}) + ROW_NUMBER() OVER (ORDER BY 1) as A_KEY, + CAST({{ get_workflow_history_key() }} AS NUMBER(38,0)) AS A_DWH_LOAD_SET_FK, + CAST(AGENCY_ID AS VARCHAR2(2 CHAR)) AS AGENCY_ID, + CAST(AGENCY_NAME AS VARCHAR2(20 CHAR)) AS AGENCY_NAME, + CAST({{ clean_boolean_fxcd('ENABLED_FLAG') }} AS VARCHAR2(1 CHAR)) AS ENABLED_FLAG +FROM + {{ source('ods', 'F_RATING_AGENCY') }} +WHERE + A_WORKFLOW_HISTORY_KEY = {{ filter_workflow_history_max_key("w_ODS_FXCD_F_RATING_AGENCY",get_main_task_name(model.name))}} +{% endsnapshot %} \ No newline at end of file diff --git a/dbt/tests/.gitkeep b/dbt/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/code_generation/generate_code.py b/python/code_generation/generate_code.py new file mode 100644 index 0000000..9130fed --- /dev/null +++ b/python/code_generation/generate_code.py @@ -0,0 +1,73 @@ +from jinja2 import Environment, FileSystemLoader +import csv + +environment = Environment(loader=FileSystemLoader("templates/")) +template_sq = environment.get_template("m_Template_sq.sql") +template_tgt = environment.get_template("m_Template_target.sql") +template_db = environment.get_template("db_Template.sql") +template_dag = environment.get_template("dag_Template.py") + +csvFile = csv.reader(open("ods_mopdb_plain.txt", "r")) +header = next(csvFile) # skip header + +for row in csvFile: + + (schema, table) = row + + # ou_tms,ACTIVITYLOGDUE,TMS,T_ACTIVITYLOGDUE,m_MOPDB_TMS_T_ACTIVITYLOGDUE_OU_TMS_ACTIVITYLOGDUE,w_ODS_TMS_ACTIVITYLOGDUE + + target_table = f"T_{table}" + source_schema = f"OU_{schema}" + source_schema_lower = f"{source_schema}".lower() + source_table = table + mapping_name = f"m_MOPDB_{schema}_{target_table}_{source_schema}_{source_table}" + workflow_name = f"w_MOPDB_{schema}_{target_table}" + + + content = template_sq.render( + source_schema = source_schema_lower, + source_table = table, + target_schema = schema, + target_table = target_table, + mapping_name = mapping_name, + workflow_name = workflow_name + ) + filename=f"source_qualifiers/{mapping_name}_SQ.sql" + with open(filename, mode="w", encoding="utf-8") as message: + message.write(content) + print(f"... wrote {filename}") + + content = template_tgt.render( + source_schema = source_schema_lower, + source_table = table, + target_schema = schema, + target_table = target_table, + mapping_name = mapping_name, + workflow_name = workflow_name + ) + filename=f"targets/{mapping_name}.sql" + with open(filename, mode="w", encoding="utf-8") as message: + message.write(content) + print(f"... wrote {filename}") + + content = template_dag.render( + table = table + ) + filename=f"dags/{workflow_name}.py" + with open(filename, mode="w", encoding="utf-8") as message: + message.write(content) + print(f"... wrote {filename}") + + content = template_db.render( + table = table, + schema = schema + ) + filename=f"db/{source_schema}_{source_table}.sql" + with open(filename, mode="w", encoding="utf-8") as message: + message.write(content) + print(f"... wrote {filename}") + filename=f"db/table_changes.sql" + with open(filename, mode="a", encoding="utf-8") as message: + message.write(f"{content}\n") + print(f"... wrote {filename}") + diff --git a/python/code_generation/ods_mopdb_plain.txt b/python/code_generation/ods_mopdb_plain.txt new file mode 100644 index 0000000..c242b25 --- /dev/null +++ b/python/code_generation/ods_mopdb_plain.txt @@ -0,0 +1,30 @@ +SCHEMA,TABLE +TMS,ACMENTRYSTATELEDGERGROUP +TMS,ACTIVITYLOGDUE +TMS,ACTIVITY_LOG +TMS,BALANCE +TMS,BLACKOUT_LOG +TMS,BRANCH +TMS,CALENDAR +TMS,CASHFLOW +TMS,CLIENT +TMS,CUSTODYBALANCE +TMS,ECBINSTRUMENTBONDCASHFLOW +TMS,EFFECTIVEROLEPROFILE +TMS,FINMESSAGELOG +TMS,HISTORY_LOG +TMS,INSTRUMENTBONDCASHFLOW +TMS,INSTRUMENT_REPORT +TMS,MARKETINFO +TMS,PARAMETER +TMS,PORTFOLIOTREE +TMS,PRICES +TMS,PROPERTY +TMS,RECONCILIATION +TMS,ROLEPORTFOLIOPROFILE +TMS,RULES +TMS,SDM_ENTITY_STATE +TMS,SECURITYPOSITION +TMS,SETTLEMENTCASHFLOW +TMS,SETTLEMENTLOG +TMS,USERINFORMATION diff --git a/python/code_generation/templates/dag_Template.py b/python/code_generation/templates/dag_Template.py new file mode 100644 index 0000000..33cbadb --- /dev/null +++ b/python/code_generation/templates/dag_Template.py @@ -0,0 +1,95 @@ +from airflow.decorators import dag +from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from airflow.utils.dates import days_ago +from airflow.utils.trigger_rule import TriggerRule +from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig, RenderConfig + +# Define paths to your dbt files +dbt_root_path = "/home/dbt/DBT/mrds" +dbt_profiles_dir = "/home/dbt/.dbt/profiles.yml" + +ODS_TABLE = "{{table}}" +DATABASE_NAME = "MOPDB" +DAG_NAME = f"w_{DATABASE_NAME}_TMS_T_{ODS_TABLE}_OU_TMS_{ODS_TABLE}" +MAPPING_NAME = f"m_{DATABASE_NAME}_TMS_T_{ODS_TABLE}_OU_TMS_{ODS_TABLE}" + + +# Define function for the retrieval of the current run_id +def retrieve_run_id(**kwargs): + # Retrieve the run_id from the Airflow context + run_id = kwargs['run_id'] + # Store the run_id in XCom for future reference + ti = kwargs['ti'] + ti.xcom_push(key='run_id', value=run_id) + return run_id + +def check_dag_status(**kwargs): + for task_instance in kwargs['dag_run'].get_task_instances(): + if task_instance.state == 'failed' and task_instance.task_id != kwargs['task_instance'].task_id: + raise Exception("Task {} failed. Failing this DAG run".format(task_instance.task_id)) + +# Define function for the check of the status of the previous tasks +def determine_workflow_status(**kwargs): + # Check the status of previous tasks + task_statuses = kwargs['ti'].xcom_pull(task_ids=['retrieve_run_id', 'control_external_run_start', 'mapping_mopdb']) + # If any task failed, set workflow_status to 'N', otherwise 'Y' + workflow_status = 'N' if any(status != 'success' for status in task_statuses) else 'Y' + return workflow_status + +@dag( + dag_id=DAG_NAME, + schedule_interval=None, + start_date=days_ago(2), + catchup=False +) +def run_dag(): + # Retrieve run_id + retrieve_run_id_task = PythonOperator( + task_id='retrieve_run_id', + python_callable=retrieve_run_id, + provide_context=True, + # pool='my_custom_pool', # Create pool in Airflow Web UI with one slot to ensure that only one dag can run it at a time. + ) + # Run dbt macro control_external_run_start + control_external_run_start = BashOperator( + task_id='control_external_run_start', + bash_command=( + 'cd /home/dbt/DBT/mrds && ' + 'dbt run-operation control_external_run_start --vars \'{"orchestration_run_id": "{% raw %}{{{% endraw %} task_instance.xcom_pull(task_ids="retrieve_run_id", key="run_id") {% raw %}}}{% endraw %}", "input_service_name": "' + DATABASE_NAME + '", "workflow_name": "' + DAG_NAME + '"}\' ' + '--profiles-dir /home/dbt/.dbt/ --target dev' + ) + ) + # run dbt taskGroup with tag of the mapping name + dbtTaskGroup = DbtTaskGroup( + group_id=MAPPING_NAME, + project_config=ProjectConfig( + dbt_project_path = dbt_root_path), + profile_config=ProfileConfig( + profiles_yml_filepath = dbt_profiles_dir, + profile_name="mrds", + target_name="dev"), + render_config=RenderConfig(select=[f"tag:{MAPPING_NAME}"],), + operator_args={'vars': {'orchestration_run_id': '{% raw %}{{{% endraw %} task_instance.xcom_pull(task_ids="retrieve_run_id", key="run_id") {% raw %}}}{% endraw %}', "input_service_name": DATABASE_NAME, "workflow_name": DAG_NAME }} + ) + + control_external_run_end = BashOperator( + task_id='control_external_run_end', + bash_command=( + 'cd /home/dbt/DBT/mrds && ' + 'dbt run-operation control_external_run_end --vars \'{"orchestration_run_id": "{% raw %}{{{% endraw %} task_instance.xcom_pull(task_ids="retrieve_run_id", key="run_id") {% raw %}}}{% endraw %}", "input_service_name": "' + DATABASE_NAME + '", "workflow_name": "' + DAG_NAME + '"}\' ' + '--profiles-dir /home/dbt/.dbt/ --target dev' + ), + trigger_rule=TriggerRule.ALL_DONE # Run regardless of previous task outcomes + ) + + dag_status = PythonOperator( + task_id='dag_status', + provide_context=True, + python_callable=check_dag_status, + trigger_rule=TriggerRule.ALL_DONE, # Ensures this task runs even if upstream fails + ) + + # Set task dependencies + retrieve_run_id_task >> control_external_run_start >> [dbtTaskGroup] >> control_external_run_end >> dag_status +globals()[DAG_NAME] = run_dag() diff --git a/python/code_generation/templates/db_Template.sql b/python/code_generation/templates/db_Template.sql new file mode 100644 index 0000000..1747d40 --- /dev/null +++ b/python/code_generation/templates/db_Template.sql @@ -0,0 +1,2 @@ +ALTER TABLE OU_{{schema}}.{{table}} RENAME COLUMN A_ETL_LOAD_SET_FK TO A_WORKFLOW_HISTORY_KEY; +DROP TABLE {{schema}}.T_{{table}}; diff --git a/python/code_generation/templates/m_Template_sq.sql b/python/code_generation/templates/m_Template_sq.sql new file mode 100644 index 0000000..0e669e2 --- /dev/null +++ b/python/code_generation/templates/m_Template_sq.sql @@ -0,0 +1,11 @@ +{% raw %}{{{% endraw %} + config( + materialized="table", + tags=["{{mapping_name}}", "{{source_schema}}", "{{target_schema}}"], + alias="{{target_table}}_SQ", + schema="{{target_schema}}" + ) +{% raw %}}}{% endraw %} +{% raw %}{{{% endraw %} + create_table_from_source("{{source_schema}}","{{source_table}}","{{workflow_name}}",get_main_task_name(model.name),['A_KEY','A_WORKFLOW_HISTORY_KEY']) +{% raw %}}}{% endraw %} diff --git a/python/code_generation/templates/m_Template_target.sql b/python/code_generation/templates/m_Template_target.sql new file mode 100644 index 0000000..5ec313f --- /dev/null +++ b/python/code_generation/templates/m_Template_target.sql @@ -0,0 +1,13 @@ +{% raw %}-- depends_on: {{{% endraw %} ref('{{mapping_name}}_SQ') {% raw %}}} {% endraw %} +{% raw %}{{{% endraw %} + config( + tags=["{{mapping_name}}","MOPDB","{{target_schema}}"], + alias="{{target_table}}", + schema="{{target_schema}}", + materialized="incremental" , + incremental_strategy="merge" + ) +{% raw %}}}{% endraw %} +{% raw %}{{{% endraw %} +create_table_target('{{mapping_name}}_SQ') +{% raw %}}}{% endraw %} diff --git a/python/connectors/casper/casper_rqsd.py b/python/connectors/casper/casper_rqsd.py new file mode 100644 index 0000000..3d48396 --- /dev/null +++ b/python/connectors/casper/casper_rqsd.py @@ -0,0 +1,201 @@ +import requests +import io +import zipfile +import pandas as pd +import os +from datetime import datetime +import oci +from mrds.utils.secrets import get_secret +import mrds.utils.manage_runs as runManager +import mrds.utils.manage_files as fileManager +import mrds.utils.sql_statements as sqls +import sys +import yaml +TASK_HISTORY_MULTIPLIER = 1_000_000_000 + +def initialize_task(workflow_context, task_name): + # Initialize task + a_task_history_key = runManager.init_task( + task_name, + workflow_context["run_id"], + workflow_context["a_workflow_history_key"], + ) + + return a_task_history_key + +def rqsd_parser(fileName,bucket_path,file,bucket_name): + if "SCOPA" in fileName or "SCOPF" in fileName: + print("SCOP") + annex_1_1(fileName,bucket_path,file,bucket_name) + annex_1_2(fileName,bucket_path,file,bucket_name) + elif "RQSDC" in fileName: + print("RQSDC") + return annex_2(fileName, bucket_path,file,bucket_name) + +def annex_1_1(fileName, bucket_path,file,bucket_name): + fileData=fileName.split("_") + csv_file_path = fileName[:-4]+".csv" + version_number = fileData[6] + ref_exercise = fileData[2] + ncb = fileData[4] + df = pd.read_excel(file, sheet_name="Counterparties in scope", skiprows=3) + df = df.dropna(axis=1, how='all').dropna(axis=0, how='all') + df['file_name'] = os.path.basename(fileName) + df['ingestion_timestamp'] = datetime.now().isoformat() + df['version_number'] = version_number + df['ref_exercise'] = ref_exercise + df['ncb'] = ncb + signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + client = oci.object_storage.ObjectStorageClient({}, signer = signer) # the first empyty bracket is an empty config + client.put_object("frcnomajoc7v",bucket_name,bucket_path+"1_1/"+csv_file_path,bytes(df.to_csv( index=False), encoding='utf-8')) + print("Finished uploading {}".format(csv_file_path)) + print(f"CSV saved to {csv_file_path}") + +def annex_1_2(fileName, bucket_path,file,bucket_name): + fileData=fileName.split("_") + csv_file_path = fileName[:-4]+".csv" + version_number = fileData[6] + ref_exercise = fileData[2] + ncb = fileData[4] + df = pd.read_excel(file, sheet_name="Entities to which data relates", skiprows=3) + df = df.dropna(axis=1, how='all').dropna(axis=0, how='all') + df['file_name'] = os.path.basename(fileName) + df['ingestion_timestamp'] = datetime.now().isoformat() + df['version_number'] = version_number + df['ref_exercise'] = ref_exercise + df['ncb'] = ncb + signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + client = oci.object_storage.ObjectStorageClient({}, signer = signer) # the first empyty bracket is an empty config + client.put_object("frcnomajoc7v",bucket_name,bucket_path+"1_2/"+csv_file_path,bytes(df.to_csv( index=False), encoding='utf-8')) + print("Finished uploading {}".format(csv_file_path)) + print(f"CSV saved to {csv_file_path}") + +def annex_2(fileName,bucket_path,file,bucket_name): + fileData=fileName.split("_") + # Parameters + version_number = fileData[6] + ref_exercise = fileData[2] + ncb = fileData[4] + + # Read the first sheet, skip the metadata rows + df = pd.read_excel(file.getvalue(), sheet_name="Data collection template", skiprows=6) + + # Clean empty rows/columns + df = df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + # Add metadata columns + df['file_name'] = os.path.basename(fileName) + df['ingestion_timestamp'] = datetime.now().isoformat() + df['version_number'] = version_number + df['ref_exercise'] = ref_exercise + df['ncb'] = ncb + + csvName=fileName[:-4]+"csv" + # Save to CSV + + signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + client = oci.object_storage.ObjectStorageClient({}, signer = signer) # the first empyty bracket is an empty config + client.put_object("frcnomajoc7v",bucket_name,bucket_path+"2/"+csvName,bytes(df.to_csv( index=False), encoding='utf-8')) + print("Finished uploading {}".format(csvName)) + print(f"CSV saved to {csvName}") + + + + + +def rqsd_preflow(secret_crt_id,secret_key_id,casper_api_url,collection_id): #downloads the list of files + + crt_path=os.getcwd()+"/rqsd_tst.crt" + key_path=os.getcwd()+"/rqsd_tst.key.pem" + try: + with open(key_path,"w") as keyfile: + keyfile.write(get_secret(secret_key_id)) + with open (crt_path,"w") as crtfile: + crtfile.write(get_secret(secret_crt_id)) + except: + print("Failed to retrieve certificates from secrets") + os.remove(crt_path) + os.remove(key_path) + raise(Exception) + + protected_resource_url=casper_api_url+"/casper-api/filevault/" + try: + + max_date=fileManager.execute_query("SELECT to_char(max(processing_end_time),'YYYY-MM-DD HH24:mi:ss') as MAX_PROCESSING_END_TIME FROM ct_ods.a_casper_filevault") + if max_date is not []: + filterString='isTest eq False and processingStatus eq "PS_COMPLETED" and processingEndTime gt '+max_date[0].split(' ')[0] + else: + filterString='isTest eq False and processingStatus eq "PS_COMPLETED"' + response=requests.get(protected_resource_url+"files/"+collection_id ,headers={"accept": "application/json"},cert=(crt_path,key_path), verify=False, params={"filter": filterString}) + + print(response.text) + files=response.json() + except: + print("Failed to retrieve ACC metadata, error during connection or request") + raise(Exception) + return files + + +def rqsd_process(files,casper_api_url,bucket_path,bucket_name): + crt_path=os.getcwd()+"/rqsd_tst.crt" + key_path=os.getcwd()+"/rqsd_tst.key.pem" + + # GET request to a protected + for downloadable in files: + try: + print("\n\n") + response=requests.get(casper_api_url+"/casper-api/filevault/download/"+str(downloadable["dcId"])+'/'+str(downloadable["fileID"]) ,headers={"accept": "application/json"},cert=(crt_path, key_path),verify=False) + rqsd_parser(downloadable["fileName"],bucket_path,io.BytesIO(response.content),bucket_name) + except: + print(f"Failed to upload file into target bucket, files saved locally in {os.getcwd()}") + os.remove(crt_path) + os.remove(key_path) + raise(Exception) + + +def add_a_key_column(headers, data_rows, task_history_key): + headers.insert(0, 'A_KEY') + for i, row in enumerate(data_rows, start=1): + a_key_value = int(task_history_key) * TASK_HISTORY_MULTIPLIER + i + row.insert(0, str(a_key_value)) + +def add_workflow_key_column(headers, data_rows, workflow_key): + headers.insert(1, 'A_WORKFLOW_HISTORY_KEY') + for row in data_rows: + row.insert(0, workflow_key) + +def initialize_config(config_file_path): + # Ensure the file exists + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + + # Load the configuration + with open(config_file_path, "r") as f: + config_data = yaml.safe_load(f) + + return config_data + + +def main(workflow_context, flow_config_path, env_config_path, env): + + #init setup + flow_info = initialize_config(flow_config_path) + envs_info = initialize_config(env_config_path) + environment_info = envs_info[env] + + current_time = datetime.now().strftime("%Y%m%d_%H%M%S") + + + a_task_history_key = initialize_task(workflow_context, flow_info['TASK_NAME']) + + # get list of files + try: + + files = rqsd_preflow(environment_info["CERTIFICATE_FILE"],environment_info["CERTIFICATE_KEY"],environment_info["CASPER_URL"],flow_info["COLLECTION_ID"]) + rqsd_process(files,environment_info["CASPER_URL"],flow_info["ODS_PREFIX"],environment_info["BUCKET"]) + except: + print("Failed to retrieve DEVO data, error during connection or request") + raise(Exception) + + # Finalize task + runManager.finalise_task(a_task_history_key, 'Y') diff --git a/python/connectors/casper/config/env_config.yaml b/python/connectors/casper/config/env_config.yaml new file mode 100644 index 0000000..7dfd169 --- /dev/null +++ b/python/connectors/casper/config/env_config.yaml @@ -0,0 +1,27 @@ +# Environment Configuration +dev: + BUCKET: "mrds_inbox_dev" + CERTIFICATE_KEY: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya7r33ocatalf6jn6kg2xjhnya6kazlqd3e5gw6yghpd5q" + CERTIFICATE_FILE: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyaeva4zvj6xdihljookamhse7jlyassfjb4p45xp46bwba" + CASPER_URL: "https://internet.api.casper.tst.aws.tadnet.net" + RQSD_COLLECTION_ID: "1537" +tst: + BUCKET: "mrds_inbox_tst" + CERTIFICATE_KEY: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya7r33ocatalf6jn6kg2xjhnya6kazlqd3e5gw6yghpd5q" + CERTIFICATE_FILE: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyaeva4zvj6xdihljookamhse7jlyassfjb4p45xp46bwba" + CASPER_URL: "https://internet.api.casper.tst.aws.tadnet.net" + RQSD_COLLECTION_ID: "1537" +acc: + BUCKET: "mrds_inbox_acc" + CERTIFICATE_KEY: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya5snmftggydoszwchjra3ifa4pyiilgc26uqlhejnhcca" + CERTIFICATE_FILE: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyaho5t4qgmlqctew6g6mcnwpz2p7z4nhxooyl6hc5sonfa" + CASPER_URL: "https://internet.api.casper.stg.aws.ecb.de" + RQSD_COLLECTION_ID: "1116" +prd: + BUCKET: "mrds_inbox_prd" + CERTIFICATE_KEY: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyahmv5sopfsv7nytxdyycehoyl5pd7sz5t2drn27qaneta" + CERTIFICATE_FILE: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyame3chyqs6cdl2igeyrvzpj3s4vrndhbgeayt533uhgqa" + CASPER_URL: "https://internet.api.casper.prd.aws.ecb.de" + RQSD_COLLECTION_ID: "1030" + + diff --git a/python/connectors/devo/config/env_config.yaml b/python/connectors/devo/config/env_config.yaml new file mode 100644 index 0000000..20a685f --- /dev/null +++ b/python/connectors/devo/config/env_config.yaml @@ -0,0 +1,25 @@ +# Environment Configuration +dev: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_dev" + DEVO_USERNAME: "ap-informatica-ipcwt" + DEVO_HOSTNAME: "impala-proxy-devo-lab21-impala01.dw-devo-lab21.om2y56.b0.cloudera.site" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyavrevwxke46wjgj5nz3cc5kwwsybmngbji4zepones55q" +tst: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_tst" + DEVO_USERNAME: "ap-informatica-ipcwt" + DEVO_HOSTNAME: "t-impala.devo.escb.eu" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyaxxx7yfifpgpdnxuj6dcowpoktwa6745kwwpezysd44oa" +acc: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_acc" + DEVO_USERNAME: "ap-informatica-ipcwa" + DEVO_HOSTNAME: "impala-proxy-devo-acc21-impala01.dw-devo-acc21.inym23.b0.cloudera.site" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya4uttfadlzreloouw2e5bifgl2dvihffym5xoq3b3jmva" +prd: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_prd" + DEVO_USERNAME: "ap-informatica-ipcwp" + DEVO_HOSTNAME: "impala-proxy-devo-prd21-impala01.dw-devo-prd21.inym23.b0.cloudera.site" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyanbahqlucid7qtzvoohsf4xrlul7cvhlsqttmbro4n66a" diff --git a/python/connectors/devo/config/env_config_rqsd.yaml b/python/connectors/devo/config/env_config_rqsd.yaml new file mode 100644 index 0000000..663e8c6 --- /dev/null +++ b/python/connectors/devo/config/env_config_rqsd.yaml @@ -0,0 +1,25 @@ +# Environment Configuration +dev: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_dev" + DEVO_USERNAME: "ap-devo-rqsd-tst" + DEVO_HOSTNAME: "t-impala.devo.escb.eu" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyap6wtzobzob7qizvk4nocszlcaxhwijgzejbvryt3uzbq" +tst: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_tst" + DEVO_USERNAME: "ap-devo-rqsd-tst" + DEVO_HOSTNAME: "t-impala.devo.escb.eu" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyap6wtzobzob7qizvk4nocszlcaxhwijgzejbvryt3uzbq" +acc: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_acc" + DEVO_USERNAME: "ap-devo-rqsd-acc" + DEVO_HOSTNAME: "impala-proxy-devo-acc21-impala01.dw-devo-acc21.inym23.b0.cloudera.site" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyamzhgatnso57mubvg3c6k4ens3orcx4dieo6efukuvm4a" +prd: + BUCKET_NAMESPACE: "frcnomajoc7v" + BUCKET: "mrds_inbox_prd" + DEVO_USERNAME: "ap-devo-rqsd-prd" + DEVO_HOSTNAME: "impala-proxy-devo-prd21-impala01.dw-devo-prd21.inym23.b0.cloudera.site" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyawpahgevgxv6csqnwil3p37vi6pthl466onnkg6k7undq" \ No newline at end of file diff --git a/python/connectors/devo/devo_connector.py b/python/connectors/devo/devo_connector.py new file mode 100644 index 0000000..b66f02a --- /dev/null +++ b/python/connectors/devo/devo_connector.py @@ -0,0 +1,259 @@ +# devo_impala_exporter.py + +import os +import io +import yaml +import datetime +import logging +from typing import Any, Dict, List, Optional, Tuple + +import pandas as pd +from mrds.utils.secrets import get_secret +import mrds.utils.manage_runs as runManager +import mrds.utils.objectstore as objectstore + +import oci + +from impala.dbapi import ( + connect, + ProgrammingError, + DatabaseError, + IntegrityError, + OperationalError, +) +from impala.error import HiveServer2Error + +TASK_HISTORY_MULTIPLIER = 1_000_000_000 + + +class DevoConnector: + """ + Export the result of an Impala (DEVO) query to OCI Object Storage as CSV, + while recording task run metadata via mrds.runManager. + + Usage: + exporter = DevoImpalaExporter( + flow_config_path="/path/to/flow.yaml", + env_config_path="/path/to/env.yaml", + env="dev", + logger=my_logger, # optional + oci_client=my_object_storage, # optional ObjectStorageClient + oci_signer=my_signer, # optional signer (used if client not provided) + ) + exporter.run({"run_id": 34, "a_workflow_history_key": 6}) + """ + + def __init__( + self, + flow_config_path: str, + env_config_path: str, + env: str, + logger: Optional[logging.Logger] = None, + oci_client: Optional[oci.object_storage.ObjectStorageClient] = None, + oci_signer: Optional[Any] = None, + ) -> None: + self.flow_info = self._initialize_config(flow_config_path) + envs_info = self._initialize_config(env_config_path) + BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE", "frcnomajoc7v") + + if env not in envs_info: + raise KeyError(f"Environment '{env}' not found in {env_config_path}") + self.environment_info = envs_info[env] + self.environment_info["BUCKET_NAMESPACE"]=BUCKET_NAMESPACE + self.env = env + + # logging + self.logger = logger or self._default_logger(self.flow_info.get("TASK_NAME", "devo_task")) + + # OCI client/signer + self.oci_client = oci_client + self.oci_signer = oci_signer + + # ------------------------- + # Public API + # ------------------------- + + def run(self, workflow_context: Dict[str, Any]) -> None: + """Main entry point; executes query, uploads CSV, and finalizes task.""" + task_name = self.flow_info["TASK_NAME"] + a_task_history_key = self._initialize_task(workflow_context, task_name) + + try: + # credentials + devo_secret_name = self.environment_info["DEVO_SECRET"] + password = get_secret(devo_secret_name) + self.logger.info("Retrieved secret for DEVO connection.") + + # query + query = self.flow_info["DEVO_QUERY"] + user = self.environment_info["DEVO_USERNAME"] + host = self.environment_info["DEVO_HOSTNAME"] + + columns, data, rowcount = self._execute_query(query=query, user=user, hostname=host, password=password) + df = self._tuple_to_dataframe((columns, data)) + self.logger.info("Query executed and DataFrame created with %d rows.", len(df)) + + # upload + if rowcount > 0: + csv_name = f"{self.flow_info['OUTPUT_TABLE']}.csv" + file_path = self._compose_object_path(self.flow_info["ODS_PREFIX"], csv_name) + self._upload_dataframe_to_oci(df, csv_name, file_path) + self.logger.info("Finished uploading %s to %s.", csv_name, file_path) + else: + return 0 + + # success + runManager.finalise_task(a_task_history_key, "Y") + self.logger.info("Task %s finalized successfully.", task_name) + return rowcount + + except Exception as e: + # failure + self.logger.exception("Run failed: %s", e) + try: + runManager.finalise_task(a_task_history_key, "N") + finally: + # re-raise for upstream handling if used as a library + raise + + # ------------------------- + # Impala / DEVO + # ------------------------- + + @staticmethod + def _get_impala_connection(hostname: str, user: str, secret: str): + return connect( + host=hostname, + port=443, + auth_mechanism="PLAIN", + user=user, + password=secret, + use_http_transport=True, + http_path="cliservice", + use_ssl=True, + ) + + def _execute_query(self, query: str, user: str, hostname: str, password: str) -> Tuple[List[str], List[List[Any]]]: + conn = self._get_impala_connection(hostname, user, password) + cursor = None + self.logger.info("Executing Impala query against host '%s' as user '%s'.", hostname, user) + try: + cursor = conn.cursor() + cursor.execute(query) + + if query.strip().lower().startswith("select") or "select" in query.strip().lower() : + rows = cursor.fetchall() + columns = [col[0] for col in cursor.description] + return columns, rows, cursor.rowcount + else: + # Non-SELECT: return rowcount (still return a columns list for consistency) + return [], [[cursor.rowcount]] + + except OperationalError as oe: + raise RuntimeError("Failed to connect to Impala: " + str(oe)) from oe + except ProgrammingError as pe: + raise ValueError("Query syntax error: " + str(pe)) from pe + except IntegrityError as ie: + raise PermissionError("Insufficient permissions: " + str(ie)) from ie + except DatabaseError as db_err: + raise RuntimeError("Database error: " + str(db_err)) from db_err + except HiveServer2Error as au_err: + raise PermissionError("HiveServer2Error error: " + str(au_err)) from au_err + except Exception as e: + raise RuntimeError("An unexpected error occurred: " + str(e)) from e + finally: + try: + if cursor: + cursor.close() + finally: + try: + conn.close() + except Exception: + # log but don't mask the original exception + self.logger.warning("Failed to close Impala connection cleanly.", exc_info=True) + + # ------------------------- + # OCI Upload + # ------------------------- + + + def _upload_dataframe_to_oci(self, df: pd.DataFrame, csv_name: str, object_path: str) -> None: + namespace = self.environment_info["BUCKET_NAMESPACE"] + bucket = self.environment_info["BUCKET"] + # convert DataFrame to CSV bytes without index + csv_bytes = df.to_csv(index=False).encode("utf-8") + client=objectstore.get_client() + client.put_object(namespace, bucket, object_path, csv_bytes) + self.logger.info("CSV '%s' uploaded to bucket '%s' (ns: '%s', key: '%s').", csv_name, bucket, namespace, object_path) + + # ------------------------- + # Utilities + # ------------------------- + + @staticmethod + def _tuple_to_dataframe(data_tuple: Tuple[List[str], List[List[Any]]]) -> pd.DataFrame: + columns, data = data_tuple + if not columns: + # for non-SELECT queries we returned rowcount; represent it in a DataFrame + return pd.DataFrame(data, columns=["rowcount"]) + return pd.DataFrame(data, columns=columns) + + @staticmethod + def _initialize_config(config_file_path: str) -> Dict[str, Any]: + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + with open(config_file_path, "r") as f: + return yaml.safe_load(f) + + @staticmethod + def _initialize_task(workflow_context: Dict[str, Any], task_name: str) -> int: + return runManager.init_task( + task_name, + workflow_context["run_id"], + workflow_context["a_workflow_history_key"], + ) + + @staticmethod + def add_a_key_column(headers: List[str], data_rows: List[List[Any]], task_history_key: int) -> None: + """Optionally add an A_KEY column (kept for parity with original script).""" + headers.insert(0, "A_KEY") + for i, row in enumerate(data_rows, start=1): + a_key_value = int(task_history_key) * TASK_HISTORY_MULTIPLIER + i + row.insert(0, str(a_key_value)) + + @staticmethod + def add_workflow_key_column(headers: List[str], data_rows: List[List[Any]], workflow_key: int) -> None: + """Optionally add the workflow key column right after A_KEY if present, otherwise at position 0.""" + insert_idx = 1 if headers and headers[0] == "A_KEY" else 0 + headers.insert(insert_idx, "A_WORKFLOW_HISTORY_KEY") + for row in data_rows: + row.insert(insert_idx, workflow_key) + + @staticmethod + def _compose_object_path(prefix: str, filename: str) -> str: + if prefix.endswith("/"): + return f"{prefix}{filename}" + return f"{prefix}/{filename}" + + @staticmethod + def _default_logger(task_name: str) -> logging.Logger: + logger = logging.getLogger(f"{task_name}_logger") + if not logger.handlers: + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + fmt = logging.Formatter(f"%(asctime)s [{task_name}] %(levelname)s: %(message)s") + handler.setFormatter(fmt) + logger.addHandler(handler) + return logger + + +# Optional: quick-run convenience if you ever want to execute this module directly. +if __name__ == "__main__": + # Example only—adjust paths/env/context as needed or remove this block. + exporter = DevoConnector( + flow_config_path="/home/dbt/Marco/mrds_elt/airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml", + env_config_path="/home/dbt/Marco/mrds_elt/python/connectors/devo/config/env_config_rqsd.yaml", + env="dev", + ) + exporter.run({"run_id": 34, "a_workflow_history_key": 6}) diff --git a/python/connectors/tms/TMSDBT.py b/python/connectors/tms/TMSDBT.py new file mode 100644 index 0000000..1e9206d --- /dev/null +++ b/python/connectors/tms/TMSDBT.py @@ -0,0 +1,294 @@ + + +import argparse +from TMSQuery import XMLQuery + +import mrds.utils.objectstore +import tempfile +import re +import csv +from io import StringIO + +import os.path +import os, psutil +import sys + + +namespace = os.getenv("BUCKET_NAMESPACE", "frcnomajoc7v") + +def memory_usage(): + # return the memory usage in percentage like top + process = psutil.Process(os.getpid()) + mem = process.memory_info().rss/(1024*1024*1024) + return mem + + +def protect_keyword(s): + s = s.lower() + s = s.replace(' ', '_') + + match s.lower(): + case 'comment': + #return '"comment"' + return 'comment_' + case 'date': + #return '"date"' + return 'date_' + case 'number': + #return '"number"' + return 'number_' + case _: + return s + + +cModelsDir = sys.path[0] + '/../dbt/mrds/models/ods/' +cDatasetMultiplier = 10000000 + + + +parser = argparse.ArgumentParser() +parser.add_argument("command", choices=['create-model', 'create-oracle-table', 'retrieve'], help="create-model retrieve") +parser.add_argument("-n", "--name", help="Name") +parser.add_argument("-u", "--url", required=True, help="URL of TMS service") +parser.add_argument("-U", "--user", required=True, help="TMS user") +parser.add_argument("-P", "--password", required=True, help="TMS password") +parser.add_argument("-x", "--xmlfile", help="XML file") +parser.add_argument("-l", "--layoutfile", help="layout file") +parser.add_argument("-f", "--format", help="output format") +parser.add_argument("-p", "--parameter", action="append", help="Parameter") +parser.add_argument("-c", "--column", action="append", help="Additional column") +parser.add_argument("-d", "--destination", help="destination") +parser.add_argument("-s", "--dataset", help="data set ID", type=int) +parser.add_argument("-v", "--version", help="data model version", type=int, default=1) +args = parser.parse_args() + + +query = XMLQuery() + +if args.xmlfile: + with open(args.xmlfile) as f: + xml = f.read() + query.xml = xml + + +if args.layoutfile: + with open(args.layoutfile) as f: + layout = f.read() + query.layout = layout + +if args.format: + query.format = args.format + +if args.parameter: + for p in args.parameter: + [name, value] = p.split('=', 1) + query.parameter[name] = value + +additional_columns = [] +if args.column: + for p in args.column: + [name, value] = p.split('=', 1) + t = re.split(r'(?:\|)|(?:/)|(?::)', name, maxsplit = 2) + name = t[0] + type = None + if len(t) == 2: + type = t[1] + if not type: + type = 'varchar2(255)' + additional_columns.append((name, type, value)) + + +query.normalize_output() + +from pathlib import Path +import pprint +p = Path('/tmp/kurt.xml') +p.write_text(str(query)) + + +if args.command == 'create-oracle-table': + + d = query.describe(args.url, args.user, args. password) + + + columns = [" a_key number(38, 0)", "a_workflow_history_key number(38, 0)"] + for c in additional_columns: + columns.append("%s %s"%(c[0], c[1])) + + for col in d: + name = protect_keyword(col[0]) + match col[1]: + case 'text': + columns.append(name + " varchar2(512 char)") + case 'int': + columns.append(name + " number(38,0)") + case 'money': + columns.append(name + " number(19,4)") + case 'floating': + columns.append(name + " binary_double") + case 'datetime': + columns.append(name + " date") + case 'integer': + columns.append(name + " number(12, 0)") + + + sql = "create table ct_et_templates." + args.name + " (\n" + sql = sql + ",\n ".join(columns) + sql = sql + "\n)\n" + + if not args.destination or args.destination == '-': + print(sql) + else: + with open(args.destination, 'w') as f: + f.write(sql) + + + +elif args.command == 'create-ods-model': + + d = query.describe(args.url, args.user, args. password) + + file_name = cModelsDir + args.name + '.yml' + f = open(file_name, 'w') # open file in append mode + + f.write('version: %d\n' % args.version) + + f.write('models:' + '\n') + f.write(' - name: ' + args.name + '_dbt\n') + f.write(' description: "A starter dbt model"' + '\n') + f.write(' columns:' + '\n') + for col in d: + f.write(' - name: ' + col[0] + '\n') + f.write(' data_type: ' + col[1] + '\n') + f.close() + + + file_name = cModelsDir + args.name + '.sql' + f = open(file_name, 'w') # open file in append mode + + + if args.destination and args.destination != '-': + if ':' in args.destination: + dest = args.destination.split(':', 2) + path = dest[1] + else: + path = args.destination + prefix = os.path.dirname(path) + else: + prefix = 'INBOX/TMS/' + args.name.upper() + '/' + + + + + pars = "ptablename => '%s', ptemplatetablename => 'ou_tms.%s', pprefix => '%s'" % (args.name, args.name, prefix) + print(f"creating table {args.name}") + f.write('{{\n config(\n post_hook = "call ct_mrds.file_manager.create_external_table(%s)"\n )\n}}\n\n' % pars) + f.write("{{ config(materialized='table') }}" + "\n") + f.write('with source_data as (' + "\n") + columns = [] + columns.append("cast (1 as number(38,0)) as a_key") + columns.append("cast (1 as number(38,0)) as a_workflow_history_key") + for col in d: + name = protect_keyword(col[0]) + match col[1]: + case 'text': + columns.append("cast ('x' as varchar2(255 char)) as " + name) + case 'int': + columns.append("cast (1 as number(38, 0)) as " + name) + case 'money': + columns.append("cast (1.0 as number(19,4)) as " + name) + case 'floating': + columns.append("cast (1.0 as binary_double) as " + name) + case 'datetime': + columns.append("cast (sysdate as date) as " + name) + case 'integer': + columns.append("cast (1 as number(12, 0)) as " + name) + f.write(' select\n ' + ',\n '.join(columns) + '\n') + f.write(')\nselect * from source_data\n ') + f.close() + + + +elif args.command == 'retrieve': + ret = query.execute(args.url, args.user, args. password) + + if query.format in ('scsv', 'standard_csv') and args.dataset: + + # Save result to temporary spooled file for further processing + # We avoid doing this in memory to prevent issues with flow EffectivePermissions + + f = tempfile.SpooledTemporaryFile(mode = 'w+', max_size = 200*1024*1024) + f.write(ret) + del ret + f.seek(0) + + + # Replace embedded newlines for '
' + reader = csv.reader(f) + + sio = StringIO() + + writer = csv.writer(sio) + for l in reader: + l_tmp = [s.replace('\n', '
') for s in l] + writer.writerow(l_tmp) + f.close() + + + # Necessary to read the data into an array of lines for further processing + sio.seek(0) + lines_tmp = sio.readlines() + del sio + + + if not lines_tmp: + ret = "" + else: + # Adding artificial columns A_KEY and A_WORKFLOW_HISTORY_KEY and added columns + additional_headers = [t[0] for t in additional_columns] + additional_values = [t[2] for t in additional_columns] + headers = ['A_KEY','A_WORKFLOW_HISTORY_KEY'] + additional_headers + [protect_keyword(h) for h in lines_tmp[0].split(',')] + lines = [','.join(headers) ] + + i = 0 + for l in lines_tmp[1:]: + lines.append(str(args.dataset*cDatasetMultiplier + i) + ',' + str(args.dataset) + ',' + ','.join(additional_values + [l]) ) + i += 1 + + del lines_tmp + + # Spooling again to temporary file to avoid duplication memory needs + f = tempfile.SpooledTemporaryFile(mode = 'w+', max_size = 200*1024*1024) + f.writelines(lines) + del lines + f.seek(0) + ret = f.read() + f.close() + + if not args.destination or args.destination == '-': + print(ret, end='') + elif ':' not in args.destination: + with open(args.destination, 'w') as f: + f.write(ret) + else: + f = tempfile.NamedTemporaryFile(delete = False, mode = 'w', prefix = 'TMSDBT-', suffix = '.csv') + f.write(ret) + f.close() + + dest = args.destination.split(':', 2) + bucket = dest[0] + dirname = os.path.dirname(dest[1]) + filename = os.path.basename(dest[1]) + client = mrds.utils.objectstore.get_client() + with open(f.name, "r") as file: + print(file.read()) + mrds.utils.objectstore.upload_file(client, f.name,namespace, bucket, dirname, filename) + + os.remove(f.name) + + if ret: + sys.exit(0) + else: + sys.exit(1) + + diff --git a/python/connectors/tms/TMSQuery.py b/python/connectors/tms/TMSQuery.py new file mode 100644 index 0000000..9bf0a31 --- /dev/null +++ b/python/connectors/tms/TMSQuery.py @@ -0,0 +1,197 @@ + +import xml.etree.ElementTree as ET +import re +import base64 +import sys + + +class XMLQuery: + + + def __init__(self, xml = None): + + self._format = 'xml' + self._layout = '' + self._parameter = {} + + if xml: + + self._parse_xml(xml) + + + def _parse_xml(self, xml): + + self._tree = ET.fromstring(xml) + + + layout_b64 = self._tree.find('layout').text + self._layout = base64.b64decode(layout_b64).decode('utf-8') + + self._format = self._tree.find('format').get('type') + + + self._parameter = {} + for p in self._tree.findall('parameters/parameter'): + self._parameter[p.get('name')] = p.text + + + + def execute(self, url, user, password): + + # curl -X POST --basic -u schilli:chili03 --data @tms_activity_interval.xml https://tmsxd104.ecbt1.tadnet.net:9443/report/ + + import requests + from requests.auth import HTTPBasicAuth + + + data = str(self) + basic = HTTPBasicAuth(user, password) + + response = requests.post(url, data=data, auth=basic, verify=False) + + if response.status_code == 200: + response.encoding = "utf-8" + return response.text + else: + return None + + + def describe(self, url, user, password): + + orig_format = self.format + self.format = 'xml' + + ret = self.execute(url, user, password) + + m = re.match('^.*?\.*?\<\/PlainRow\>', ret, re.DOTALL) + s = m[0] + '\n' + + tree = ET.fromstring(s) + + ret = [] + row = tree.find('PlainRow') + for c in row.findall('Column'): + #name = c.get('name') + name = c.text + type = c.get('type') + if type == 'unknown': type = 'integer' + + ret.append((name, type)) + + return ret + + + + def describe_simple(url, user, password, xml): + + query = XMLQuery(xml) + + query.format='xml' + + ret = query.execute(url = url, user = user, password = password) + + tree = ET.fromstring(ret) + + ret = [] + row = tree.find('PlainRow') + for c in row.findall('Column'): + #name = c.get('name') + name = c.text + type = c.get('type') + if type == 'unknown': type = 'integer' + + ret.append((name, type)) + + return ret + + + def normalize_output(self, date_format = 'dd/MM/yyyy', time_format = 'HH:mm:ss'): + + lines = self.layout.splitlines() + + lines = [re.sub(r'^date_format\s*=.*', 'date_format=' + date_format, l) for l in lines] + lines = [re.sub(r'^time_format\s*=.*', 'time_format=' + time_format, l) for l in lines] + lines = [re.sub(r'^NoNumberFormatting\s*=.*', 'NoNumberFormatting=1', l) for l in lines] + + self.layout = '\n'.join(lines) + + + + def __setattr__(self, name, value): + if name == 'format' and value not in ('bin','xml','xml3','html','txt','csv','standard_csv', 'scsv', 'pdf'): + raise Exception("Invalid report format '" + value + "'") + + if not name.startswith('_'): + name = '_' + name + + if name == '_layout' and not value.endswith('\n'): + value = value + '\n' + + if name == '_xml': + self._parse_xml(value) + return + + try: + self.__dict__[name] = value + except KeyError: + raise AttributeError + + + + def __getattr__(self, name): + + if not name.startswith('_'): + name = '_' + name + + try: + return self.__dict__[name] + except KeyError: + raise AttributeError(name) + + + def __str__(self): + parameters = '' + for k in self._parameter: + parameters = parameters + "\n%s" % (k, self._parameter[k]) + + layout_b64 = base64.b64encode(self.layout.encode('utf-8')).decode('utf-8') + return ('\n' + \ + '\n' + \ + ' \n' + \ + ' \n%s\n' + \ + ' %s\n' + \ + '') % (self._format, layout_b64, parameters) + + + + + + +if __name__ == "__main__": + + + file = sys.argv[1] + print(file) + + with open(file) as f: + xml = f.read() + + + query = XMLQuery(xml) + + print(query.layout) + query.normalize_output() + print(query.layout) + + + + #query.format='xml' + + #ret = query.execute(url = 'https://tmsxd104.ecbt1.tadnet.net:9443/report/', user = 'schilli', password = 'chili03') + + #print(ret) + + + desc = XMLQuery.describe_simple(url = 'https://tmsxd104.ecbt1.tadnet.net:9443/report/', user = 'schilli', password = 'chili03', xml = xml) + + print(str(desc)) diff --git a/python/connectors/tms/sample_DAG.py b/python/connectors/tms/sample_DAG.py new file mode 100644 index 0000000..cd21b64 --- /dev/null +++ b/python/connectors/tms/sample_DAG.py @@ -0,0 +1,355 @@ +""" +DAG: w_ODS_TMS_TRANSACTION (expanded example) +Purpose: + - Load layout+parameter metadata from TMS-layouts/w_ODS_TMS_TRANSACTION.yml + - Call connectors/tms/TMSDBT.py to retrieve data into CSV in object storage + - On first run, generate Oracle DDL and create an external table + - Process file and record status in MRDS workflow tables +Notes: + - This is an expanded, readable version of the factory-generated DAG. + - Replace paths/usernames/password references as appropriate. +""" + +import copy +import itertools +import json +import logging +import os +import re +import subprocess +import sys +import traceback +from datetime import datetime, timedelta +from pathlib import Path + +import yaml +from airflow import DAG +from airflow.operators.python import PythonOperator +from pytz import timezone + +# --- Project-specific deps (must exist in your Airflow image) --- +from mrds.core import main # noqa: F401 # imported to mirror the factory env +import mrds.utils.manage_files as mf +import mrds.utils.manage_runs as mr + +# ---------- Paths & constants ---------- +gScriptDir = Path(globals().get("__file__", "./_")).absolute().parent +gDataDir = str(gScriptDir / "TMS-layouts") + "/" +gConfigDir = str(gScriptDir / "config") +gConnDir = "/opt/airflow/python/connectors/tms/" +gTableDir = str(gScriptDir / "TMS-tables") + "/" + +DAG_NAME = "w_ODS_TMS_TRANSACTION" +ODS_TABLE = DAG_NAME +DATABASE_NAME = "ODS" +WF_NAME = DAG_NAME + +default_args = { + "owner": "ecb", + "depends_on_past": False, + "email_on_failure": False, + "email_on_retry": False, + "retries": 0, + "execution_timeout": timedelta(minutes=60), + "retry_delay": timedelta(minutes=5), +} + +# ---------- Load YAML configs once on parse ---------- +with open(gDataDir + DAG_NAME + ".yml", "r") as f: + report_desc = yaml.safe_load(f) or {} + +with open(gConfigDir + "/TMS.yml", "r") as f: + tms_config = yaml.safe_load(f) + +# TMS + storage config +tms_url = tms_config["TMS-URL"] +tms_user = tms_config["TMS-user"] +tms_pwd = tms_config["TMS-password"] +prefix = tms_config["dest-prefix"] + DAG_NAME + "/" + DAG_NAME + "/" +data_prefix = tms_config["data-prefix"] + DAG_NAME + "/" +dest = tms_config["dest-bucket"] + ":" + prefix + +# Visible vs hidden params (from layout YAML) +params_visible = {} +params_hidden = {} +params_dict = report_desc.get("parameters") or {} +for p, meta in params_dict.items(): + val = meta.get("value", None) + if not meta.get("hidden", False): + params_visible[p] = val + else: + params_hidden[p] = val + +# ---------- Helpers (parameter handling) ---------- +def _enum_param_combinations_recursive(params, keys): + """ + Build all combinations of params (cartesian product), supporting + 'column()' derived lists aligned by index. + """ + k = None + result = [] + keys = list(keys) # safe copy + + while keys: + k = keys.pop(0) + v = params[k] + if v or v == "": + break + + if not k: + return [] + + v = v if isinstance(v, list) else [v] + + # derived columns aligned with v (same length) + derived_columns = [] + # params_dict[k] holds the definition, not just the value + pdef = params_dict.get(k, {}) + for c in list(pdef): + if re.match(r"column\(.*\)$", c): + vtmp = pdef[c] + vtmp = vtmp if isinstance(vtmp, list) else [vtmp] + derived_columns.append((c, vtmp)) + + if not keys: + for i, value in enumerate(v): + row = [(k, value)] + for col_key, aligned_values in derived_columns: + row.append((col_key, aligned_values[i])) + result.append(row) + return result + + combinations = _enum_param_combinations_recursive(params, keys) + for row in combinations: + for i, vtmp in enumerate(v): + new_row = copy.deepcopy(row) + new_row.append((k, vtmp)) + for col_key, aligned_values in derived_columns: + new_row.append((col_key, aligned_values[i])) + result.append(new_row) + + return result + + +def _enum_param_combinations(params, sequential=False): + # Sequential path omitted (buggy in factory; not used there either) + return _enum_param_combinations_recursive(params, list(params)) + + +def _allowed_select(table, expression, condition="1 = 1"): + """ + Guarded select used by eval_params(select(...)). + Whitelist tables to avoid arbitrary reads. + """ + if table.upper() not in ( + ODS_TABLE.upper(), + "DUAL", + "CT_MRDS.A_WORKFLOW_HISTORY", + ): + raise Exception(f"Not allowed to select from {table}") + res = mr.select_ods_tab(table, expression, condition) + return res[0] + + +def _eval_param(v): + """ + Evaluate special functional values: + - select(...) => guarded DB helper above + - eval(...) => strongly discouraged; keep disabled or restricted + """ + s = str(v) if v is not None else "" + if re.match(r"\s*select\(.*\)", s): + # Expose only 'select' symbol to eval + return eval(s, {"select": _allowed_select}, {}) + if re.match(r"\s*eval\(.*\)\s*$", s): + # If you really must support eval, strictly sandbox or remove this path. + raise ValueError("eval(...) not allowed in this hardened DAG.") + return v + + +def _finalize_param_list(param_list): + """ + Apply replacements and drop virtual params according to YAML definitions. + """ + d = dict(param_list) + + # Replace parameter tokens inside another parameter (string replace) + for p, meta in params_dict.items(): + if meta.get("replace_parameter"): + target = meta["replace_parameter"] + if target in d and p in d and isinstance(d[target], str): + d[target] = d[target].replace(p, str(d[p])) + + # Drop 'virtual' params + cleaned = [] + for k, v in d.items(): + meta = params_dict.get(k, {}) + if not meta.get("virtual", False): + cleaned.append((k, v)) + return cleaned + + +# ---------- Core work ---------- +def execute_report(**context): + """ + For each parameter combination: + - create workflow key + - call TMSDBT.py retrieve to land CSV + - if first time, create Oracle table from generated DDL + - process file, finalize workflow Y/N + """ + logger = logging.getLogger("airflow.task") + logger.setLevel(logging.DEBUG) + + run_id = context["dag_run"].run_id + all_params = {**params_visible, **params_hidden} + + # 1) Compute combinations + combos = _enum_param_combinations(all_params) + + # 2) Evaluate select(...) etc and finalize + evaluated = [] + for combo in combos or [[]]: + # first pass: special evaluations + pair_list = [] + for k, v in combo: + pair_list.append((k, _eval_param(v))) + # second pass: replacements + pruning + evaluated.append(_finalize_param_list(pair_list)) + + # if no combos at all, ensure we run once + if not evaluated: + evaluated = [[]] + + # Timing + workflow + ts = "{:%Y%m%d_%H%M%S}".format(datetime.now(timezone("Europe/Berlin"))) + + for idx, param_list in enumerate(evaluated, start=1): + wf_key = mr.init_workflow(DATABASE_NAME, WF_NAME, run_id) + file_name = f"{WF_NAME}.{wf_key}.{ts}.csv" + + try: + # Build connector command safely (no shell quoting games) + cmd = [ + sys.executable, # 'python' + os.path.join(gConnDir, "TMSDBT.py"), + "retrieve", + "--name", WF_NAME, + "--url", tms_url, + "-U", tms_user, + "--password", tms_pwd, + "--layoutfile", gDataDir + DAG_NAME + ".fkr", + "-f", "scsv", + "--dataset", str(wf_key), + "-d", dest + file_name, + ] + + # Map params to -p or -c switches + for k, v in param_list: + sval = "" if v is None else str(v).rstrip() + m = re.match(r"column\((.*)\)$", k) + if m: + cmd.extend(["-c", f'{m.group(1)}={sval}']) + else: + cmd.extend(["-p", f"{k}={sval}"]) + mr.set_workflow_property(wf_key, DATABASE_NAME, k, sval) + + logger.debug("Running connector: %s", json.dumps(cmd)) + res = subprocess.run(cmd, capture_output=True, check=False) + logger.debug("stdout: %s", res.stdout.decode(errors="ignore")) + logger.debug("stderr: %s", res.stderr.decode(errors="ignore")) + + if res.returncode is None: + raise RuntimeError("Connector returned no status") + if res.returncode == 1: + logger.info("No data returned for wf_key=%s (continuing)", wf_key) + mr.finalise_workflow(wf_key, "Y") + continue + if res.returncode != 0: + raise RuntimeError(f"Connector failed (rc={res.returncode})") + + # Data landed -> ensure source config exists, bootstrap table if needed + cfg = mf.execute_query( + "select * from CT_MRDS.A_SOURCE_FILE_CONFIG " + f"where a_source_key = 'TMS' and table_id = '{ODS_TABLE}'" + ) + + if not cfg: + # Generate DDL file + ddl_cmd = [ + sys.executable, + os.path.join(gConnDir, "TMSDBT.py"), + "create-oracle-table", + "--name", WF_NAME, + "--url", tms_url, + "-U", tms_user, + "--password", tms_pwd, + "--layoutfile", gDataDir + DAG_NAME + ".fkr", + "-d", gTableDir + WF_NAME + ".sql", + ] + for k, v in param_list: + sval = "" if v is None else str(v).rstrip() + m = re.match(r"column\((.*)\)$", k) + if m: + ddl_cmd.extend(["-c", f'{m.group(1)}={sval}']) + else: + ddl_cmd.extend(["-p", f"{k}={sval}"]) + + logger.debug("Generating DDL: %s", json.dumps(ddl_cmd)) + ddl_res = subprocess.run(ddl_cmd, capture_output=True, check=True) + logger.debug("DDL stdout: %s", ddl_res.stdout.decode(errors="ignore")) + logger.debug("DDL stderr: %s", ddl_res.stderr.decode(errors="ignore")) + + # Execute DDL and create external table + sql = Path(gTableDir + WF_NAME + ".sql").read_text() + mf.execute(sql) + mf.add_column_date_format( + f"CT_ET_TEMPLATES.{ODS_TABLE}", "DEFAULT", "DD/MM/YYYY HH24:MI:SS" + ) + mf.create_external_table(ODS_TABLE, f"CT_ET_TEMPLATES.{ODS_TABLE}", data_prefix) + mf.add_source_file_config( + "TMS", + "INPUT", + DAG_NAME, + DAG_NAME, + r".*\.csv", + ODS_TABLE, + f"CT_ET_TEMPLATES.{ODS_TABLE}", + ) + + # Process landed file (register, move, etc. as per your mf impl) + mf.process_source_file(prefix, file_name) + mr.finalise_workflow(wf_key, "Y") + + except BaseException as ex: + # rich error logging, then mark workflow failed and re-raise + ex_type, ex_value, ex_tb = sys.exc_info() + tb = traceback.extract_tb(ex_tb) + stack = [ + f"File: {t[0]}, Line: {t[1]}, Func: {t[2]}, Code: {t[3]}" + for t in tb + ] + logging.error("Exception type: %s", ex_type.__name__) + logging.error("Exception message: %s", ex_value) + logging.error("Stack trace: %s", stack) + mr.finalise_workflow(wf_key, "N") + raise + + +# ---------- DAG definition ---------- +with DAG( + dag_id=DAG_NAME, + default_args=default_args, + description=DAG_NAME, + schedule_interval=None, # manual trigger + params=params_visible, # visible-only; hidden merged inside task + start_date=datetime(2025, 1, 1), + catchup=False, + tags=[DAG_NAME], +) as dag: + + retrieve_report = PythonOperator( + task_id="retrieve_report", + python_callable=execute_report, + execution_timeout=timedelta(minutes=30), + ) diff --git a/python/devo_replicator/config/.gitkeep b/python/devo_replicator/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/devo_replicator/config/env_config.yaml b/python/devo_replicator/config/env_config.yaml new file mode 100644 index 0000000..97b12ee --- /dev/null +++ b/python/devo_replicator/config/env_config.yaml @@ -0,0 +1,86 @@ +dev: + DEVO_USERNAME: "ap-devo_lab-mrds" + IMPALA_HOSTNAME: "impala-proxy-devo-lab21-impala01.dw-devo-lab21.om2y56.b0.cloudera.site" + HIVE_HOSTNAME: 'hs2-devo-lab21-hive01.dw-devo-lab21.om2y56.b0.cloudera.site' + RANGER_HOSTNAME: "https://devo-lab21-dl-gateway.devo-lab.om2y56.b0.cloudera.site:443/devo-lab21-dl/cdp-proxy-api/ranger" + BUCKET_PREFIX: "s3a://devo-crp-ffppyd8q/" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya3tsglrzfgiyfisxchref774l5y4nrler2vn54lr3li7q" + S3_LOCATION_URI: "https://devo-crp-ffppyd8q.bucket.vpce-040b28f5818b670c1-owicl3ow.s3.eu-central-1.vpce.amazonaws.com/{0}/db/" + DEVO_USERNAME_RQSD: "ap-devo-rqsd-lab" + DEVO_SECRET_RQSD: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyap6wtzobzob7qizvk4nocszlcaxhwijgzejbvryt3uzbq" + FULL_ACCESS_LIST_RAR: "DISC-DC-RAR-R" + FULL_ACCESS_LIST_MOPDB: "" + FULL_ACCESS_LIST_RQSD: "" +tst: + DEVO_USERNAME: "ap-devo_tst-mrds" + IMPALA_HOSTNAME: "t-impala.devo.escb.eu" + HIVE_HOSTNAME: "hs2-devo-tst21-hive01.dw-devo-tst21.om2y56.b0.cloudera.site" + RANGER_HOSTNAME: "https://devo-tst21-dl-gateway.devo-tst.om2y56.b0.cloudera.site:443/devo-tst21-dl/cdp-proxy-api/ranger" + BUCKET_PREFIX: "s3a://devo-crp-sbul3ju3/" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyayqqotyowhpoml3v5szkwhmtu4rq6bplpkvdruzupz3ma" + S3_LOCATION_URI: "https://devo-crp-sbul3ju3.bucket.vpce-040b28f5818b670c1-owicl3ow.s3.eu-central-1.vpce.amazonaws.com/{0}/db/" + DEVO_USERNAME_RQSD: "ap-devo-rqsd-tst" + DEVO_SECRET_RQSD: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyap6wtzobzob7qizvk4nocszlcaxhwijgzejbvryt3uzbq" + FULL_ACCESS_LIST_RAR: "DISC-TC-RAR-R" + FULL_ACCESS_LIST_MOPDB: "" + FULL_ACCESS_LIST_RQSD: "" +acc: + DEVO_USERNAME: "ap-devo_acc-mrds" + IMPALA_HOSTNAME: "impala-proxy-devo-acc21-impala01.dw-devo-acc21.inym23.b0.cloudera.site" + HIVE_HOSTNAME: "hs2-devo-acc21-hive01.dw-devo-acc21.inym23.b0.cloudera.site" + RANGER_HOSTNAME: "https://devo-acc21-dl-gateway.devo-acc.inym23.b0.cloudera.site/devo-acc21-dl/cdp-proxy-api/ranger/" + BUCKET_PREFIX: "s3a://devo-crp-sbc9vbsu/" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya3x3nic3vxsnpzlfshz2ubj6kekny5tvaqsnwkuh2hw2a" + S3_LOCATION_URI: "https://devo-crp-sbc9vbsu.bucket.vpce-0bf4fa440fb60935d-6m9iqoo9.s3.eu-central-1.vpce.amazonaws.com/{0}/db/" + DEVO_USERNAME_RQSD: "ap-devo-rqsd-acc" + DEVO_SECRET_RQSD: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyamzhgatnso57mubvg3c6k4ens3orcx4dieo6efukuvm4a" + FULL_ACCESS_LIST_RAR: "DISC-AC-RAR-R" + FULL_ACCESS_LIST_MOPDB: "" + FULL_ACCESS_LIST_RQSD: "" +prd: + DEVO_USERNAME: "ap-devo_prd-mrds" + IMPALA_HOSTNAME: "impala-proxy-devo-prd21-impala01.dw-devo-prd21.inym23.b0.cloudera.site" + HIVE_HOSTNAME: "hs2-devo-prd21-hive01.dw-devo-prd21.inym23.b0.cloudera.site" + RANGER_HOSTNAME: "https://devo-prd21-dl-gateway.devo-prd.inym23.b0.cloudera.site/devo-prd21-dl/cdp-proxy-api/ranger/" + BUCKET_PREFIX: "s3a://devo-crp-2gn5maj9/" + DEVO_SECRET: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyace73o3xowa3f3jkw4diqzoiyc6skt34sqnnx4yrbykmq" + S3_LOCATION_URI: "https://devo-crp-2gn5maj9.bucket.vpce-0aa6cf4490536dfd5-qgy4w5sz.s3.eu-central-1.vpce.amazonaws.com/{0}/db/" + DEVO_USERNAME_RQSD: "ap-devo-rqsd-prd" + DEVO_SECRET_RQSD: "ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjyacodc43tfgumkw4qyzw4s3j4jp42vp2elakkpwwrmivqa" + FULL_ACCESS_LIST_RAR: "DISC-PC-RAR-R" + FULL_ACCESS_LIST_MOPDB: "" + FULL_ACCESS_LIST_RQSD: "" + + +rar: + corporate_store: "crp_rar" + oracle_metadata_table: "CORR_RAR.NH_METADATA_INVENTORY" + oracle_igam_table: "CT_MRDS.A_DEVO_SOURCES_IGAM" + oracle_mgmt_table: "CT_MRDS.A_DEVO_REPLICA_MGMT_RAR" + target_s3_bucket: "rar/db" + tech_meta_data_fields: "tec_ingestion_date String, tec_execution_date String, tec_run_id String" +mopdb: + corporate_store: "crp_mopdb" + oracle_metadata_table: "CT_MOPDB.MOPDB_METADATA_INVENTORY" + oracle_igam_table: "CT_MRDS.A_DEVO_SOURCES_IGAM" + oracle_mgmt_table: "CT_MRDS.A_DEVO_REPLICA_MGMT_MOPDB" + target_s3_bucket: "mopdb/db" + tech_meta_data_fields: "tec_ingestion_date String, tec_execution_date String, tec_run_id String" +rqsd: + corporate_store: "crp_rqsd" + oracle_metadata_table: "CT_MRDS.A_DEVO_METADATA_INVENTORY" + oracle_igam_table: "CT_MRDS.A_DEVO_SOURCES_IGAM" + oracle_mgmt_table: "CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD" + target_s3_bucket: "rqsd/db" + tech_meta_data_fields: "tec_ingestion_date String, tec_execution_date String, tec_run_id String" + +# -- target table name as +# SELECT DISTINCT TABLE_ALIAS FROM {oracle_mgmt_table} +# WHERE OWNER = '' +# AND TABLE_NAME = ''; + +# -- type of access +# SELECT DISTINCT RAR3_TYPE_OF_ACCESS FROM {oracle_metadata_table} +# WHERE A_VALID_TO > SYSDATE AND +# OWNER = '' +# AND TABLE_NAME = ''; \ No newline at end of file diff --git a/python/devo_replicator/data_replicator/.gitkeep b/python/devo_replicator/data_replicator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/devo_replicator/data_replicator/diag_s3_access.py b/python/devo_replicator/data_replicator/diag_s3_access.py new file mode 100644 index 0000000..cbbace5 --- /dev/null +++ b/python/devo_replicator/data_replicator/diag_s3_access.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import sys, json +import boto3 +from botocore.exceptions import ClientError +from botocore.config import Config + +BUCKET = "devo-crp-sbc9vbsu" +PREFIX = "mopdb/db/" # adjust if needed + +def show(e): + # Print the structured error if present + resp = getattr(e, "response", {}) + code = resp.get("Error", {}).get("Code") + msg = resp.get("Error", {}).get("Message") + rid = resp.get("ResponseMetadata", {}).get("RequestId") + print(f"{type(e).__name__}: {code} {msg} (RequestId={rid})", file=sys.stderr) + +def main(endpoint_url=None, region=None, force_path=False): + session = boto3.Session() + cfg = Config(s3={"addressing_style": "path" if force_path else "auto"}) + s3 = session.client("s3", region_name=region, endpoint_url=endpoint_url, config=cfg) + sts = session.client("sts", region_name=region) + + # Who am I? + try: + ident = sts.get_caller_identity() + print(f"Caller: {ident['Arn']} (acct {ident['Account']})") + except Exception as e: + print("Could not call STS get-caller-identity — credentials not valid for STS.", file=sys.stderr) + show(e); return 1 + + # Is the bucket reachable at all? + try: + s3.head_bucket(Bucket=BUCKET) + print(f"head_bucket OK on s3://{BUCKET}") + except ClientError as e: + print("head_bucket failed:", file=sys.stderr) + show(e); return 2 + + # List with zero keys to test just the ListBucket permission + try: + s3.list_objects_v2(Bucket=BUCKET, Prefix=PREFIX, MaxKeys=0) + print(f"list_objects_v2 OK on prefix '{PREFIX}' (permission exists)") + except ClientError as e: + print("list_objects_v2 failed:", file=sys.stderr) + show(e); return 3 + + # Ask for 1 key to confirm data path works + try: + resp = s3.list_objects_v2(Bucket=BUCKET, Prefix=PREFIX, MaxKeys=1) + print("First key:", resp.get("Contents", [{}])[0].get("Key")) + except ClientError as e: + print("list_objects_v2 (MaxKeys=1) failed:", file=sys.stderr) + show(e); return 4 + + return 0 + +if __name__ == "__main__": + # Allow optional args: --endpoint-url URL --region eu-central-1 --force-path + url = None; reg = None; force = False + for i,a in enumerate(sys.argv): + if a == "--endpoint-url": url = sys.argv[i+1] + if a == "--region": reg = sys.argv[i+1] + if a == "--force-path": force = True + sys.exit(main(endpoint_url=url, region=reg, force_path=force)) diff --git a/python/devo_replicator/data_replicator/impala_refresher.py b/python/devo_replicator/data_replicator/impala_refresher.py new file mode 100644 index 0000000..7e9a3b6 --- /dev/null +++ b/python/devo_replicator/data_replicator/impala_refresher.py @@ -0,0 +1,129 @@ +import os +import yaml +import datetime +import pandas as pd +from mrds.utils.secrets import get_secret +import mrds.utils.manage_runs as runManager +import mrds.utils.manage_files as fileManager +import mrds.utils.sql_statements as sqls + +import oci + +from impala.dbapi import ( + connect, + ProgrammingError, + DatabaseError, + IntegrityError, + OperationalError, +) +from impala.error import HiveServer2Error + + +def get_impala_connection(hostname: str, user: str, secret: str): + conn = connect( + host=hostname, + port=443, + auth_mechanism="PLAIN", + user=user, + password=secret, + use_http_transport=True, + http_path="cliservice", + use_ssl=True + ) + return conn + +def execute_query(query: str,user,hostname,password): + conn = get_impala_connection(hostname, user, password) + print(conn) + columns, result = execute_devo_query(query, conn) + return columns, result + +def execute_devo_query(query: str, conn): + #impersonation_configuration = {"impala.doas.user": userid} # to be changed + #impersonation_configuration = {} # to be changed + cursor = conn.cursor() + print("executing query") + try: + cursor.execute(query) + # Check if the query is a SELECT query (i.e., reads data) + return None, cursor.rowcount # rowcount returns the number of rows affected + except OperationalError as oe: + raise Exception( + status_code=500, detail="Failed to connect to Impala: " + str(oe) + ) + + except ProgrammingError as pe: + raise Exception(status_code=400, detail="Query syntax error: " + str(pe)) + + except IntegrityError as ie: + raise Exception( + status_code=403, detail="Insufficient permissions: " + str(ie) + ) + + except DatabaseError as db_err: + raise Exception(status_code=500, detail="Database error: " + str(db_err)) + + except HiveServer2Error as au_err: + raise Exception( + status_code=403, detail="HiveServer2Error error: " + str(au_err) + ) + + except Exception as e: + raise Exception( + status_code=500, detail="An unexpected error occurred: " + str(e) + ) from e + + finally: + try: + if cursor: + cursor.close() + if conn: + conn.close() + except Exception as e: + raise Exception( + status_code=500, detail="Failed to close the connection: " + str(e) + ) + +def initialize_task(workflow_context, task_name): + # Initialize task + a_task_history_key = runManager.init_task( + task_name, + workflow_context["run_id"], + workflow_context["a_workflow_history_key"], + ) + + return a_task_history_key + +def initialize_config(config_file_path): + # Ensure the file exists + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + + # Load the configuration + with open(config_file_path, "r") as f: + config_data = yaml.safe_load(f) + + return config_data + +def main(env_config_path, env, table, corporate_store): + + #init setup + envs_info = initialize_config(env_config_path) + environment_info = envs_info[env] + + try: + devo_secret_name = environment_info["DEVO_SECRET"] + password = get_secret(devo_secret_name) + except: + print("Failed to retrieve credentials from secrets") + raise(Exception) + # get devo data + try: + execute_query(f"INVALIDATE METADATA {corporate_store}.{table}", environment_info['DEVO_USERNAME'], environment_info['IMPALA_HOSTNAME'], password) + execute_query(f"COMPUTE STATS {corporate_store}.{table}", environment_info['DEVO_USERNAME'], environment_info['IMPALA_HOSTNAME'], password) + + except: + print("Failed to retrieve DEVO data, error during connection or request") + raise(Exception) + return True + diff --git a/python/devo_replicator/data_replicator/list_s3_files_fast.py b/python/devo_replicator/data_replicator/list_s3_files_fast.py new file mode 100644 index 0000000..e93635a --- /dev/null +++ b/python/devo_replicator/data_replicator/list_s3_files_fast.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse, sys +from urllib.parse import urlparse +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError, ReadTimeoutError, ConnectTimeoutError + +def parse_s3_uri(s3_uri: str): + if not s3_uri.startswith("s3://"): + raise ValueError("S3 URI must start with 's3://'") + p = urlparse(s3_uri) + if not p.netloc: + raise ValueError("Missing bucket in S3 URI") + return p.netloc, p.path.lstrip("/") + +def parse_location(location: str): + """Accept s3://... OR https://...amazonaws.com/... and return (bucket, prefix).""" + if location.startswith("s3://"): + return parse_s3_uri(location) + + if location.startswith(("http://", "https://")): + p = urlparse(location) + host = p.netloc + path = p.path.lstrip("/") + # Bucket-scoped VPCe host: .bucket.vpce-xxxx.s3..vpce.amazonaws.com + if ".bucket." in host: + bucket = host.split(".bucket.", 1)[0] + return bucket, path + # Virtual-hosted: .s3.... + if ".s3." in host and not host.startswith("s3."): + bucket = host.split(".s3.", 1)[0] + return bucket, path + # Path-style: s3....//... + if host.startswith("s3."): + parts = path.split("/", 1) + bucket = parts[0] + prefix = parts[1] if len(parts) > 1 else "" + return bucket, prefix + + raise ValueError(f"Unsupported location: {location}") + +def iter_keys(s3, bucket: str, prefix: str, page_size: int, max_items: int, verbose: bool): + print('here') + paginator = s3.get_paginator("list_objects_v2") + kwargs = {"Bucket": bucket, "Prefix": prefix} + pagination = {"PageSize": page_size} + if max_items > 0: + pagination["MaxItems"] = max_items + + total = 0 + page_num = 0 + for page in paginator.paginate(**kwargs, PaginationConfig=pagination): + page_num += 1 + contents = page.get("Contents", []) or [] + if verbose: + print(f"[page {page_num}] fetched {len(contents)} keys (running total={total + len(contents)})", + file=sys.stderr, flush=True) + for obj in contents: + key = obj["Key"] + yield key + total += 1 + +def main(): + ap = argparse.ArgumentParser(description="List files under an S3 location quickly and safely.") + ap.add_argument("location", help="s3://bucket/prefix/ OR https:///") + ap.add_argument("--region", default=None, help="AWS region (e.g., eu-central-1)") + ap.add_argument("--profile", default=None, help="AWS profile to use") + ap.add_argument("--endpoint-url", default=None, + help="Custom S3 endpoint (e.g., https://s3.eu-central-1.vpce.amazonaws.com)") + ap.add_argument("--force-path-addressing", action="store_true", + help="Force path-style addressing (useful with bucket-scoped VPCe hostnames)") + ap.add_argument("--page-size", type=int, default=1000, help="S3 page size (default 1000)") + ap.add_argument("--max-items", type=int, default=0, help="Stop after N keys (0 = no limit)") + ap.add_argument("--connect-timeout", type=float, default=10.0, help="Seconds (default 10)") + ap.add_argument("--read-timeout", type=float, default=30.0, help="Seconds (default 30)") + ap.add_argument("--retries", type=int, default=3, help="Max retry attempts (default 3)") + ap.add_argument("--relative", action="store_true", help="Print keys relative to the prefix") + ap.add_argument("--verbose", "-v", action="store_true", help="Print progress to stderr") + args = ap.parse_args() + + bucket, prefix = parse_location(args.location) + + # Session & client with explicit timeouts and optional path addressing + sess_kwargs = {} + if args.profile: + sess_kwargs["profile_name"] = args.profile + session = boto3.Session(**sess_kwargs) + + cfg = Config( + connect_timeout=args.connect_timeout, + read_timeout=args.read_timeout, + retries={"max_attempts": args.retries, "mode": "standard"}, + s3={"addressing_style": "path" if args.force_path_addressing else "auto"}, + ) + + s3 = session.client("s3", region_name=args.region, endpoint_url=args.endpoint_url, config=cfg) + + # Quick preflight: try a 0-key list to surface auth/endpoint issues fast + try: + _ = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=0) + except ClientError as e: + print(f"Preflight failed (auth/permissions/endpoint): {e}", file=sys.stderr) + sys.exit(1) + except (EndpointConnectionError, ReadTimeoutError, ConnectTimeoutError) as e: + print(f"Network/endpoint error: {e}", file=sys.stderr) + sys.exit(1) + + try: + for key in iter_keys(s3, bucket, prefix, args.page_size, args.max_items, args.verbose): + if args.relative and prefix and key.startswith(prefix): + print(key[len(prefix):].lstrip("/")) + else: + print(f"s3://{bucket}/{key}") + except KeyboardInterrupt: + print("\nInterrupted.", file=sys.stderr) + sys.exit(130) + except NoCredentialsError: + print("No AWS credentials found. Set env vars or use --profile.", file=sys.stderr) + sys.exit(1) + except (EndpointConnectionError, ReadTimeoutError, ConnectTimeoutError) as e: + print(f"Network/timeout listing objects: {e}", file=sys.stderr) + sys.exit(1) + except ClientError as e: + print(f"AWS error: {e}", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/python/devo_replicator/table_generator/.gitkeep b/python/devo_replicator/table_generator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/devo_replicator/table_generator/FlowOptions.py b/python/devo_replicator/table_generator/FlowOptions.py new file mode 100644 index 0000000..2994e6f --- /dev/null +++ b/python/devo_replicator/table_generator/FlowOptions.py @@ -0,0 +1,145 @@ +class Options: + def __init__(self, args): + self.options = { + "corporate_store": None, + "service_name": None, + "source_schema": None, + "source_table": None, + "access_type": None, + "oracle_metadata_table": None, + "oracle_igam_table": None, + "query_metadata_access_type1": None, + "query_metadata_access_type2a": None, + "query_igam_roles": None, + "ora_jdbc_url_dwh": None, + "ora_jdbc_url_ods": None, + "sql_file_path": None, + "sql_filename_grants": None, + "sentry_role_environment": None, + "ranger_script": None, + "type3_access_table": None, + "type3_access_table_key_column": None, + "type3_source_table_key_column": None, + "target_s3_bucket": None, + "ranger_s3_bucket": None, + "ranger_s3_path": None, + "rar_full_access_entitlement_list": None, + "target_table": None, + "tech_meta_data_fields": None, + "full_access_entitlement_list": None + } + # Initialize options from arguments + self.initialize_options(args) + + def initialize_options(self, args): + # Assuming args is a list of key-value pairs + for key in args.keys(): + if key in self.options: + self.options[key] = args[key] + + def get_option_value(self, key): + return self.options.get(key, "") + + @property + def corporate_store(self): + return self.get_option_value("corporate_store") + + @property + def source_schema(self): + return self.get_option_value("source_schema") + + @property + def source_table(self): + return self.get_option_value("source_table") + + @property + def access_type(self): + return self.get_option_value("access_type") + + @property + def oracle_metadata_table(self): + return self.get_option_value("oracle_metadata_table") + + @property + def oracle_igam_table(self): + return self.get_option_value("oracle_igam_table") + + @property + def query_metadata_access_type1(self): + return self.get_option_value("query_metadata_access_type1") + + @property + def query_metadata_access_type2a(self): + return self.get_option_value("query_metadata_access_type2a") + + @property + def query_igam_roles(self): + return self.get_option_value("query_igam_roles") + + @property + def ora_jdbc_url_dwh(self): + return self.get_option_value("ora_jdbc_url_dwh") + + @property + def ora_jdbc_url_ods(self): + return self.get_option_value("ora_jdbc_url_ods") + + @property + def sql_file_path(self): + return self.get_option_value("sql_file_path") + + @property + def sql_filename_grants(self): + return self.get_option_value("sql_filename_grants") + + @property + def sentry_role_environment(self): + return self.get_option_value("sentry_role_environment") + + @property + def ranger_script(self): + return self.get_option_value("ranger_script") + + @property + def type3_access_table(self): + return self.get_option_value("type3_access_table") + + @property + def type3_access_table_key_column(self): + return self.get_option_value("type3_access_table_key_column") + + @property + def type3_source_table_key_column(self): + return self.get_option_value("type3_source_table_key_column") + + @property + def target_s3_bucket(self): + return self.get_option_value("target_s3_bucket") + + @property + def ranger_s3_bucket(self): + return self.get_option_value("ranger_s3_bucket") + + @property + def ranger_s3_path(self): + return self.get_option_value("ranger_s3_path") + + @property + def rar_full_access_entitlement_list(self): + return self.get_option_value("rar_full_access_entitlement_list") + + @property + def target_table(self): + return self.get_option_value("target_table") + + @property + def tech_meta_data_fields(self): + return self.get_option_value("tech_meta_data_fields") + + @property + def full_access_entitlement_list(self): + return self.get_option_value("full_access_entitlement_list") + + @property + def service_name(self): + return self.get_option_value("service_name") diff --git a/python/devo_replicator/table_generator/devo_query.py b/python/devo_replicator/table_generator/devo_query.py new file mode 100644 index 0000000..4bbe040 --- /dev/null +++ b/python/devo_replicator/table_generator/devo_query.py @@ -0,0 +1,73 @@ +from impala.dbapi import ( + connect, + ProgrammingError, + DatabaseError, + IntegrityError, + OperationalError, +) +from impala.error import HiveServer2Error + + +def get_DEVO_connection(hostname: str, user: str, secret: str): + conn = connect( + host=hostname, + port=443, + auth_mechanism="PLAIN", + user=user, + password=secret, + use_http_transport=True, + http_path="cliservice", + use_ssl=True, + ) + return conn + + +def execute_devo_query(query: str, conn): + cursor = None + try: + cursor = conn.cursor() + cursor.execute(query) # Check if the query is a SELECT query (i.e., reads data) + if query.strip().lower().startswith("select"): + rows = cursor.fetchall() + columns = [col[0] for col in cursor.description] + return columns, rows + else: + # For non-SELECT queries (e.g., INSERT, UPDATE, DELETE), just return affected rows + return None, cursor.rowcount # rowcount returns the number of rows affected + + except OperationalError as oe: + raise Exception("Failed to connect to DEVO: " + str(oe)) + + except ProgrammingError as pe: + raise Exception("Query syntax error: " + str(pe)) + + except IntegrityError as ie: + raise Exception("Insufficient permissions: " + str(ie)) + + except DatabaseError as db_err: + raise Exception("Database error: " + str(db_err)) + + except HiveServer2Error as au_err: + raise Exception("HiveServer2Error error: " + str(au_err)) + + finally: + try: + if cursor: + cursor.close() + if not conn: + conn.close() + except Exception as e: + raise Exception(status_code=500, detail=f"Failed to close the cursor or impala connection: {str(e)}") from e + +def execute_query(query: str, user: str, hostname: str,password): + conn = get_DEVO_connection(hostname, user, password) + columns, result = execute_devo_query(query, conn) + return columns, result + + +#sql="CREATE EXTERNAL TABLE IF NOT EXISTS crp_rar.testInternalTable ( iid STRING,RANDOM_DATE DATE, number int) ;" +#sql_drop="DROP TABLE IF EXISTS crp_rar.NH_PRICE" + +#print( execute_query("SELECT 1","ap-informatica-ipcwt","t-impala.devo.escb.eu","Start_123456789")) +#print( execute_query("SELECT 1","ap-devo_tst-mrds","t-impala.devo.escb.eu","V1XqZ*#fvwQl=nRG*idI")) +#print( execute_query("SELECT 1","ap-devo_lab-mrds","impala-proxy-devo-lab21-impala01.dw-devo-lab21.om2y56.b0.cloudera.site","PHkvyVonyePAmZD8wUuw!")) diff --git a/python/devo_replicator/table_generator/ranger_test.py b/python/devo_replicator/table_generator/ranger_test.py new file mode 100644 index 0000000..bf0a008 --- /dev/null +++ b/python/devo_replicator/table_generator/ranger_test.py @@ -0,0 +1,69 @@ +## Step 3: Let's create a policy + +from apache_ranger.model.ranger_service import * +from apache_ranger.client.ranger_client import * +from apache_ranger.model.ranger_policy import * +from mrds.utils.secrets import get_secret + +## Step 1: create a client to connect to Apache Ranger admin +ranger_url ="https://devo-lab21-dl-gateway.devo-lab.om2y56.b0.cloudera.site:443/devo-lab21-dl/cdp-proxy-api/ranger" +password= get_secret("ocid1.vaultsecret.oc1.eu-frankfurt-1.amaaaaaa2ky4jjya3tsglrzfgiyfisxchref774l5y4nrler2vn54lr3li7q") +ranger_auth = ('ap-devo_lab-mrds', password) + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + +ranger = RangerClient(ranger_url, ranger_auth) +ranger.session.verify = False + +# to disable SSL certificate validation (not recommended for production use!) +# +# ranger.session.verify = False + + +## Step 2: Let's create a service + +policy = RangerPolicy() +policy.service = "cm_hive" #da hardcodare +policy.name = 'cpo_crp_mopdb_sgroi_1' #corporatestore_table_accessType +policy.resources = { 'database': RangerPolicyResource({ 'values': ['crp_RQSD'] }), + 'table': RangerPolicyResource({ 'values': ['ANNEX_1_1_ALL'] }), + 'column': RangerPolicyResource({ 'values': ['*'] }) } #change with correct values +allowItem1 = RangerPolicyItem() #to try allowItem1.groups +allowItem1.groups = ["d_mopdb_mpec"] +#allowItem1.users = [] #to try for single users +allowItem1.accesses = [ RangerPolicyItemAccess({ 'type': 'create' }), + RangerPolicyItemAccess({ 'type': 'alter' }), + RangerPolicyItemAccess({ 'type': 'select' }), + RangerPolicyItemAccess({ 'type': 'drop' }) ] + +"""denyItem1 = RangerPolicyItem() +denyItem1.users = [ 'admin' ] #does it make sense to deny and not allow? +denyItem1.accesses = [ RangerPolicyItemAccess({ 'type': 'drop' }) ]""" + +policy.policyItems = [ allowItem1 ] +#policy.denyPolicyItems = [ denyItem1 ] +#policy2=ranger.get_policy_by_id(policyId=5086) +#print(ranger.get_policy(serviceName="cm_hive",policyName='crp_rar_testinternalTable_alcesso1')) +#print(ranger.find_policies({"service": "cm_hive", "resources": {"database": {"values": ["crp_rar"], "isExcludes": False , "isRecursive": False}, "column": {"values": ["*"], "isExcludes": False, "isRecursive": False}, "table": {"values": ["testInternalTable"], "isExcludes": False, "isRecursive": False}}})) +#print(ranger.delete_policy(serviceName="cm_hive",policyName="crp_rar_testinternalTable_alcesso1")) +#print(policy2) +#print('Creating policy: name=' + policy.name) +#created_policy = ranger.create_policy(policy) + +#print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + +## Step 4: Delete policy and service created above +#print('Deleting policy: id=' + str(created_policy.id)) + +#ranger.delete_policy_by_id(created_policy.id) + +data=ranger.get_policies_in_service(serviceName="cm_hive") +with open("output.txt", "w") as file: + for string in data: + file.write(str(string)) + file.close() + diff --git a/python/devo_replicator/table_generator/ranger_updater.py b/python/devo_replicator/table_generator/ranger_updater.py new file mode 100644 index 0000000..dbae9a2 --- /dev/null +++ b/python/devo_replicator/table_generator/ranger_updater.py @@ -0,0 +1,250 @@ +from typing import List, Optional +from apache_ranger.model.ranger_service import * +from apache_ranger.client.ranger_client import * +from apache_ranger.model.ranger_policy import * +import re +def add_table_permission_groups(corporate_store: str, target_table: str, access_type: str, source_table: str, igam_entitlement_list: List[str], columns_list: Optional[List[str]] = None, row_list: Optional[List[str]] = None): + igam_entitlements = igam_entitlement_list + ["public"] if source_table.lower() == "rar_sources_igam_sentry" else igam_entitlement_list + column_details = columns_list if columns_list is not None else ["*"] + columns = column_details + row_filter = row_list if row_list is not None else ["*"] + filter_condition = ','.join([f"'{row}'" for row in row_filter]) + igam_roles = [x.lower() for x in igam_entitlements if x !=""] + + return { + 'corporate_store': corporate_store, + 'target_table': target_table, + 'access_type': access_type, + 'columns': columns, + 'rows': filter_condition, + 'igam_roles': igam_roles + } + +from typing import List, Optional +# --- helpers --------------------------------------------------------------- + +def _policy_name_from_params(config, policy_id: Optional[str] = None) -> Optional[str]: + """ + Build the exact policy name used by your create functions. + Returns None for types where we need to match multiple (e.g., 2a without id). + """ + cs = config['corporate_store'].lower() + tbl = config['target_table'].lower() + at = config['access_type'].lower() + base = f"cpo_{cs}_{tbl}_{at}" + + if at == "1": + # yaml_format_1 + return base + elif at == "2a": + # yaml_format_2a -> requires policy_id to be exact + if policy_id: + return f"{base}_policy_{policy_id}" + # without policy_id, we’ll delete all that start with this prefix + return None + elif at == "2b": + # yaml_format_2b + return f"{base}_row_level_policy" + elif at == "3": + # yaml_format_3 uses same name pattern as 2b in your script + return f"{base}_row_level_policy" + else: + raise ValueError(f"Invalid access type '{config['access_type']}'. Expected one of: 1, 2a, 2b, 3.") + + +def _ranger_client(env_config) -> RangerClient: + ranger_url = env_config['RANGER_HOSTNAME'] + ranger_auth = ( env_config['DEVO_USERNAME'], env_config['DEVO_SECRET']) + client = RangerClient(ranger_url, ranger_auth) + client.session.verify = False + return client + + +# --- main deletion API ----------------------------------------------------- + +def delete_policy(config,env_config, policy_id: Optional[str] = None) -> List[str]: + """ + Delete Ranger policy/policies by name based on: + - params['corporate_store'] + - params['target_table'] + - typeOfAccess: "1", "2a", "2b", "3" + - policy_id: optional (only meaningful for '2a') + + Returns a list of deleted policy names. + """ + ranger = _ranger_client(env_config) + service_name = "cm_hive" + + # Try build exact name + deleted: List[str] = [] + + + # If we don’t have an exact name (e.g. type 2a without policy_id), + # delete *all* that match the expected prefix. + cs = config['corporate_store'].lower() + tbl = config['target_table'].lower() + at = config['access_type'].lower() + prefix = f"cpo_{cs}_{tbl}_" + print(prefix) + # Fetch all policies for the table and filter client-side to reduce calls. + start = 0 + candidates = [] + page_size=1000 + service_name="cm_hive" + while True: + params = {"pageSize": page_size, "startIndex": start} + page = ranger.get_policies_in_service(service_name, params=params) or [] + candidates.extend(page) + if len(page) < page_size: + break + start += len(page) + for p in candidates: + name = p["name"] + print(f"analizing policy:{name}") + if re.fullmatch(f"{prefix}([0-9]?[a-z]?)(_policy_)?([0-9]*)?(_row_level_policy)?(full_access)?$",name) != None: + try: + ranger.delete_policy_by_id(p["id"]) + deleted.append(name) + except Exception: + # continue attempting others + pass + if not deleted: + raise RuntimeError( + f"No matching policies found for deletion with prefix '{prefix}'. " + ) + return deleted + + + +def generate_policy(params,env_config, policy_id: Optional[str] = None): + access_type = params['access_type'].lower() + if access_type == "1": + return yaml_format_1(params,env_config) + elif access_type == "2a": + return yaml_format_2a(params, env_config, policy_id) + elif access_type == "2b": + return yaml_format_1(params,env_config) + elif access_type == "3": + return yaml_format_3(params) + else: + raise Exception(f"Invalid access type {params['access_type']}. Please check the input param") + +def yaml_format_1(params,env_config) -> str: + ranger=_ranger_client(env_config) + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}" #corporatestore_table_accessType + policy.resources = { 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] }), + 'column': RangerPolicyResource({ 'values': params['columns'] }) } + + allowItem1 = RangerPolicyItem() + allowItem1.groups = params['igam_roles'] + allowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + policy.policyItems = [ allowItem1 ] + created_policy = ranger.create_policy(policy) + print('Created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + return policy + +def yaml_format_2a(params, env_config,policy_id: Optional[str]) -> str: + policy_ID = policy_id if policy_id is not None else "0" + + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_policy_{policy_ID}" #corporatestore_table_accessType + policy.resources = { 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] }), + 'column': RangerPolicyResource({ 'values': params['columns'] }) } + allowItem1 = RangerPolicyItem() + allowItem1.groups = params['igam_roles'] + allowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + policy.policyItems = [ allowItem1 ] + created_policy = ranger.create_policy(policy) + print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + return policy + +def yaml_format_2b(params,env_config, full_access_list: Optional[List]) -> str: + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_row_level_policy" #corporatestore_table_accessType + policy.isEnabled = True + policy.resources ={ 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] })} + rowFilterAllowItem1= RangerRowFilterPolicyItem() + rowFilterAllowItem1.groups = params['igam_roles'] + rowFilterAllowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem1.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"lower(source) IN (select lower(rar_subsource_id) from {params['corporate_store'].lower()}.t_ref_rar_sources_igam_sentry where lower(rar_igam_entitlement) IN (select ad_group from {params['corporate_store'].lower()}.active_directory_user_groups where username = lower(regexp_extract(current_user(),'[^@]*',0))))" }) + rowFilterAllowItem2= RangerRowFilterPolicyItem() + rowFilterAllowItem2.groups = [x.lower() for x in full_access_list] + rowFilterAllowItem2.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem2.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"1=1" }) + policy.rowFilterPolicyItems= [rowFilterAllowItem1, rowFilterAllowItem2] + + created_policy = ranger.create_policy(policy) + print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + return policy + + +def yaml_format_3(params, env_config,filterString, full_access_list: Optional[List]) -> str: + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" # hardcoded + policy.name = ( + f"cpo_{params['corporate_store'].lower()}_" + f"{params['target_table'].lower()}_" + f"{params['access_type'].lower()}_row_level_policy" + ) + policy.isEnabled = True + policy.resources = { + "database": RangerPolicyResource({"values": [params["corporate_store"].lower()]}), + "table": RangerPolicyResource({"values": [params["target_table"]]}), + } + + # Row filter item + rowFilterAllowItem = RangerRowFilterPolicyItem() + rowFilterAllowItem.groups = params["igam_roles"] + rowFilterAllowItem.accesses = [RangerPolicyItemAccess({"type": "select"})] + rowFilterAllowItem.rowFilterInfo = RangerPolicyItemRowFilterInfo( + { + "filterExpr": filterString + } + ) + rowFilterAllowItem2= RangerRowFilterPolicyItem() + rowFilterAllowItem2.groups = [x.lower() for x in full_access_list] + rowFilterAllowItem2.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem2.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"1=1" }) + + policy.rowFilterPolicyItems = [rowFilterAllowItem,rowFilterAllowItem2] + + # Create policy in Ranger + created_policy = ranger.create_policy(policy) + print(f" created policy: name={created_policy.name}, id={created_policy.id}") + + return policy diff --git a/python/devo_replicator/table_generator/ranger_updater_old.py b/python/devo_replicator/table_generator/ranger_updater_old.py new file mode 100644 index 0000000..6b32444 --- /dev/null +++ b/python/devo_replicator/table_generator/ranger_updater_old.py @@ -0,0 +1,345 @@ +from typing import List, Optional +from apache_ranger.model.ranger_service import * +from apache_ranger.client.ranger_client import * +from apache_ranger.model.ranger_policy import * +import re +def add_table_permission_groups(corporate_store: str, target_table: str, access_type: str, source_table: str, igam_entitlement_list: List[str], columns_list: Optional[List[str]] = None, row_list: Optional[List[str]] = None): + igam_entitlements = igam_entitlement_list + ["public"] if source_table.lower() == "rar_sources_igam_sentry" else igam_entitlement_list + column_details = columns_list if columns_list is not None else ["*"] + columns = column_details + row_filter = row_list if row_list is not None else ["*"] + filter_condition = ','.join([f"'{row}'" for row in row_filter]) + igam_roles = [x.lower() for x in igam_entitlements if x !=""] + + return { + 'corporate_store': corporate_store, + 'target_table': target_table, + 'access_type': access_type, + 'columns': columns, + 'rows': filter_condition, + 'igam_roles': igam_roles + } + +from typing import List, Optional +# --- helpers --------------------------------------------------------------- + +def _policy_name_from_params(config, policy_id: Optional[str] = None) -> Optional[str]: + """ + Build the exact policy name used by your create functions. + Returns None for types where we need to match multiple (e.g., 2a without id). + """ + cs = config.corporate_store.lower() + tbl = config.target_table.lower() + at = config.access_type.lower() + base = f"cpo_{cs}_{tbl}_{at}" + + if at == "1": + # yaml_format_1 + return base + elif at == "2a": + # yaml_format_2a -> requires policy_id to be exact + if policy_id: + return f"{base}_policy_{policy_id}" + # without policy_id, we’ll delete all that start with this prefix + return None + elif at == "2b": + # yaml_format_2b + return f"{base}_row_level_policy" + elif at == "3": + # yaml_format_3 uses same name pattern as 2b in your script + return f"{base}_row_level_policy" + else: + raise ValueError(f"Invalid access type '{config.access_type}'. Expected one of: 1, 2a, 2b, 3.") + + +def _ranger_client(env_config) -> RangerClient: + ranger_url = env_config['RANGER_HOSTNAME'] + ranger_auth = ( env_config['DEVO_USERNAME'], env_config['DEVO_SECRET']) + client = RangerClient(ranger_url, ranger_auth) + client.session.verify = False + return client + + +# --- main deletion API ----------------------------------------------------- + +def delete_policy(config,env_config, policy_id: Optional[str] = None) -> List[str]: + """ + Delete Ranger policy/policies by name based on: + - params['corporate_store'] + - params['target_table'] + - typeOfAccess: "1", "2a", "2b", "3" + - policy_id: optional (only meaningful for '2a') + + Returns a list of deleted policy names. + """ + ranger = _ranger_client(env_config) + service_name = "cm_hive" + + # Try build exact name + deleted: List[str] = [] + + + # If we don’t have an exact name (e.g. type 2a without policy_id), + # delete *all* that match the expected prefix. + cs = config.corporate_store.lower() + tbl = config.target_table.lower() + at = config.access_type.lower() + prefix = f"cpo_{cs}_{tbl}_" + # Fetch all policies for the table and filter client-side to reduce calls. + start = 0 + candidates = [] + page_size=1000 + service_name="cm_hive" + while True: + params = {"pageSize": page_size, "startIndex": start} + page = ranger.get_policies_in_service(service_name, params=params) or [] + candidates.extend(page) + if len(page) < page_size: + break + start += len(page) + for p in candidates: + name = p["name"] + print(f"analizing policy:{name}") + if re.fullmatch(f"{prefix}([0-9]?[a-z]?)(_policy_)?([0-9]*)?(_row_level_policy)?(full_access)?$",name) != None: + try: + ranger.delete_policy_by_id(p["id"]) + deleted.append(name) + except Exception: + # continue attempting others + pass + if not deleted: + raise RuntimeError( + f"No matching policies found for deletion with prefix '{prefix}'. " + f"Provide 'policy_id' to delete a specific 2a policy." + ) + return deleted + + + +def generate_policy(params,env_config, policy_id: Optional[str] = None): + access_type = params['access_type'].lower() + if access_type == "1": + return yaml_format_1(params,env_config) + elif access_type == "2a": + return yaml_format_2a(params, env_config, policy_id) + elif access_type == "2b": + return yaml_format_1(params,env_config) + elif access_type == "3": + return yaml_format_3(params) + else: + raise Exception(f"Invalid access type {params['access_type']}. Please check the input param") + +def yaml_format_1(params,env_config) -> str: + ranger=_ranger_client(env_config) + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}" #corporatestore_table_accessType + policy.resources = { 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] }), + 'column': RangerPolicyResource({ 'values': params['columns'] }) } + + allowItem1 = RangerPolicyItem() + allowItem1.groups = params['igam_roles'] + allowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + policy.policyItems = [ allowItem1 ] + print(policy) + try: + created_policy = ranger.create_policy(policy) + print('Created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + except: + pass + ''' + yaml_format = f"""- name: "{{{{ ecb_env }}}} : {{{{ cdp_env_name }}}}: allow CRP RAR users to select core tables" + devo_ranger_client: + name: "cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}" + policy: + service: cm_hive + resources: + database: + values: + - {params['corporate_store'].lower()} + table: + values: + - {params['target_table']} + column: + values: + {params['columns']} + policyItems: + - groups: + {params['igam_roles'].lower()} + accesses: + - select +""" + return yaml_format' +''' + +def yaml_format_2a(params, env_config,policy_id: Optional[str]) -> str: + policy_ID = policy_id if policy_id is not None else "0" + + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_policy_{policy_ID}" #corporatestore_table_accessType + policy.resources = { 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] }), + 'column': RangerPolicyResource({ 'values': params['columns'] }) } + allowItem1 = RangerPolicyItem() + allowItem1.groups = params['igam_roles'] + allowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + policy.policyItems = [ allowItem1 ] + + + print(policy) + print("\n\n") + + + + + #created_policy = ranger.create_policy(policy) + #print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + + ''' + yaml_format = f"""- name: "{{{{ ecb_env }}}} : {{{{ cdp_env_name }}}}: allow CRP RAR users to select core tables" + devo_ranger_client: + name: "cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_policy_{policy_ID}" + policy: + service: cm_hive + resources: + database: + values: + - {params['corporate_store'].lower()} + table: + values: + - {params['target_table']} + column: + values: + {params['columns']} + policyItems: + - groups: + {params['igam_roles'].lower()} + accesses: + - select +""" + return yaml_format' + ''' + +def yaml_format_2b(params,env_config, full_access_list: Optional[List]) -> str: + +# For Kerberos authentication +# +# from requests_kerberos import HTTPKerberosAuth +# +# ranger_auth = HTTPKerberosAuth() + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" #harcoded + policy.name = f"cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_row_level_policy" #corporatestore_table_accessType + policy.isEnabled = True + policy.resources ={ 'database': RangerPolicyResource({ 'values': [params['corporate_store'].lower()] }), + 'table': RangerPolicyResource({ 'values': [params['target_table']] })} + rowFilterAllowItem1= RangerRowFilterPolicyItem() + rowFilterAllowItem1.groups = params['igam_roles'] + rowFilterAllowItem1.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem1.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"lower(source) IN (select lower(rar_subsource_id) from {params['corporate_store'].lower()}.t_ref_rar_sources_igam_sentry where lower(rar_igam_entitlement) IN (select ad_group from {params['corporate_store'].lower()}.active_directory_user_groups where username = lower(regexp_extract(current_user(),'[^@]*',0))))" }) + rowFilterAllowItem2= RangerRowFilterPolicyItem() + rowFilterAllowItem2.groups = [x.lower() for x in full_access_list] + rowFilterAllowItem2.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem2.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"1=1" }) + policy.rowFilterPolicyItems= [rowFilterAllowItem1, rowFilterAllowItem2] + print(policy) + + created_policy = ranger.create_policy(policy) + print(' created policy: name=' + created_policy.name + ', id=' + str(created_policy.id)) + ''' + yaml_format = f"""- name: "{{{{ ecb_env }}}} : {{{{ cdp_env_name }}}}: filter by confidentiality level" + devo_ranger_client: + name: "cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_row_level_policy" + policy: + isEnabled: "true" + service: cm_hive + resources: + database: + values: + - {params['corporate_store'].lower()} + table: + values: + - {params['target_table']} + rowFilterPolicyItems: + """ + return yaml_format + ''' + +def yaml_format_3(params, env_config,filterString, full_access_list: Optional[List]) -> str: + + ranger = _ranger_client(env_config) + + policy = RangerPolicy() + policy.service = "cm_hive" # hardcoded + policy.name = ( + f"cpo_{params['corporate_store'].lower()}_" + f"{params['target_table'].lower()}_" + f"{params['access_type'].lower()}_row_level_policy" + ) + policy.isEnabled = True + policy.resources = { + "database": RangerPolicyResource({"values": [params["corporate_store"].lower()]}), + "table": RangerPolicyResource({"values": [params["target_table"]]}), + } + + # Row filter item + rowFilterAllowItem = RangerRowFilterPolicyItem() + rowFilterAllowItem.groups = params["igam_roles"] + rowFilterAllowItem.accesses = [RangerPolicyItemAccess({"type": "select"})] + rowFilterAllowItem.rowFilterInfo = RangerPolicyItemRowFilterInfo( + { + "filterExpr": filterString + } + ) + rowFilterAllowItem2= RangerRowFilterPolicyItem() + rowFilterAllowItem2.groups = [x.lower() for x in full_access_list] + rowFilterAllowItem2.accesses = [RangerPolicyItemAccess({ 'type': 'select' })] + rowFilterAllowItem2.rowFilterInfo = RangerPolicyItemRowFilterInfo({ 'filterExpr': f"1=1" }) + + policy.rowFilterPolicyItems = [rowFilterAllowItem,rowFilterAllowItem2] + print(policy) + # Create policy in Ranger + created_policy = ranger.create_policy(policy) + print(f" created policy: name={created_policy.name}, id={created_policy.id}") + + return created_policy + + + """ + yaml_format = f"- name: "{{{{ ecb_env }}}} : {{{{ cdp_env_name }}}}: filter by confidentiality level" + devo_ranger_client: + name: "cpo_{params['corporate_store'].lower()}_{params['target_table'].lower()}_{params['access_type'].lower()}_row_level_policy" + policy: + isEnabled: "true" + service: cm_hive + resources: + database: + values: + - {params['corporate_store'].lower()} + table: + values: + - {params['target_table']} + rowFilterPolicyItems: + return yaml_format + """ \ No newline at end of file diff --git a/python/devo_replicator/table_generator/tableBuilderProcessor_3.py b/python/devo_replicator/table_generator/tableBuilderProcessor_3.py new file mode 100644 index 0000000..6c85653 --- /dev/null +++ b/python/devo_replicator/table_generator/tableBuilderProcessor_3.py @@ -0,0 +1,793 @@ +import pandasql as ps +import pandas as pd +import mrds.utils.manage_files as fileManager +import logging +import tableBuilderQueries as tbq +from devo_query import execute_query +import ranger_updater_old as ranger +import os +import yaml +import FlowOptions as fo +import numpy as np +from mrds.utils.secrets import get_secret +import traceback +from mrds.utils import oraconn + +# Set up basic configuration for logging +logging.basicConfig(level=logging.INFO) + +# Create a logger object +logger = logging.getLogger(__name__) +import re + +#0 utilities +def initialize_config(config_file_path): + # Ensure the file exists + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + + # Load the configuration + with open(config_file_path, "r") as f: + config_data = yaml.safe_load(f) + + return config_data + +def fix_impala_sql(sql: str) -> str: + # List of reserved keywords in Impala that need backticks if used as column names + impala_reserved_keywords = { + 'date', 'value', 'source', 'comment', 'partition', 'row', 'select', 'insert', + 'table', 'external', 'format', 'location', 'stored', 'inputformat', 'outputformat', + 'scenario', 'string', 'int', 'decimal', 'timestamp', 'float', 'double','procedure', 'floor' + } + + # Regex pattern to find column definitions + pattern = re.compile( + r'(?P`?\w+`?)\s+(?P[A-Za-z]+\s*(?:\([^)]+\))?)\s*(?Pcomment\s*\'[^\']*\'|)?', + re.IGNORECASE + ) + + def replace(match): + col = match.group('col').strip('`') + dtype = match.group('type') + comment = match.group('comment') or '' + # Add backticks only if column name is a reserved keyword or contains special chars + if col.lower() in impala_reserved_keywords or not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', col): + col = f'`{col}`' + return f"{col} {dtype} {comment}".strip() + + # Only replace column list part between parentheses + table_def_start = sql.find('(') + table_def_end = sql.find('ROW FORMAT SERDE', table_def_start) + if table_def_start == -1 or table_def_end == -1: + raise ValueError("Invalid SQL format: Missing column definition parentheses.") + + before = sql[:table_def_start + 1] + columns = sql[table_def_start + 1:table_def_end] + after = sql[table_def_end:] + + # Replace all columns inside definition + fixed_columns = pattern.sub(replace, columns) + + # Combine and return + final= before + fixed_columns + after + final=final.replace("\\'", "").replace('\\\\', '\\') + return final + + +def applyQueryParameters(query: str, parameters: str) -> str: + """ + Replaces placeholders in the query with values from parameters. + + Parameters: + - query: Original query string with placeholders like $$$1, $$$2, etc. + - parameters: Semicolon-separated string of parameter values. + + Returns: + - String with the query filled with parameter values. + """ + filled_query = query + if parameters: + # Split the parameters string and reverse the list + params_array = parameters.split(';')[::-1] + index = len(params_array) + for param in params_array: + # Replace the placeholder $$$ with the parameter + placeholder = f"$$${index}" + filled_query = filled_query.replace(placeholder, param) + index -= 1 # Decrement the index + return filled_query + +def format_column_definition(row): + if pd.isnull(row['data_description']): + # If data_description is null, only include column_name and data_type_string + return f"{row['column_name']} {row['data_type_string']}" + else: + # If data_description is present, include it with a comment + # Ensure data_description does not contain single quotes + data_description = str(row['data_description']).replace("'", "\\'") + return f"{row['column_name']} {row['data_type_string']} comment '{data_description}'" +#1 receive table name and check for target table and access type + +def execute_oracle_query(sql): + oracle_conn = oraconn.connect('MRDS_LOADER_MOPDB') + cursor = oracle_conn.cursor() + options=cursor.execute(sql).fetchall() + oracle_conn.commit() + + df = pd.DataFrame(options,columns= [row[0].lower() for row in cursor.description]) + ## fetch db dtypes + cursor.close() + oracle_conn.close() + return df + + +def get_target_table(oracle_mgmt_table,source_schema,source_table, env): + sql=f"SELECT DISTINCT TABLE_ALIAS FROM {oracle_mgmt_table} WHERE OWNER = '{source_schema}' AND TABLE_NAME = '{source_table}'" + df=execute_oracle_query(sql) + return df + +def get_type_ofAccess(oracle_metadata_table,source_schema,source_table,env): + sql=f"SELECT DISTINCT RAR3_TYPE_OF_ACCESS FROM {oracle_metadata_table} WHERE A_VALID_TO > SYSDATE AND OWNER = '{source_schema}'AND TABLE_NAME = '{source_table}'" + df=execute_oracle_query(sql) + return df + +#2 load metadata +def readIGAMRoles( config ,env): + queryParams = "'" + config.sentry_role_environment + "'" + igamRolesQuery = tbq.get_query_igam_roles(config.oracle_igam_table,config.service_name) + logger.info(f"Querying the IGAM Table") + + queryWithParamsIgamSentry = applyQueryParameters(igamRolesQuery, queryParams) + + logger.info(f"Replaced params to IGAM Table:") + + igamRoleDF = execute_oracle_query(queryWithParamsIgamSentry) + return igamRoleDF + +def loadMetadataTable( config,env ): + + metadataQuery = tbq.get_query_metadata(config.oracle_metadata_table, config.source_schema, config.source_table) + + logger.info("Map Oracle metadata (data types) to Hive query: ") + + jdbcMetaDataDF = df=execute_oracle_query(metadataQuery) + + logger.info("Fetch all fields for table and concatenate them separated by ','") + tableDataList = jdbcMetaDataDF.apply(format_column_definition, axis=1).tolist() + tableFields = ",".join(tableDataList) + + return tableFields + + +#3 drop table and policies +def deleteExternalTable(config,env_config): + try: + deleted=ranger.delete_policy(config,env_config) + except Exception as e: + pass + sql_drop = f"DROP TABLE IF EXISTS {config.corporate_store}.{config.target_table}" + execute_query( + sql_drop, + env_config['DEVO_USERNAME'], env_config['IMPALA_HOSTNAME'], env_config['DEVO_SECRET'], + ) + +#4 create external table and policies +def createExternalTables( config, tableFields,env_config ): + sql_create = ( + f"CREATE EXTERNAL TABLE {config.corporate_store}.{config.target_table} " + f"({tableFields}, {config.tech_meta_data_fields}) " + "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' " + "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' " + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' " + f"LOCATION '{config.target_s3_bucket}/{config.target_table}' " + "TBLPROPERTIES (" + "'external.table.purge'='true', " + "'parquet.compression'='snappy')" + ) + sql_create=fix_impala_sql(sql_create) + execute_query(sql_create,env_config['DEVO_USERNAME'],env_config['HIVE_HOSTNAME'],env_config['DEVO_SECRET']) + +def createTableFromExternal( config, tableFields,env_config ): + + sql_create = ( + f"CREATE EXTERNAL TABLE {config.corporate_store}.{config.target_table} AS " + f"SELECT * FROM {config.corporate_store}.{config.target_table}_EXT" + ) + + execute_query(sql_create,env_config['DEVO_USERNAME'],env_config['HIVE_HOSTNAME'],env_config['DEVO_SECRET']) + +def accessTypeMapper(config, env_config, igamRoleDF): + + if config.access_type.lower() == '1': + accessType_1(config, env_config, igamRoleDF) + elif (config.access_type.lower() == '2a'): + accessType_2A(config, env_config, igamRoleDF) + elif (config.access_type.lower() == '2b'): + accessType_2B(config, env_config, igamRoleDF) + elif (config.access_type.lower() == '3'): + accessType_3(config, env_config, igamRoleDF) + else: + logger.info(f"Invalid access type {config.access_type}. Please check the input param") + +def accessType_1(config, env_config, igamRoleDF): + logger.info("Grant privileges for access type 1") + logger.info("Fetch metadata from Oracle for access type 1") + + # ---- Construct query and fetch from Oracle ---- + queryParams = f"'{config.source_schema}.{config.source_table}'" + queryMetadataAccessType1 = tbq.get_query_metadata_access_type1(config.oracle_metadata_table) + queryWithParamsAccessType1 = applyQueryParameters(queryMetadataAccessType1, queryParams) + + logger.info("Metadata table query: " ) + jdbcMetaDataAccessType1DF = df=execute_oracle_query(queryWithParamsAccessType1) + + # ---- Normalize columns ---- + df = jdbcMetaDataAccessType1DF.copy() + df["rar3_type_of_access"] = df["rar3_type_of_access"].astype(str).str.strip() + df["source"] = df["source"].astype(str).str.strip().str.upper() + igamRoleDF["datasource"] = igamRoleDF["datasource"].astype(str).str.strip().str.upper() + + # ---- Branch A: source != 'RAR' ---- + left_a = ( + df.loc[ + (df["rar3_type_of_access"] == "1") & (df["source"] != config.service_name), + ["table_name", "source"] + ] + .drop_duplicates() + ) + + branch_a = ( + left_a.merge( + igamRoleDF, + left_on="source", + right_on="datasource", + how="inner" + ) + [["table_name", "source", "subsource_id", "igam_entitlement", "environment"]] + .drop_duplicates() + ) + + # ---- Branch B: source == 'RAR' (CROSS JOIN with igamRoleDF) ---- + left_b = ( + df.loc[ + (df["rar3_type_of_access"] == "1") & (df["source"] == config.service_name), + ["table_name", "source"] + ] + .drop_duplicates() + ) + + if not left_b.empty: + branch_b = ( + left_b.merge(igamRoleDF, how="cross") + [["table_name", "source", "subsource_id", "igam_entitlement", "environment"]] + .drop_duplicates() + ) + else: + branch_b = pd.DataFrame(columns=["table_name", "source", "subsource_id", "igam_entitlement", "environment"]) + + # ---- UNION (distinct) ---- + typeOneDF = ( + pd.concat([branch_a, branch_b], ignore_index=True) + .drop_duplicates() + .reset_index(drop=True) + ) + + # ---- Collect IGAM entitlements ---- + igam_entitlements = ( + typeOneDF["igam_entitlement"] + .dropna() + .astype(str) + .str.strip() + .tolist() + ) + # Extract IGAM entitlements + + # Merge with optional full access list + if config.full_access_entitlement_list is None: + combined_entitlements = igam_entitlements + else: + full_access_list_clean = config.full_access_entitlement_list + combined_entitlements = igam_entitlements + full_access_list_clean + + # Add table permission groups using YAMLFormatter + params = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + combined_entitlements + ) + + # Generate the final YAML policy + formattedYaml = ranger.generate_policy(params,env_config, None) + logger.info(f"Final YAML format") + + return formattedYaml + + + + + +def accessType_2A(config, env_config, igamRoleDF): + logger.info("Grant privileges for access type 2a") + logger.info("Fetch the metadata in Oracle for access type 2a") + + # ---- Construct query and fetch from Oracle ---- + queryParams = f"'{config.source_schema}.{config.source_table}'" + queryMetadataAccessType2a = tbq.get_query_metadata_access_type2a(config.oracle_metadata_table) + queryWithParamsAccessType2a = applyQueryParameters(queryMetadataAccessType2a, queryParams) + + logger.info(f"Meta data table query: {queryWithParamsAccessType2a} ") + jdbcMetaDataAccessType2aDF = execute_oracle_query(queryWithParamsAccessType2a) + + # ---- Normalize columns ---- + df = jdbcMetaDataAccessType2aDF.copy() + df["rar3_type_of_access"] = df["rar3_type_of_access"].astype(str).str.strip().str.lower() + df["source"] = df["source"].astype(str).str.strip().str.upper() + print(df) + + roles = igamRoleDF.copy() + # expected columns in igamRoleDF: rar_subsource_id, igam_entitlement, environment (plus anything else you keep) + roles["subsource_id"] = roles["subsource_id"].astype(str).str.strip().str.upper() + roles["igam_entitlement"] = roles["igam_entitlement"].astype(str).str.strip() + + # ---- Branch A: source != service_name -> INNER JOIN on source == rar_subsource_id ---- + left_a = ( + df.loc[ + (df["rar3_type_of_access"] == "2a") + & (df["source"] != config.service_name.upper()), + ["table_name", "column_name", "source"] + ] + ) + + branch_a = ( + left_a.merge( + roles, + left_on="source", + right_on="subsource_id", + how="inner" + ) + .drop(columns=["subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + + # ---- Branch B: source == service_name -> CROSS JOIN with igamRoleDF ---- + left_b = ( + df.loc[ + (df["rar3_type_of_access"] == "2a") + & (df["source"] == config.service_name.upper()), + ["table_name", "column_name", "source"] + ] + ) + + if not left_b.empty: + try: + branch_b = ( + left_b.merge(roles, how="cross") + .drop(columns=["subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + except TypeError: + # pandas < 1.2 fallback + left_b["_cj"] = 1 + roles["_cj"] = 1 + branch_b = ( + left_b.merge(roles, on="_cj") + .drop(columns=["_cj", "subsource_id", "source"], errors="ignore") + [["table_name", "column_name", "igam_entitlement", "environment"]] + ) + # (optional) cleanup if you keep using roles later + roles.drop(columns=["_cj"], inplace=True, errors="ignore") + else: + branch_b = pd.DataFrame(columns=["table_name", "column_name", "igam_entitlement", "environment"]) + + # ---- UNION (distinct) ---- + one_df = ( + pd.concat([branch_a, branch_b], ignore_index=True) + .reset_index(drop=True) + ) + + # ---- Group 1: (table_name, igam_entitlement) -> sorted, comma-joined column_list ---- + tmp = one_df.sort_values(["table_name", "igam_entitlement", "column_name"], kind="mergesort") + new_df = ( + tmp.groupby(["table_name", "igam_entitlement"], as_index=False)["column_name"] + .apply(lambda s: ",".join(s.dropna().astype(str).tolist())) + .rename(columns={"column_name": "column_list"}) + ) + # Columns: table_name, igam_entitlement, column_list + + # ---- Group 2: (table_name, column_list) -> comma-joined igam_entitlement ---- + grouped = ( + new_df.groupby(["table_name", "column_list"], as_index=False)["igam_entitlement"] + .apply(lambda s: ",".join(s.dropna().astype(str).tolist())) + ) + # Columns: table_name, column_list, igam_entitlement + + # ---- ROW_NUMBER() OVER (ORDER BY column_list) -> policy_id ---- + grouped = grouped.sort_values(["column_list"], kind="mergesort") + grouped["policy_id"] = np.arange(1, len(grouped) + 1).astype(int) + + # ---- Emit policies: one per (table_name, column_list) row ---- + for _, row in grouped.iterrows(): + entitlements_list = [e.strip() for e in str(row["igam_entitlement"]).split(",") if e.strip()] + columns_list = [c.strip() for c in str(row["column_list"]).split(",") if c.strip()] + policy_id = str(int(row["policy_id"])) + + params = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, # "2a" + config.source_table, + entitlements_list, + columns_list=columns_list + ) + ranger.generate_policy(params, env_config, policy_id) + + + # ---- Optional: append full-access YAML if list provided on config ---- + if getattr(config, "full_access_entitlement_list", None): + # If your code already provides a list, use it directly; otherwise split string. + if isinstance(config.full_access_entitlement_list, list): + full_access_list = config.full_access_entitlement_list + else: + full_access_list = [s.strip() for s in str(config.full_access_entitlement_list).split(",") if s.strip()] + + params_full = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, # keep same access type per your pattern + config.source_table, + full_access_list + ) + ranger.generate_policy(params_full, env_config, "full_access") + + + + +def accessType_2B(config, env_config,igamRoleDF): + logger.info(f"Grant privileges for access type {config.access_type}") + logger.info("Fetch the metadata in Oracle for access type 2b") + + # --- Validate required columns --- + required = {"environment", "igam_entitlement", "subsource_id"} + missing = required - set(igamRoleDF.columns) + if missing: + raise KeyError(f"igamRoleDF missing required column(s): {sorted(missing)}") + + # --- Normalize to strings (robust against None/NaN) --- + igamRoleDF = igamRoleDF.copy() + igamRoleDF["environment"] = igamRoleDF["environment"].astype(str).str.strip() + igamRoleDF["igam_entitlement"] = igamRoleDF["igam_entitlement"].astype(str).str.strip() + igamRoleDF["subsource_id"] = igamRoleDF["subsource_id"].astype(str).str.strip() + + # --- Aggregation: per (environment, igam_entitlement) collect unique subsource_id list --- + # Keep a stable order by sorting; remove empties. + agg_df = ( + igamRoleDF.loc[igamRoleDF["subsource_id"].ne(""), ["environment", "igam_entitlement", "subsource_id"]] + .drop_duplicates() + .sort_values(["environment", "igam_entitlement", "subsource_id"], kind="mergesort") + .groupby(["environment", "igam_entitlement"], as_index=False)["subsource_id"] + .agg(lambda s: ",".join(s.unique())) + .rename(columns={"subsource_id": "subsource_id_list"}) + ) + + # List of tuples (IGAM_ENTITLEMENT, subsource_id_list) — mirrors your log payload + accessType2bValidList = list(zip( + agg_df["igam_entitlement"].astype(str), + agg_df["subsource_id_list"].astype(str) + )) + + # --- Entitlements for policy generation (unique, non-empty) --- + igam_entitlements = ( + igamRoleDF["igam_entitlement"] + .dropna() + .map(str) + .str.strip() + .loc[lambda s: s.ne("")] + .drop_duplicates() + .tolist() + ) + + # --- Row-level permissions (per your existing API) --- + params_row_level = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + igam_entitlements + ) + + # --- Table-level permissions, merging in full-access entitlements if provided --- + if getattr(config, "full_access_entitlement_list", None): + combined_entitlements = igam_entitlements + config.full_access_entitlement_list + else: + combined_entitlements = igam_entitlements + + + # --- Emit YAML using your helpers --- + + if getattr(config, "full_access_entitlement_list", None): + params = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + combined_entitlements + ) + ranger.generate_policy(params, env_config) + + + ranger.yaml_format_2b(params_row_level,env_config, config.full_access_entitlement_list) # row-level policy + logger.info("Final YAML format emitted for 2B.") + + + + +def accessType_3(config,env_config, igamRoleDF): + """ + Python/pandas translation of the Scala accessType_3. + Expects igamRoleDF to have at least: ['igam_entitlement', 'subsource_id']. + The `config` object should expose the attributes used below (names match your Scala/Python usage). + Uses a YAML formatter module `ranger` with: + - add_table_permission_groups(corporate_store, target_table, access_type, source_table, entitlements) + - yaml_format_3(params) + - yaml_format_1(params) + """ + + # --- 1) Filter entitlements where subsource_id = 'TMS' --- + if not {"igam_entitlement", "subsource_id"}.issubset(igamRoleDF.columns): + missing = {"igam_entitlement", "subsource_id"} - set(igamRoleDF.columns) + raise KeyError(f"igamRoleDF missing required column(s): {sorted(missing)}") + + new_df = ( + igamRoleDF.loc[ + igamRoleDF["subsource_id"].astype(str).str.upper() == "TMS", + ["igam_entitlement"] + ].drop_duplicates() + ) + + accessType3ValidList = new_df["igam_entitlement"].astype(str).str.strip().tolist() + + # --- 2) Build params for row-level groups (type 3) --- + params_row_level = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + accessType3ValidList + ) + + corp = str(config.corporate_store).lower() + src_tbl = str(config.source_table).lower() + + # --- 3) Compose the filter expressions (match Scala strings) --- + sqlCreateView3NonRestrString_Ptree = ( + "(parent_fk in ( " + f"select portfolio_fk from {corp}.nh_portfolio_access " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + ")) AND (child_fk in ( " + f"select portfolio_fk from {corp}.nh_portfolio_access " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + "))" + ) + + sqlCreateView3NonRestrString_Pos = ( + "position_key in ( " + f"select position_key from {corp}.nh_portfolio_access a " + f"inner join {corp}.nh_position b on ( " + "(b.portfolio_fk = a.portfolio_fk and b.portfolio_fk is not NULL) or " + "(b.portfolio_compare_fk = a.portfolio_fk and b.portfolio_compare_fk is not NULL) " + ") " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp() " + ")" + ) + + sqlCreateView3PortAccess = "lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%')" + sqlCreateView3LimAccess = "lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%')" + + # Standard case uses the configured key columns/table names + key_col = getattr(config, "type3SourceTableKeyColumn", None) + acc_col = getattr(config, "type3AccessTableKeyColumn", None) + acc_table= getattr(config, "type3AccessTable", None) + if not all([key_col, acc_col, acc_table]): + # Only needed for the default branch; keep None if your config doesn't use the default + key_col = key_col or "source_key_col" + acc_col = acc_col or "access_key_col" + acc_table = acc_table or "type3_access_table" + + sqlCreateView3NonRestrString_Stdrd = ( + f"{key_col} in (select {acc_col} from {corp}.{acc_table} " + "where lower(user_id) LIKE concat('%', lower(regexp_extract(current_user(),'[^@]*',0)), '%') " + "AND to_date(a_valid_to) > current_timestamp())" + ) + + # --- 4) Choose the filter by source table (matches Scala match/case) --- + if src_tbl == "nh_portfoliotree": + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Ptree + elif src_tbl == "nh_position": + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Pos + elif src_tbl == "nh_portfolio_access": + sqlCreateViewType3Filter = sqlCreateView3PortAccess + elif src_tbl == "nh_limit_access": + sqlCreateViewType3Filter = sqlCreateView3LimAccess + else: + sqlCreateViewType3Filter = sqlCreateView3NonRestrString_Stdrd + + # --- 5) Row filter YAML block (uses groups from params_row_level) --- + # Expecting params_row_level like {'igam_roles': '...'}; adjust key if your API differs. + igam_roles_lower = str(params_row_level.get("igam_roles", "")).lower() + rowFilter = ( + "- groups:\n" + f" {igam_roles_lower}\n" + " accesses:\n" + " - select\n" + f" filterExpr: \"{sqlCreateViewType3Filter}\"\n" + " " + ) + + # --- 6) Handle optional full access entitlements --- + + + if config.full_access_entitlement_list: + paramsFullAccess = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + config.full_access_entitlement_list + ) + + full_groups_lower = str(paramsFullAccess.get("igam_roles", "")).lower() + """ + fullAccessFilter = ( + "- groups:\n" + f" {full_groups_lower}\n" + " accesses:\n" + " - select\n" + " filterExpr: \"1=1\"\n" + " " + ) + """ + params_table_level = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + accessType3ValidList + config.full_access_entitlement_list + ) + else: + fullAccessFilter = "" + params_table_level = ranger.add_table_permission_groups( + config.corporate_store, + config.target_table, + config.access_type, + config.source_table, + accessType3ValidList + ) + + # --- 7) Render YAML and merge like Scala --- + ranger.yaml_format_3(params_row_level,env_config,sqlCreateViewType3Filter,config.full_access_entitlement_list ) # base type 3 yaml + ranger.yaml_format_1(params_table_level,env_config) # table-level yaml + + +#5 create extra policies for super-users +#6 refresh metadata + +def run_process(env_file, env, service_name,source_schema,source_table,sentry_role_environment): + #1 receive table name and check for target table and access type + env_dict=initialize_config(env_file) + env_config=env_dict[env] + if service_name.lower()=='rqsd': + env_config["DEVO_SECRET"]=env_config["DEVO_SECRET_RQSD"] + env_config["DEVO_USERNAME"]=env_config["DEVO_USERNAME_RQSD"] + try: + devo_secret_name = env_config["DEVO_SECRET"] + env_config["DEVO_SECRET"]= get_secret(devo_secret_name) + except: + logger.error("Failed to retrieve credentials from secrets") + raise(Exception) + db_config=env_dict[service_name] + try: + target_table=get_target_table(db_config['oracle_mgmt_table'],source_schema,source_table,env)['table_alias'][0] + except Exception as e: + logger.error("Table not found in oracle management table") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + raise + try: + access_type=get_type_ofAccess(db_config['oracle_metadata_table'],source_schema,source_table,env)['rar3_type_of_access'][0].strip() + except Exception as e: + logger.error("Table not found in oracle metadata inventory") + logger.error("Exception: %s", e) + logger.error("Traceback:\n%s", traceback.format_exc()) + raise + args={ + 'corporate_store':db_config['corporate_store'], + 'service_name': service_name, + 'source_schema':source_schema, + 'source_table':source_table, + 'oracle_metadata_table':db_config['oracle_metadata_table'], + 'oracle_igam_table':db_config['oracle_igam_table'], + 'oracle_mgmt_table': db_config['oracle_mgmt_table'], + 'target_table':target_table, + 'sentry_role_environment':sentry_role_environment, + 'target_s3_bucket': env_config["BUCKET_PREFIX"]+db_config['target_s3_bucket'] , + 'tech_meta_data_fields': db_config['tech_meta_data_fields'], + 'full_access_entitlement_list':env_config[f"FULL_ACCESS_LIST_{service_name.upper()}"].split(','), + 'access_type': access_type + } + config=fo.Options(args) + #2 load metadata + tableFields=loadMetadataTable(config,env) + igamRoles=readIGAMRoles(config,env) + #3 drop table and policies + deleteExternalTable(config,env_config) + #4 create external table and policies + if (config.target_table[-4:].upper() == '_EXT'): + createExternalTables( config, tableFields,env_config ) + else: + createTableFromExternal( config, tableFields,env_config) + + accessTypeMapper(config,env_config,igamRoles) + #5 refresh metadata + #execute_query(f"INVALIDATE METADATA {config.corporate_store}.{config.target_table}",env_config["DEVO_USERNAME"],env_config['IMPALA_HOSTNAME'],env_config['DEVO_SECRET']) + #execute_query(f"COMPUTE STATS {config.corporate_store}.{config.target_table}",env_config["DEVO_USERNAME"],env_config['IMPALA_HOSTNAME'],env_config['DEVO_SECRET']) + +#run_process("/home/dbt/Marco/mrds_elt/python/devo_replicator/env_config.yaml",'tst','mopdb','MPEC','T_MPEC','TEST/INTEGRATION') + +#run_process("/home/dbt/Marco/mrds_elt/python/devo_replicator/config/env_config.yaml",'tst','rar','CORR_RAR','NH_ASSET','TEST/INTEGRATION') +#run_process("/home/dbt/Marco/mrds_elt/python/devo_replicator/config/env_config.yaml",'dev','rar','CORR_RAR','NH_LIMIT','TEST/INTEGRATION') + +run_process("/home/dbt/Marco/mrds_elt/python/devo_replicator/config/env_config.yaml",'dev','rar','CORR_RAR','NH_Asset_transactial_data'.upper(),'TEST/INTEGRATION') +""" +df=execute_oracle_query('select owner, table_name from CT_MRDS.A_DEVO_REPLICA_MGMT_RQSD') +listfail=[] +for index, row in df.iterrows(): + + try: + print("running table: ",row["table_name"]) + run_process("/home/dbt/Marco/mrds_elt/python/devo_replicator/config/env_config.yaml",'tst','rqsd',row['owner'],row['table_name'].upper(),'TEST/INTEGRATION') + except: + print("failed") + listfail.append(row["table_name"]) + print("succeded") +print(listfail) +""" + + +'''{"id": 48754, "guid": "d75f1491-538d-402a-a8ac-e7e21ac0be53", "isEnabled": true, "version": 1, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_2a_policy_1", "policyType": 0, "policyPriority": 0, "description": "created-ranger_client-v0.0.6-2025-10-17T14:28:24.135108", "isAuditEnabled": true, "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, +"column": {"values": ["ASSET_FK", "ASSET_FK", "A_DWH_LOAD_SET_FK", "A_DWH_LOAD_SET_FK", "A_VALID_FROM", "A_VALID_FROM", "A_VALID_TO", "A_VALID_TO", "CODE_TYPE_NO_ID", "CODE_TYPE_NO_ID", "CODE_VALUE", "CODE_VALUE", "INDEP_SUBPROCESS_FK", "INDEP_SUBPROCESS_FK", "TEC_EXECUTION_DATE", "TEC_EXECUTION_DATE", "TEC_INGESTION_DATE", "TEC_INGESTION_DATE", "TEC_RUN_ID", "TEC_RUN_ID"], "isExcludes": false, "isRecursive": false}, +"table": {"values": ["MU_ASSET_CODE_MAP"], "isExcludes": false, "isRecursive": false}}, "policyItems": [{"accesses": [{"type": "select", "isAllowed": true}], +"groups": ["a_mopdb_ea", "disc-au-bda"], "delegateAdmin": false}], "serviceType": "hive", "isDenyAllElse": false} + +{"id": 48755, "guid": "5ff857c2-3683-4178-98ce-5932c0677cd4", "isEnabled": true, "version": 1, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_2a_policy_2", "policyType": 0, "policyPriority": 0, "description": "created-ranger_client-v0.0.6-2025-10-17T14:28:24.135108", "isAuditEnabled": true, "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, +"column": {"values": ["ASSET_FK", "A_DWH_LOAD_SET_FK", "A_VALID_FROM", "A_VALID_TO", "CODE_TYPE_NO_ID", "CODE_VALUE", "INDEP_SUBPROCESS_FK", "TEC_EXECUTION_DATE", "TEC_INGESTION_DATE", "TEC_RUN_ID"], "isExcludes": false, "isRecursive": false}, + "table": {"values": ["MU_ASSET_CODE_MAP"], "isExcludes": false, "isRecursive": false}}, "policyItems": [{"accesses": [{"type": "select", "isAllowed": true}], " + groups": ["su-omd-reuters-users", "a_mopdb_excess_liquidity", "a-mora-lba-exp-a", "a_rar_csdb_reference_data", "a_mopdb_uc", "a_rar_csdb_ratings_data", "a_mopdb_credit_operations", "a_rar_fxcd_data", "a_rar_mdp_bbg_data", "disc-ac-riad_cnf_n-r", "a-mora-lba-ana-a", "a_mopdb_tms_data", "disc-ac-riad_core-r", "a_mopdb_mpec", "a-led-ana-a", "a-led-exp-a", "a_mopdb_ela_all"], "delegateAdmin": false}], "serviceType": "hive", "isDenyAllElse": false} + + {"id": 48756, "guid": "1071767f-8ef6-47be-bb9b-7077ed9e9a90", "isEnabled": true, "version": 1, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_2a_policy_full_access", "policyType": 0, "policyPriority": 0, "description": "created-ranger_client-v0.0.6-2025-10-17T14:28:24.135108", "isAuditEnabled": true, "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, "column": {"values": ["*"], "isExcludes": false, "isRecursive": false}, "table": {"values": ["MU_ASSET_CODE_MAP"], "isExcludes": false, "isRecursive": false}}, +"policyItems": [{"accesses": [{"type": "select", "isAllowed": true}], "groups": ["disc-ac-rar-r"], "delegateAdmin": false}], "serviceType": "hive", "isDenyAllElse": false}''' + + +''' +{"isEnabled": true, "isDenyAllElse": false, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_ext_2a_policy_1", "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, "table": {"values": ["MU_ASSET_CODE_MAP_EXT"], "isExcludes": false, "isRecursive": false}, +"column": {"values": ["ASSET_FK", "ASSET_FK", "A_DWH_LOAD_SET_FK", "A_DWH_LOAD_SET_FK", "A_VALID_FROM", "A_VALID_FROM", "A_VALID_TO", "A_VALID_TO", "CODE_TYPE_NO_ID", "CODE_TYPE_NO_ID", "CODE_VALUE", "CODE_VALUE", "INDEP_SUBPROCESS_FK", "INDEP_SUBPROCESS_FK", "TEC_EXECUTION_DATE", "TEC_EXECUTION_DATE", "TEC_INGESTION_DATE", "TEC_INGESTION_DATE", "TEC_RUN_ID", "TEC_RUN_ID"], "isExcludes": false, "isRecursive": false}}, "policyItems": [{"delegateAdmin": false, + "groups": ["disc-tu-bda", "t_mopdb_ea"], "accesses": [{"type": "select", "isAllowed": true}]}]} + + + +{"isEnabled": true, "isDenyAllElse": false, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_ext_2a_policy_2", "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, +"table": {"values": ["MU_ASSET_CODE_MAP_EXT"], "isExcludes": false, "isRecursive": false}, + "column": {"values": ["ASSET_FK", "A_DWH_LOAD_SET_FK", "A_VALID_FROM", "A_VALID_TO", "CODE_TYPE_NO_ID", "CODE_VALUE", "INDEP_SUBPROCESS_FK", "TEC_EXECUTION_DATE", "TEC_INGESTION_DATE", "TEC_RUN_ID"], "isExcludes": false, "isRecursive": false}}, "policyItems": [{"delegateAdmin": false, + "groups": ["a-led-ana-t", "a-led-exp-t", "a-mora-lba-ana-t", "a-mora-lba-exp-t", "disc-tc-riad_cnf_n-r", "disc-tc-riad_core-r", "su-omd-reuters-users", "t_mopdb_credit_operations", "t_mopdb_ela_all", "t_mopdb_excess_liquidity", "t_mopdb_mpec", "t_mopdb_tms_data", "t_mopdb_uc", "t_rar_csdb_ratings_data", "t_rar_csdb_reference_data", "t_rar_fxcd_data", "t_rar_mdp_bbg_data"], + "accesses": [{"type": "select", "isAllowed": true}]}]} + + + +{"isEnabled": true, "isDenyAllElse": false, "service": "cm_hive", "name": "cpo_crp_rar_mu_asset_code_map_ext_2a_policy_full_access", "resources": {"database": {"values": ["crp_rar"], "isExcludes": false, "isRecursive": false}, + "table": {"values": ["MU_ASSET_CODE_MAP_EXT"], "isExcludes": false, "isRecursive": false}, "column": {"values": ["*"], "isExcludes": false, "isRecursive": false}}, "policyItems": [{"delegateAdmin": false, + "groups": ["disc-dc-rar-r"], "accesses": [{"type": "select", "isAllowed": true}]}]} +''' \ No newline at end of file diff --git a/python/devo_replicator/table_generator/tableBuilderQueries.py b/python/devo_replicator/table_generator/tableBuilderQueries.py new file mode 100644 index 0000000..3f3dee5 --- /dev/null +++ b/python/devo_replicator/table_generator/tableBuilderQueries.py @@ -0,0 +1,130 @@ +metadata_table = "DW_RAR.NH_METADATA_INVENTORY" + +def get_query_metadata(metadata_table, owner, table_name): + query_metadata = ( + "WITH metaDF AS ( " + "SELECT owner, table_name, column_id, column_name, data_type, data_precision, data_scale, " + "CASE WHEN data_precision IS NULL AND data_scale IS NULL THEN NULL " + "WHEN data_precision IS NOT NULL AND data_scale IS NULL THEN data_precision " + "WHEN CAST(data_precision AS INT) >= CAST(data_scale AS INT) AND CAST(data_scale AS INT) >= 0 THEN data_precision " + "WHEN CAST(data_precision AS INT) < CAST(data_scale AS INT) AND CAST(data_scale AS INT) <= 38 AND CAST(data_scale AS INT) > 0 THEN data_scale " + "WHEN CAST(data_precision AS INT) < CAST(data_scale AS INT) AND (CAST(data_scale AS INT) > 38 OR CAST(data_scale AS INT) < 0) THEN NULL " + "ELSE NULL END AS data_precision_hive, " + "CASE WHEN data_precision IS NULL AND data_scale IS NULL THEN NULL " + "WHEN data_precision IS NOT NULL AND data_scale IS NULL THEN CAST(0 AS INT) " + "WHEN CAST(data_precision AS INT) >= CAST(data_scale AS INT) AND CAST(data_scale AS INT) >= 0 THEN data_scale " + "WHEN CAST(data_precision AS INT) < CAST(data_scale AS INT) AND CAST(data_scale AS INT) <= 38 AND CAST(data_scale AS INT) > 0 THEN data_scale " + "WHEN CAST(data_precision AS INT) < CAST(data_scale AS INT) AND (CAST(data_scale AS INT) > 38 OR CAST(data_scale AS INT) < 0) THEN NULL " + "ELSE NULL END AS data_scale_hive, " + "CASE WHEN data_type LIKE '%NUMBER%' AND data_precision IS NULL AND data_scale IS NULL THEN 'String' " + "WHEN data_type LIKE '%NUMBER%' AND data_precision IS NOT NULL AND data_scale IS NULL THEN 'Decimal' " + "WHEN data_type LIKE '%NUMBER%' AND CAST(data_precision AS INT) >= CAST(data_scale AS INT) AND CAST(data_scale AS INT) >= 0 THEN 'Decimal' " + "WHEN data_type LIKE '%NUMBER%' AND CAST(data_precision AS INT) < CAST(data_scale AS INT) AND CAST(data_scale AS INT) <= 38 AND CAST(data_scale AS INT) > 0 THEN 'Decimal' " + "WHEN data_type LIKE '%NUMBER%' AND CAST(data_precision AS INT) < CAST(data_scale AS INT) AND (CAST(data_scale AS INT) > 38 OR CAST(data_scale AS INT) < 0) THEN 'String' " + "WHEN data_type LIKE '%CHAR%' THEN 'String' " + "WHEN data_type LIKE '%VARCHAR2%' THEN 'String' " + "WHEN data_type LIKE '%TIMESTAMP%' THEN 'String' " + "WHEN data_type LIKE '%DATE%' THEN 'String' " + "ELSE 'String' END AS data_type_hive, " + "REGEXP_REPLACE(data_description, '''', '\\''') AS data_description " + "FROM {0} " + "WHERE lower(owner||'.'||table_name) = lower('{1}'||'.'||'{2}') " + "AND a_valid_to > sysdate) " + "SELECT owner, table_name, column_id, column_name, data_type, data_precision, data_scale, " + "data_precision_hive, data_scale_hive, data_type_hive, " + "CASE WHEN data_type_hive = 'Decimal' THEN 'Decimal(' || COALESCE(CAST(data_precision_hive AS VARCHAR2(30)), '') || ',' || COALESCE(CAST(data_scale_hive AS VARCHAR2(30)), '') || ')' " + "ELSE data_type_hive END AS data_type_string, data_description " + "FROM metaDF " + "ORDER BY CAST(column_id AS INT) " + ).format(metadata_table, owner, table_name) + + return query_metadata + +def get_query_metadata_access_type1(metadata_table): + query_metadata_access_type1 = ( + "SELECT owner, table_name, list_of_sources as SOURCE, rar3_type_of_access " + "FROM {0} " + "WHERE a_valid_to > sysdate " + "AND rar3_type_of_access = '1' " + "AND list_of_sources NOT IN 'RAR' " + "AND lower(owner||'.'||table_name) = lower($$$1) " + "UNION " + "SELECT owner, table_name, list_of_sources as SOURCE, rar3_type_of_access " + "FROM {0} " + "WHERE a_valid_to > sysdate " + "AND rar3_type_of_access = '1' " + "AND owner = 'CORR_REF_MAIN' " + "AND lower(owner||'.'||table_name) = lower($$$1) " + ).format(metadata_table) + + return query_metadata_access_type1 + +def get_query_metadata_access_type2a(metadata_table): + query_metadata_access_type2a = ( + "WITH rar_columns AS ( " + "SELECT owner, table_name, column_name, source, rar3_type_of_access " + "FROM ( " + "SELECT owner, table_name, column_name, rar3_type_of_access, list_of_sources, " + "tms, c2d_ea, c2d_ela, c2d_mpec, c2d_uc, ceph, lm, csdb_reference, " + "csdb_ratings, fxcd, mdp_bbg, mdp_reu, riad_cl, riad_ou, sdw_estr, sdw_fx, " + "top, rar, rtm, led, mdp_cma " + "FROM {0} " + "WHERE a_valid_to > sysdate " + "AND rar3_type_of_access = '2a' " + "AND lower(owner || '.' || table_name) = lower($$$1) " + "AND list_of_sources NOT LIKE '%,%' " + "AND upper(column_name) NOT IN ('DATABASE') " + ") a " + "UNPIVOT ( " + "val FOR (source) IN ( " + "tms AS 'TMS', c2d_ea AS 'C2D_EA', c2d_ela AS 'C2D_ELA', c2d_mpec AS 'C2D_MPEC', " + "c2d_uc AS 'C2D_UC', ceph AS 'CEPH', lm AS 'LM', csdb_reference AS 'CSDB_REFERENCE', " + "csdb_ratings AS 'CSDB_RATINGS', fxcd AS 'FXCD', mdp_bbg AS 'MDP_BBG', mdp_reu AS 'MDP_REU', " + "riad_cl AS 'RIAD_CL', riad_ou AS 'RIAD_OU', sdw_estr AS 'SDW_ESTR', sdw_fx AS 'SDW_FX', " + "top AS 'TOP', rar AS 'RAR', rtm AS 'RTM', led AS 'LED', mdp_cma AS 'MDP_CMA') " + ") " + "ORDER BY owner, table_name, column_name " + "), " + "dummy_entry AS ( " + "SELECT owner, table_name, 'RAR' as SOURCE, rar3_type_of_access " + "FROM rar_columns " + "FETCH FIRST ROW ONLY " + "), " + "disc_tec_fields AS ( " + "SELECT owner, table_name, 'TEC_INGESTION_DATE' AS column_name, source, rar3_type_of_access " + "FROM dummy_entry " + "UNION " + "( " + "SELECT owner, table_name, 'TEC_EXECUTION_DATE' AS column_name, source, rar3_type_of_access " + "FROM dummy_entry " + ") " + "UNION " + "( " + "SELECT owner, table_name, 'TEC_RUN_ID' AS column_name, source, rar3_type_of_access " + "FROM dummy_entry " + ") " + ") " + "SELECT owner, table_name, column_name, source, rar3_type_of_access " + "FROM disc_tec_fields " + "UNION " + "SELECT owner, table_name, column_name, source, rar3_type_of_access " + "FROM rar_columns " + ).format(metadata_table) + + return query_metadata_access_type2a + +def get_query_igam_roles(igam_table,service): + if service.lower() == 'rar': + service_entitlement='mrds' + elif service.lower()=='mopdb': + service_entitlement='mrds' + else: + service_entitlement='mrds' + query_igam_roles = ( + "SELECT MRDS_subsource_id as Datasource, " + "MRDS_subsource_id as subsource_id, " + "MRDS_entitlement as IGAM_Entitlement, " + "environment " + "FROM {0} where lower(environment) = lower($$$1) and SERVICE_NAME='{1}'").format(igam_table,service.upper()) + + return query_igam_roles \ No newline at end of file diff --git a/python/devo_replicator/table_generator/testScript.py b/python/devo_replicator/table_generator/testScript.py new file mode 100644 index 0000000..58bed7c --- /dev/null +++ b/python/devo_replicator/table_generator/testScript.py @@ -0,0 +1,54 @@ +import os + +import mrds_elt.python.devo_replicator.FlowOptions as ro +import tableBuilderProcessor_2 as tbp + +# setting variables +# args = [ +# 'corporate_store=crp_rar', +# 'source_schema=CORR_RAR', +# 'source_table=NH_F_RATING', +# 'target_table=NH_F_RATING', +# 'access_type=1', +# 'oracle_metadata_table=CORR_RAR.NH_METADATA_INVENTORY', +# 'oracle_igam_table=CT_REF.RAR_SOURCES_IGAM_SENTRY', +# 'sentry_role_environment=production', +# 'target_s3_bucket=s3a://devo-crp-ffppyd8q', +# 'tech_meta_data_fields=tec_ingestion_date String, tec_execution_date String, tec_run_id String', +# 'full_access_entitlement_list=DISC-PC-RAR-R' +# ] + +# args = [ +# 'corporate_store=crp_rar', +# 'source_schema=CORR_RAR', +# 'source_table=NH_ASSET', +# 'target_table=NH_ASSET', +# 'access_type=2a', +# 'oracle_metadata_table=CORR_RAR.NH_METADATA_INVENTORY', +# 'oracle_igam_table=CT_REF.RAR_SOURCES_IGAM_SENTRY', +# 'sentry_role_environment=production', +# 'target_s3_bucket=s3a://devo-crp-ffppyd8q', +# 'tech_meta_data_fields=tec_ingestion_date String, tec_execution_date String, tec_run_id String', +# 'full_access_entitlement_list=DISC-PC-RAR-R' +# ] + + +args = [ + 'corporate_store=crp_mopdb', + 'source_schema=MPEC', + 'source_table=T_MPEC', + 'oracle_metadata_table=CT_MOPDB.MOPDB_METADATA_INVENTORY', + 'oracle_igam_table=CT_MOPDB.MOPDB_SOURCES_IGAM_SENTRY', + 'sentry_role_environment=production', + 'target_s3_bucket=s3a://devo-crp-sbul3ju3/mopdb/db', + 'tech_meta_data_fields=tec_ingestion_date String, tec_execution_date String, tec_run_id String', + 'full_access_entitlement_list=' +] + +rar_options = ro.Options(args) + +tableFields = tbp.loadMetadataTable(rar_options) +tbp.createExternalTables_CRP_RAR(rar_options,tableFields ) +igamRoleDF = tbp.readIGAMRoles(rar_options) + +tbp.accessTypeMapper(rar_options, igamRoleDF) \ No newline at end of file diff --git a/python/mrds_common/.gitignore b/python/mrds_common/.gitignore new file mode 100644 index 0000000..2eef557 --- /dev/null +++ b/python/mrds_common/.gitignore @@ -0,0 +1,6 @@ +__pycache__ +*.log +.venv +.tox +*.egg-info/ +build diff --git a/python/mrds_common/CHANGELOG.md b/python/mrds_common/CHANGELOG.md new file mode 100644 index 0000000..85d3164 --- /dev/null +++ b/python/mrds_common/CHANGELOG.md @@ -0,0 +1,72 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.6.0] - 13-10-2025 + +### Added + +- new type of column xpath_element_id + +## [0.5.0] - 08-10-2025 + +### Added + +- added new mandatory configuration parameter `archive_prefix`. App now archives source file to this location, before deleting it from inbox_prefix location. +- log app version at runtime. + +### Changed + +- improved logging when calling database function CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE +- removed local zip file deletion from version 0.4.0 to accomodate archiving at the end of the processing + + +## [0.4.1] - 03-10-2025 + +### Added + +- `--version` flag to CLI, now shows package version from `mrds.__version__`. ([#179](https://gitlab.sofa.dev/mrds/mrds_elt/-/merge_requests/179)) + +## [0.4.0] - 03-10-2025 + +### Added + +- App versioning! +- Streaming algorithm when reading, filtering and enriching csv files. This drastically improves application performance in regards to RAM usage. +- Unzipping now deletes local source zip file, after data has been extracted. + +## [0.3.1] - 30-09-2025 + +### Fixed + +- fixed small bug related to the new encoding setting + +## [0.3.0] - 29-09-2025 + +### Added + +- new type of config - Application config. +These will be very specific application settings to be overridden in specific cases. Consequently, such configuration will only be optional, because rare usage is expected. First such config is encoding_type + +### Changed + +- removed output of .log files when running the application + +### Fixed + +- small bug when unzipping a file + +### [0.2.0] - 17-09-2025 + +### Added + +- automatic deletion of the source file, and all temporary files created by the app. +- two new cli paramters - --keep-source-file and --keep-tmp-dir flags, to be used to avoid deleting the source file and/or temporary working directory when testing. +- row count output in log files after enrichment. + +### Fixed + +- source and output columns in csv extraction were mistakenly swapped. This is now fixed. diff --git a/python/mrds_common/README.md b/python/mrds_common/README.md new file mode 100644 index 0000000..f7ecc53 --- /dev/null +++ b/python/mrds_common/README.md @@ -0,0 +1,328 @@ +# MRDS APP + +The main purpose of this application is to download XML or CSV files from source, perform some basic ETL and upload them to target. +Below is a simplified workflow of the application. + +## Application workflow + +```mermaid +flowchart LR + subgraph CoreApplication + direction TB + B[Read and validate config file] --> |If valid| C[Download source file] + C[Download source file] --> D[Unzip if file is ZIP] + D[Unzip if file is ZIP] --> E[Validate source file] + E --> |If valid| G[Start task defined in config file] + G --> H[Build output file with selected data from source] + H --> I[Enrich output file with metadata] + I --> J[Upload the output file] + J --> K[Trigger remote function] + K --> L[Check if more tasks are available in config file] + L --> |Yes| G + L --> |No| M[Archive & Delete source file] + M --> N[Finish workflow] + end +A[Trigger app via CLI or Airflow DAG] --> CoreApplication +``` + +## Installation + +Checkout repository and cd to root project directory + +```shell +cd python/mrds_common +``` + +Create new virtual environment using Python >=3.11 + +```shell +python3.11 -m venv .venv +``` + +Activate virtual environment + +```shell +source .venv/bin/activate +``` + +Upgrade pip + +```shell +pip install --upgrade pip +``` + +Install app + +```shell +pip install . +``` + +## Environment variables + +There are two operating system environment variables, which are requred by the application: + +BUCKET_NAMESPACE - OCI namespace where main operating bucket is located (if not found - default value is frcnomajoc7v) + +BUCKET - main operating OCI bucket for downloading and uploading files (if not found - default value is mrds_inbox_poc) + + +## Usage + +The application accepts two required and four optional parameters. + +### Parameters + +| Parameter | Short Flag | Required | Default | Description | +|-------------------------------|------------|----------|---------|----------------------------------------------------------------------------------------------------------------------| +| `--workflow-context` | `-w` | No* | None | JSON string representing the workflow context. Must contain `run_id` and `a_workflow_history_key`. | +| `--generate-workflow-context` | | No* | | Flag type. If provided, app automatically generates and finalizes workflow context. Use this if `--workflow-context` is not provided. | +| `--source-filename` | `-s` | Yes | None | Name of the source file to be looked up in source inbox set in configuration file (`inbox_prefix`). | +| `--config-file` | `-c` | Yes | None | Path to the YAML configuration file. Can be absolute, or relative to current working directory. | +| `--keep-source-file` | | No | | Flag type. If provided, app keeps source file, instead of archiving and deleting it. | +| `--keep-tmp-dir` | | No | | Flag type. If provided, app keeps tmp directory, instead of deleting it. | + +*`--workflow-context` and `--generate-workflow-context` are both optional, however - either one of them MUST be provided for the application to run. + + +### CLI + +```shell +mrds-cli --workflow-context '{"run_id": "0ce35637-302c-4293-8069-3186d5d9a57d", "a_workflow_history_key": 352344}' \ + --source-filename 'CSDB_Debt_Daily.ZIP' \ + --config-file /home/dbt/GEORGI/projects/mrds_elt/airflow/ods/csdb/debt_daily/config/yaml/csdb_debt_daily.yaml +``` + +### Python module + +Import main function from core module and provide needed parameters: + +```python +from mrds.core import main +from mrds.utils.manage_runs import init_workflow, finalise_workflow +from mrds.utils.static_vars import status_success, status_failed + +import datetime +import logging +import sys + +# Configure logging for your needs. This is just a sample +current_time = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +log_filename = f"mrds_{current_time}.log" +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s - %(message)s", + handlers=[ + logging.FileHandler(log_filename), + logging.StreamHandler(sys.stdout), + ], +) + +STATUS_SUCCESS = status_success +STATUS_FAILURE = status_failed + +# Run time parameters + +run_id = "0ce35637-302c-4293-8069-3186d5d9a57d" +a_workflow_history_key = init_workflow(database_name='ODS', workflow_name='w_OU_C2D_UC_DISSEM', workflow_run_id=run_id) + +workflow_context = { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key, +} + +source_filename = "CSDB_Debt_Daily.ZIP" +config_file = "/home/dbt/GEORGI/projects/mrds_elt/airflow/ods/csdb/debt_daily/config/yaml/csdb_debt_daily.yaml" + +main(workflow_context, source_filename, config_file) + +# implement your desired error handling logic and provide correct status to function finalize_workflow + +finalise_workflow(workflow_context["a_workflow_history_key"], STATUS_SUCCESS) + + +``` + +## Configuration + +### Generate workflow context + +Use this if you are using the application in standalone mode. Workflow context will be generated, and then finalized. + +### Source filename + +This is the source file name to be looked up in in source inbox set in the configuration file (`inbox_prefix`). + +### Workflow context + +This is a JSON string (or from the application standpoint view - dictionary) containing run_id and a_workflow_history_key values. + +```JSON +workflow_context = { + "run_id": "0ce35637-302c-4293-8069-3186d5d9a57d", + "a_workflow_history_key": 352344, +} +``` + +run_id - this represent orchestration ID. Can be any string ID of your choice, for example Airflow DAG ID. +a_workflow_history_key - can be generated via mrds.utils.manage_runs.init_workflow() function. + +If you provide workflow context by yourself, you need to take care of finalizing it too. + +### Config file + +This is the main place which we can control the application. + +At the top, are the Application configurations. These apply to all tasks. These are all optional and are used to override some specific runtime application settings. + +```yaml +# System configurations + +encoding_type: cp1252 # Overrides default encoding type (utf-8) of the app. This encoding is used when reading source csv/xml files and when writing the output csv files of the app. For codec naming, follow guidelines here - https://docs.python.org/3/library/codecs.html#standard-encodings +``` + +After that, are the global configurations. These apply to all tasks: + +```yaml +# Global configurations +tmpdir: /tmp # root temporary directory to create runtime temporary directory, download source file and perform operations on it, before upload it to target +inbox_prefix: INBOX/C2D/UC_DISSEM # prefix for the inbox containing the source file +archive_prefix: ARCHIVE/C2D/UC_DISSEM # prefix for the archive bucket +workflow_name: w_OU_C2D_UC_DISSEM # name of the particular workflow +validation_schema_path: 'xsd/UseOfCollateralMessage.xsd' # relative path (to runtime location) to schema used to validate XML or CSV file +file_type: xml # file type of the expected source file - either CSV or XML +``` + +Following, there is a list of tasks to be performed on the source file. +We can have multiple tasks per file, meaning - we can generate more than one output file, from one source file. +Further, one of the key configuration parameters per task is "output_columns". There we define columns of the final output file. +There are several types of columns: + +xpath - this type of column is used when source file is XML. It is a standart xpath expression, pointing to path in the xml. + +xpath_element_id - this type of column is used when we need to id a particular xml element. Used to create foreign keys between two separate tasks. It is a standart xpath expression, pointing to path in the xml. + +csv_header - this type of column is used when source file is CSV. It just points to the corresponding csv header in the source file. + +a_key - generates key unique per row. + +workflow_key - generates key unique per run of the application + +static - allows the user to define column with static value + +The application respects the order of the output columns in the configuration file, when generating the output file. +Data and columns from the source file, not included in the configuration file, will not be present in the final output file. + +Example of xml task configuration: + +```yaml +# List of tasks +tasks: + - task_name: ou_lm_standing_facilities_header_create_file # name of the particular task + ods_prefix: INBOX/LM/STANDING_FACILITIES/STANDING_FACILITIES_HEADER # prefix for the upload location + output_table: standing_facilities_headers # table in Oracle + namespaces: + ns2: 'http://escb.ecb.int/sf' # XML namespace + output_columns: # Columns in the output file, order will be respected. + - type: 'a_key' # A_KEY type of column + column_header: 'A_KEY' # naming of the column in the output file + - type: 'workflow_key' # WORKFLOW_KEY type of column + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' # xpath type of column + value: '//ns2:header/ns2:version' + column_header: 'REV_NUMBER' + is_key: 'N' # value is transposed across the rows - YES/NO. Used when there is only single value in source XML + - type: 'xpath' + value: '//ns2:header/ns2:referenceDate' + column_header: 'REF_DATE' + is_key: 'N' + - type: 'static' + value: '' + column_header: 'FREE_TEXT' + + - task_name: ou_lm_standing_facilities_create_file + ods_prefix: INBOX/LM/STANDING_FACILITIES/STANDING_FACILITIES + output_table: standing_facilities + namespaces: + ns2: 'http://escb.ecb.int/sf' + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_SFH_FK' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'xpath' + value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:country' + column_header: 'COUNTRY' + - type: 'static' + value: '' + column_header: 'COMMENT_' + +``` + +Example of CSV task configuration: + +```yaml +tasks: + - task_name: ODS_CSDB_DEBT_DAILY_process_csv + ods_prefix: ODS/CSDB/DEBT_DAILY + output_table: DEBT_DAILY + output_columns: + - type: 'a_key' + column_header: 'A_KEY' + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' # csv_header type of column + value: 'Date last modified' # naming of the column in the SOURCE file + column_header: 'Date last modified' # naming of the column in the OUTPUT file + - type: 'csv_header' + value: 'Extraction date' + column_header: 'Extraction date' + - type: 'csv_header' + value: 'ISIN code' + column_header: 'ISIN code' +``` + +## Development + +### Installing requirements + +Install app + dev requirements. For easier workflow, you can install in editable mode + +``` +pip install -e .[dev] +``` + +In editable mode, instead of copying the package files to the site-packages directory, pip creates a special link that points to the source code directory. This means any changes you make to your source code will be immediately available without needing to reinstall the package. + +### Code formattting + +Run black to reformat the code before pushing changes. + +Following will reformat all files recursively from current dir. + +``` +black . +``` + +Following will only check and report what needs to be formatted, recursively from current dir. + +``` +black --check --diff . +``` + +### Tests + +Run tests with + +``` +pytest . +``` + +### Tox automation + +Tox automates runs of black checks and tests + +``` +tox . +``` diff --git a/python/mrds_common/mrds/__init__.py b/python/mrds_common/mrds/__init__.py new file mode 100644 index 0000000..906d362 --- /dev/null +++ b/python/mrds_common/mrds/__init__.py @@ -0,0 +1 @@ +__version__ = "0.6.0" diff --git a/python/mrds_common/mrds/cli.py b/python/mrds_common/mrds/cli.py new file mode 100644 index 0000000..c79447d --- /dev/null +++ b/python/mrds_common/mrds/cli.py @@ -0,0 +1,117 @@ +import click +import json +import logging +import sys + +from mrds import __version__ +from mrds.core import main + + +@click.command() +@click.version_option(version=__version__, prog_name="mrds") +@click.option( + "--workflow-context", + "-w", + required=False, + help="Workflow context to be used by the application. This is required unless --generate-workflow-context is provided.", +) +@click.option( + "--source-filename", + "-s", + required=True, + help="Source filename to be processed.", +) +@click.option( + "--config-file", + "-c", + type=click.Path(exists=True), + required=True, + help="Path to the YAML configuration file.", +) +@click.option( + "--generate-workflow-context", + is_flag=True, + default=False, + help="Generate a workflow context automatically. If this is set, --workflow-context is not required.", +) +@click.option( + "--keep-source-file", + is_flag=True, + default=False, + help="Keep source file, instead of deleting it.", +) +@click.option( + "--keep-tmp-dir", + is_flag=True, + default=False, + help="Keep tmp directory, instead of deleting it.", +) +def cli_main( + workflow_context, + source_filename, + config_file, + generate_workflow_context, + keep_source_file, + keep_tmp_dir, +): + + # Configure logging + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s - %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + ], + ) + + # Handle conflicting options + if workflow_context and generate_workflow_context: + raise click.UsageError( + "You cannot use both --workflow-context and --generate-workflow-context at the same time. " + "Please provide only one." + ) + + # Enforce that either --workflow-context or --generate-workflow-context must be provided + if not workflow_context and not generate_workflow_context: + raise click.UsageError( + "You must provide --workflow-context or use --generate-workflow-context flag." + ) + + # Parse and validate the workflow_context if provided + if workflow_context: + try: + workflow_context = json.loads(workflow_context) + except json.JSONDecodeError as e: + raise click.UsageError(f"Invalid JSON for --workflow-context: {e}") + + # Validate that the workflow_context matches the expected structure + if ( + not isinstance(workflow_context, dict) + or "run_id" not in workflow_context + or "a_workflow_history_key" not in workflow_context + ): + raise click.UsageError( + "Invalid workflow context structure. It must be a JSON object with 'run_id' and 'a_workflow_history_key'." + ) + + # Call the core processing function + main( + workflow_context, + source_filename, + config_file, + generate_workflow_context, + keep_source_file, + keep_tmp_dir, + ) + + +if __name__ == "__main__": + try: + cli_main() + sys.exit(0) + except click.UsageError as e: + logging.error(f"Usage error: {e}") + sys.exit(2) + except Exception as e: + logging.error(f"Unexpected error: {e}") + sys.exit(1) diff --git a/python/mrds_common/mrds/core.py b/python/mrds_common/mrds/core.py new file mode 100644 index 0000000..cae3062 --- /dev/null +++ b/python/mrds_common/mrds/core.py @@ -0,0 +1,366 @@ +import os +import uuid +import logging +import yaml +import zipfile +import tempfile +from dataclasses import dataclass, field + +from mrds import __version__ + +from mrds.processors import get_file_processor +from mrds.utils import ( + manage_runs, + objectstore, + static_vars, + xml_utils, +) + + +# environment variables +MRDS_ENV = os.getenv("MRDS_ENV", "poc") +BUCKET = os.getenv("INBOX_BUCKET", "mrds_inbox_poc") +BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE", "frcnomajoc7v") + + +# Static configuration variables +WORKFLOW_TYPE = "ODS" +ENCODING_TYPE = "utf-8" + +CONFIG_REQUIRED_KEYS = [ + "tmpdir", + "inbox_prefix", + "archive_prefix", + "workflow_name", + "validation_schema_path", + "tasks", + "file_type", +] + +TASK_REQUIRED_KEYS = [ + "task_name", + "ods_prefix", + "output_table", + "output_columns", +] + +STATUS_SUCCESS = static_vars.status_success +STATUS_FAILURE = static_vars.status_failed + + +@dataclass +class GlobalConfig: + tmpdir: str + inbox_prefix: str + archive_prefix: str + workflow_name: str + source_filename: str + validation_schema_path: str + bucket: str + bucket_namespace: str + file_type: str + encoding_type: str + + def __post_init__(self): + self.original_source_filename = self.source_filename # keep this in case we have a zip file to archive + + @property + def source_filepath(self) -> str: + return os.path.join(self.tmpdir, self.source_filename) + + @property + def original_source_filepath(self) -> str: + return os.path.join(self.tmpdir, self.original_source_filename) + + +@dataclass +class TaskConfig: + task_name: str + ods_prefix: str + output_table: str + namespaces: dict + output_columns: list + + +def initialize_config(source_filename, config_file_path): + logging.info(f"Source filename is set to: {source_filename}") + logging.info(f"Loading configuration from {config_file_path}") + # Ensure the file exists + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Configuration file {config_file_path} not found.") + + # Load the configuration + with open(config_file_path, "r") as f: + config_data = yaml.safe_load(f) + logging.debug(f"Configuration data: {config_data}") + + missing_keys = [key for key in CONFIG_REQUIRED_KEYS if key not in config_data] + if missing_keys: + raise ValueError(f"Missing required keys in configuration: {missing_keys}") + + # Create GlobalConfig instance + global_config = GlobalConfig( + tmpdir=config_data["tmpdir"], + inbox_prefix=config_data["inbox_prefix"], + archive_prefix=config_data["archive_prefix"], + workflow_name=config_data["workflow_name"], + source_filename=source_filename, + validation_schema_path=config_data["validation_schema_path"], + bucket=BUCKET, + bucket_namespace=BUCKET_NAMESPACE, + file_type=config_data["file_type"], + encoding_type=config_data.get("encoding_type", ENCODING_TYPE), + ) + + # Create list of TaskConfig instances + tasks_data = config_data["tasks"] + tasks = [] + for task_data in tasks_data: + # Validate required keys in task_data + missing_task_keys = [key for key in TASK_REQUIRED_KEYS if key not in task_data] + if missing_task_keys: + raise ValueError( + f"Missing required keys in task configuration: {missing_task_keys}" + ) + + task = TaskConfig( + task_name=task_data["task_name"], + ods_prefix=task_data["ods_prefix"], + output_table=task_data["output_table"], + namespaces=task_data.get("namespaces", {}), + output_columns=task_data["output_columns"], + ) + tasks.append(task) + + return global_config, tasks + + +def initialize_workflow(global_config): + + run_id = str(uuid.uuid4()) + + logging.info(f"Initializing workflow '{global_config.workflow_name}'") + a_workflow_history_key = manage_runs.init_workflow( + WORKFLOW_TYPE, global_config.workflow_name, run_id + ) + + return { + "run_id": run_id, + "a_workflow_history_key": a_workflow_history_key, + } + + +def download_source_file(client, global_config): + logging.info( + f"Downloading source file '{global_config.source_filename}' " + f"from '{global_config.bucket}/{global_config.inbox_prefix}'" + ) + objectstore.download_file( + client, + global_config.bucket_namespace, + global_config.bucket, + global_config.inbox_prefix, + global_config.source_filename, + global_config.source_filepath, + ) + logging.info(f"Source file downloaded to '{global_config.source_filepath}'") + + +def delete_source_file(client, global_config): + + logging.info( + f"Deleting source file '{global_config.bucket}/{global_config.inbox_prefix}/{global_config.original_source_filename}'" + ) + objectstore.delete_file( + client, + global_config.original_source_filename, + global_config.bucket_namespace, + global_config.bucket, + global_config.inbox_prefix, + ) + logging.info( + f"Deleted source file '{global_config.bucket}/{global_config.inbox_prefix}/{global_config.original_source_filename}'" + ) + + +def archive_source_file(client, global_config): + + logging.info( + f"Archiving source file to '{global_config.bucket}/{global_config.archive_prefix}/{global_config.original_source_filename}'" + ) + objectstore.upload_file( + client, + global_config.original_source_filepath, + global_config.bucket_namespace, + global_config.bucket, + global_config.archive_prefix, + global_config.original_source_filename, + ) + logging.info( + f"Source file archived to '{global_config.bucket}/{global_config.archive_prefix}/{global_config.original_source_filename}'" + ) + + +def unzip_source_file_if_needed(global_config): + source_filepath = global_config.source_filepath + + # If it's not a zip, nothing to do + if not zipfile.is_zipfile(source_filepath): + logging.info(f"File '{source_filepath}' is not a ZIP file.") + return True + + logging.info(f"File '{source_filepath}' is a ZIP file. Unzipping...") + + extract_dir = os.path.dirname(source_filepath) + + try: + with zipfile.ZipFile(source_filepath, "r") as zip_ref: + extracted_files = zip_ref.namelist() + + if len(extracted_files) != 1: + logging.error( + f"Expected one file in the ZIP, but found {len(extracted_files)} files." + ) + return False + + # Extract everything + zip_ref.extractall(extract_dir) + + except Exception as e: + logging.error(f"Error while extracting '{source_filepath}': {e}") + return False + + # Update the global_config to point to the extracted file + extracted_filename = extracted_files[0] + global_config.source_filename = extracted_filename + + logging.info( + f"Extracted '{extracted_filename}' to '{extract_dir}'. " + f"Updated source_filepath to '{global_config.source_filepath}'." + ) + + return True + + +def validate_source_file(global_config): + file_type = global_config.file_type.lower() + + if file_type == "xml": + xml_is_valid, xml_validation_message = xml_utils.validate_xml( + global_config.source_filepath, global_config.validation_schema_path + ) + if not xml_is_valid: + raise ValueError(f"XML validation failed: {xml_validation_message}") + logging.info(xml_validation_message) + + elif file_type == "csv": + # TODO: add CSV validation here + pass + + else: + raise ValueError(f"Unsupported file type: {file_type}") + + return True + + +def process_tasks(tasks, global_config, workflow_context, client): + + # get appropriate task processor + processor_class = get_file_processor(global_config) + + for task_conf in tasks: + + logging.info(f"Starting task '{task_conf.task_name}'") + file_processor = processor_class( + global_config, task_conf, client, workflow_context + ) + file_processor.process() + + +def finalize_workflow(workflow_context, success=True): + status = STATUS_SUCCESS if success else STATUS_FAILURE + manage_runs.finalise_workflow(workflow_context["a_workflow_history_key"], status) + if success: + logging.info("Workflow completed successfully") + else: + logging.error("Workflow failed") + + +def main( + workflow_context: dict, + source_filename: str, + config_file_path: str, + generate_workflow_context=False, + keep_source_file=False, + keep_tmp_dir=False, +): + + logging.info(f"Initializing mrds app, version {__version__}") + + tmpdir_manager = None + + try: + # get configs + global_config, tasks = initialize_config(source_filename, config_file_path) + + # Handle temporary dirs + if keep_tmp_dir: + tmpdir = tempfile.mkdtemp( + prefix="mrds_", dir=global_config.tmpdir + ) # dir is created and never deleted + logging.info( + f"Created temporary working directory (not auto-deleted): {tmpdir}" + ) + else: + tmpdir_manager = tempfile.TemporaryDirectory( + prefix="mrds_", dir=global_config.tmpdir + ) + tmpdir = tmpdir_manager.name + logging.info( + f"Created temporary working directory (auto-deleted): {tmpdir}" + ) + + # override tmpdir with newly created tmpdir + global_config.tmpdir = tmpdir + + client = objectstore.get_client() + + # Handle workflow_context generation if required + if generate_workflow_context: + logging.info("Generating workflow context automatically.") + workflow_context = initialize_workflow(global_config) + logging.info(f"Generated workflow context: {workflow_context}") + else: + logging.info(f"Using provided workflow context: {workflow_context}") + + download_source_file(client, global_config) + unzip_source_file_if_needed(global_config) + validate_source_file(global_config) + process_tasks(tasks, global_config, workflow_context, client) + + if generate_workflow_context: + finalize_workflow(workflow_context) + + if not keep_source_file: + archive_source_file(client, global_config) + delete_source_file(client, global_config) + + except Exception as e: + logging.error(f"Critical error: {str(e)}") + + # Finalize workflow with failure if needed + if generate_workflow_context and "workflow_context" in locals(): + finalize_workflow(workflow_context, success=False) + + raise RuntimeError(f"Workflow failed due to: {e}") + + finally: + # Always attempt to remove tmpdir if created a TemporaryDirectory manager + if tmpdir_manager and not keep_tmp_dir: + try: + tmpdir_manager.cleanup() + logging.info(f"Deleted temporary working directory {tmpdir}") + except Exception: + logging.exception( + f"Failed to delete up temporary working directory {tmpdir}" + ) diff --git a/python/mrds_common/mrds/docs/rqsd_sample.yaml b/python/mrds_common/mrds/docs/rqsd_sample.yaml new file mode 100644 index 0000000..6d7d750 --- /dev/null +++ b/python/mrds_common/mrds/docs/rqsd_sample.yaml @@ -0,0 +1,186 @@ +# static configs +tmpdir: /tmp +inbox_prefix: INBOX/RQSD/RQSD_PROCESS +workflow_name: w_ODS_RQSD_PROCESS_DEVO +validation_schema_path: None +file_type: csv + +# task configs +tasks: + - task_name: m_ODS_RQSD_OBSERVATIONS_PARSE + ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_OBSERVATIONS + output_table: RQSD_OBSERVATIONS + output_columns: + - type: 'workflow_key' + column_header: 'A_WORKFLOW_HISTORY_KEY' + - type: 'csv_header' + value: 'datacollectioncode' + column_header: 'datacollectioncode' + - type: 'csv_header' + value: 'datacollectionname' + column_header: 'datacollectionname' + - type: 'csv_header' + value: 'datacollectionowner' + column_header: 'datacollectionowner' + - type: 'csv_header' + value: 'reportingcyclename' + column_header: 'reportingcyclename' + - type: 'csv_header' + value: 'reportingcyclestatus' + column_header: 'reportingcyclestatus' + - type: 'csv_header' + value: 'modulecode' + column_header: 'modulecode' + - type: 'csv_header' + value: 'modulename' + column_header: 'modulename' + - type: 'csv_header' + value: 'moduleversionnumber' + column_header: 'moduleversionnumber' + - type: 'csv_header' + value: 'reportingentitycollectionuniqueid' + column_header: 'reportingentitycollectionuniqueid' + - type: 'csv_header' + value: 'entityattributereportingcode' + column_header: 'entityattributereportingcode' + - type: 'csv_header' + value: 'reportingentityname' + column_header: 'reportingentityname' + - type: 'csv_header' + value: 'reportingentityentitytype' + column_header: 'reportingentityentitytype' + - type: 'csv_header' + value: 'entityattributecountry' + column_header: 'entityattributecountry' + - type: 'csv_header' + value: 'entitygroupentityname' + column_header: 'entitygroupentityname' + - type: 'csv_header' + value: 'obligationmodulereferencedate' + column_header: 'obligationmodulereferencedate' + - type: 'csv_header' + value: 'obligationmoduleremittancedate' + column_header: 'obligationmoduleremittancedate' + - type: 'csv_header' + value: 'receivedfilereceiveddate' + column_header: 'receivedfilereceiveddate' + - type: 'csv_header' + value: 'obligationmoduleexpected' + column_header: 'obligationmoduleexpected' + - type: 'csv_header' + value: 'receivedfileversionnumber' + column_header: 'receivedfileversionnumber' + - type: 'csv_header' + value: 'revalidationversionnumber' + column_header: 'revalidationversionnumber' + - type: 'csv_header' + value: 'revalidationdate' + column_header: 'revalidationdate' + - type: 'csv_header' + value: 'receivedfilesystemfilename' + column_header: 'receivedfilesystemfilename' + - type: 'csv_header' + value: 'obligationstatusstatus' + column_header: 'obligationstatusstatus' + - type: 'csv_header' + value: 'filestatussetsubmissionstatus' + column_header: 'filestatussetsubmissionstatus' + - type: 'csv_header' + value: 'filestatussetvalidationstatus' + column_header: 'filestatussetvalidationstatus' + - type: 'csv_header' + value: 'filestatussetexternalvalidationstatus' + column_header: 'filestatussetexternalvalidationstatus' + - type: 'csv_header' + value: 'numberoferrors' + column_header: 'numberoferrors' + - type: 'csv_header' + value: 'numberofwarnings' + column_header: 'numberofwarnings' + - type: 'csv_header' + value: 'delayindays' + column_header: 'delayindays' + - type: 'csv_header' + value: 'failedattempts' + column_header: 'failedattempts' + - type: 'csv_header' + value: 'observationvalue' + column_header: 'observationvalue' + - type: 'csv_header' + value: 'observationtextvalue' + column_header: 'observationtextvalue' + - type: 'csv_header' + value: 'observationdatevalue' + column_header: 'observationdatevalue' + - type: 'csv_header' + value: 'datapointsetdatapointidentifier' + column_header: 'datapointsetdatapointidentifier' + - type: 'csv_header' + value: 'datapointsetlabel' + column_header: 'datapointsetlabel' + - type: 'csv_header' + value: 'obsrvdescdatatype' + column_header: 'obsrvdescdatatype' + - type: 'csv_header' + value: 'ordinatecode' + column_header: 'ordinatecode' + - type: 'csv_header' + value: 'ordinateposition' + column_header: 'ordinateposition' + - type: 'csv_header' + value: 'tablename' + column_header: 'tablename' + - type: 'csv_header' + value: 'isstock' + column_header: 'isstock' + - type: 'csv_header' + value: 'scale' + column_header: 'scale' + - type: 'csv_header' + value: 'currency' + column_header: 'currency' + - type: 'csv_header' + value: 'numbertype' + column_header: 'numbertype' + - type: 'csv_header' + value: 'ismandatory' + column_header: 'ismandatory' + - type: 'csv_header' + value: 'decimalplaces' + column_header: 'decimalplaces' + - type: 'csv_header' + value: 'serieskey' + column_header: 'serieskey' + - type: 'csv_header' + value: 'tec_source_system' + column_header: 'tec_source_system' + - type: 'csv_header' + value: 'tec_dataset' + column_header: 'tec_dataset' + - type: 'csv_header' + value: 'tec_surrogate_key' + column_header: 'tec_surrogate_key' + - type: 'csv_header' + value: 'tec_crc' + column_header: 'tec_crc' + - type: 'csv_header' + value: 'tec_ingestion_date' + column_header: 'tec_ingestion_date' + - type: 'csv_header' + value: 'tec_version_id' + column_header: 'tec_version_id' + - type: 'csv_header' + value: 'tec_execution_date' + column_header: 'tec_execution_date' + - type: 'csv_header' + value: 'tec_run_id' + column_header: 'tec_run_id' + - type: 'static' + value: 'test test' + column_header: 'BLABLA' + - type: 'a_key' + column_header: 'A_KEY' + - type: 'csv_header' + value: 'tec_business_date' + column_header: 'tec_business_dateTest!' + diff --git a/python/mrds_common/mrds/docs/upload.py b/python/mrds_common/mrds/docs/upload.py new file mode 100644 index 0000000..817c88c --- /dev/null +++ b/python/mrds_common/mrds/docs/upload.py @@ -0,0 +1,50 @@ +# file uploader +import os +import sys +import logging +from mrds.utils import objectstore + +BUCKET = os.getenv("INBOX_BUCKET", "mrds_inbox_poc") +BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE", "frcnomajoc7v") + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s - %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + ], +) + + +source_filepath = '/home/dbt/tmp/mrds_4twsw_ib/20250630_Pre-Production_DV_P2_DBT_I4.zip' +source_filename = '20250630_Pre-Production_DV_P2_DBT_I4.zip' +target_prefix = 'INBOX/CSDB/STC_CentralizedSecuritiesDissemination_ECB' + +def upload_file(): + + client = objectstore.get_client() + + logging.info( + f"uploading source file to '{BUCKET}/{target_prefix}/{source_filename}'" + ) + objectstore.upload_file( + client, + source_filepath, + BUCKET_NAMESPACE, + BUCKET, + target_prefix, + source_filename, + ) + logging.info( + f"Source file uploaded to '{BUCKET}/{target_prefix}/{source_filename}'" + ) + + +if __name__ == "__main__": + try: + upload_file() + sys.exit(0) + except Exception as e: + logging.error(f"Unexpected error: {e}") + sys.exit(1) diff --git a/python/mrds_common/mrds/processors/__init__.py b/python/mrds_common/mrds/processors/__init__.py new file mode 100644 index 0000000..5e1001b --- /dev/null +++ b/python/mrds_common/mrds/processors/__init__.py @@ -0,0 +1,15 @@ +from .xml_processor import XMLTaskProcessor +from .csv_processor import CSVTaskProcessor + + +def get_file_processor(global_config): + """ + Factory function to get the appropriate file processor class based on the file type in the global configuration. + """ + file_type = global_config.file_type.lower() + if file_type == "xml": + return XMLTaskProcessor + elif file_type == "csv": + return CSVTaskProcessor + else: + raise ValueError(f"Unsupported file type: {file_type}") diff --git a/python/mrds_common/mrds/processors/base.py b/python/mrds_common/mrds/processors/base.py new file mode 100644 index 0000000..610ebc5 --- /dev/null +++ b/python/mrds_common/mrds/processors/base.py @@ -0,0 +1,211 @@ +import logging +import os +import csv +from abc import ABC, abstractmethod + +from mrds.utils.utils import parse_output_columns + +from mrds.utils import ( + manage_files, + manage_runs, + objectstore, + static_vars, +) + + +OUTPUT_FILENAME_TEMPLATE = "{output_table}-{task_history_key}.csv" +STATUS_SUCCESS = static_vars.status_success # duplicated needs to be moved #TODO + + +class TaskProcessor(ABC): + def __init__(self, global_config, task_conf, client, workflow_context): + self.global_config = global_config + self.task_conf = task_conf + self.client = client + self.workflow_context = workflow_context + self._init_common() + self._post_init() + + def _init_common(self): + # Initialize task + self.a_task_history_key = manage_runs.init_task( + self.task_conf.task_name, + self.workflow_context["run_id"], + self.workflow_context["a_workflow_history_key"], + ) + logging.info(f"Task initialized with history key: {self.a_task_history_key}") + + # Define output file paths + self.output_filename = OUTPUT_FILENAME_TEMPLATE.format( + output_table=self.task_conf.output_table, + task_history_key=self.a_task_history_key, + ) + self.output_filepath = os.path.join( + self.global_config.tmpdir, self.output_filename + ) + + # Parse the output_columns + ( + self.xpath_entries, + self.csv_entries, + self.static_entries, + self.a_key_entries, + self.workflow_key_entries, + self.xml_position_entries, + self.column_order, + ) = parse_output_columns(self.task_conf.output_columns) + + def _post_init(self): + """Optional hook for classes to override""" + pass + + @abstractmethod + def _extract(self): + """Non-optional hook for classes to override""" + pass + + def _enrich(self): + """ + Stream-based enrich: read one row at a time, append static/A-key/workflow-key, + reorder columns, and write out immediately. + """ + + TASK_HISTORY_MULTIPLIER = 1_000_000_000 + + logging.info(f"Enriching CSV file at '{self.output_filepath}'") + + temp_output = self.output_filepath + ".tmp" + encoding = self.global_config.encoding_type + + with open(self.output_filepath, newline="", encoding=encoding) as inf, open( + temp_output, newline="", encoding=encoding, mode="w" + ) as outf: + + reader = csv.reader(inf) + writer = csv.writer(outf, quoting=csv.QUOTE_ALL) + + # Read the original header + original_headers = next(reader) + + # Compute the full set of headers + headers = list(original_headers) + + # Add static column headers if missing + for col_name, _ in self.static_entries: + if col_name not in headers: + headers.append(col_name) + + # Add A-key column headers if missing + for col_name in self.a_key_entries: + if col_name not in headers: + headers.append(col_name) + + # Add workflow key column headers if missing + for col_name in self.workflow_key_entries: + if col_name not in headers: + headers.append(col_name) + + # Rearrange headers to desired ordr + header_to_index = {h: i for i, h in enumerate(headers)} + out_indices = [ + header_to_index[h] for h in self.column_order if h in header_to_index + ] + out_headers = [headers[i] for i in out_indices] + + # Write the new header + writer.writerow(out_headers) + + # Stream each row, enrich in-place, reorder, and write + row_count = 0 + base_task_history = int(self.a_task_history_key) * TASK_HISTORY_MULTIPLIER + + for i, in_row in enumerate(reader, start=1): + # Build a working list that matches `headers` order. + # Start by copying the existing columns (or '' if missing) + work_row = [None] * len(headers) + for j, h in enumerate(original_headers): + idx = header_to_index[h] + work_row[idx] = in_row[j] + + # Fill static columns + for col_name, value in self.static_entries: + idx = header_to_index[col_name] + work_row[idx] = value + + # Fill A-key columns + for col_name in self.a_key_entries: + idx = header_to_index[col_name] + a_key_value = base_task_history + i + work_row[idx] = str(a_key_value) + + # Fill workflow key columns + wf_val = self.workflow_context["a_workflow_history_key"] + for col_name in self.workflow_key_entries: + idx = header_to_index[col_name] + work_row[idx] = wf_val + + # Reorder to output order and write + out_row = [work_row[j] for j in out_indices] + writer.writerow(out_row) + row_count += 1 + + # Atomically replace + os.replace(temp_output, self.output_filepath) + logging.info( + f"CSV file enriched at '{self.output_filepath}', {row_count} rows generated" + ) + + def _upload(self): + # Upload CSV to object store + logging.info( + f"Uploading CSV file to '{self.global_config.bucket}/{self.task_conf.ods_prefix}/{self.output_filename}'" + ) + objectstore.upload_file( + self.client, + self.output_filepath, + self.global_config.bucket_namespace, + self.global_config.bucket, + self.task_conf.ods_prefix, + self.output_filename, + ) + logging.info( + f"CSV file uploaded to '{self.global_config.bucket}/{self.task_conf.ods_prefix}/{self.output_filename}'" + ) + + def _process_remote(self): + # Process the source file + logging.info(f"Processing source file '{self.output_filename}' with CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE database function.") + try: + manage_files.process_source_file( + self.task_conf.ods_prefix, self.output_filename + ) + except Exception as e: + logging.error( + f"Processing source file '{self.output_filename}' failed. Cleaning up..." + ) + objectstore.delete_file( + self.client, + self.output_filename, + self.global_config.bucket_namespace, + self.global_config.bucket, + self.task_conf.ods_prefix, + ) + logging.error( + f"CSV file '{self.global_config.bucket}/{self.task_conf.ods_prefix}/{self.output_filename}' deleted." + ) + raise + else: + logging.info(f"Source file '{self.output_filename}' processed") + + def _finalize(self): + # Finalize task + manage_runs.finalise_task(self.a_task_history_key, STATUS_SUCCESS) + logging.info(f"Task '{self.task_conf.task_name}' completed successfully") + + def process(self): + # main processor function + self._extract() + self._enrich() + self._upload() + self._process_remote() + self._finalize() diff --git a/python/mrds_common/mrds/processors/csv_processor.py b/python/mrds_common/mrds/processors/csv_processor.py new file mode 100644 index 0000000..002c300 --- /dev/null +++ b/python/mrds_common/mrds/processors/csv_processor.py @@ -0,0 +1,52 @@ +import logging +import csv +import os +from .base import TaskProcessor + + +class CSVTaskProcessor(TaskProcessor): + + def _extract(self): + input_path = self.global_config.source_filepath + output_path = self.output_filepath + encoding = self.global_config.encoding_type + + logging.info(f"Reading source CSV file at '{input_path}'") + + # Open both input & output at once for streaming row-by-row + temp_output = output_path + ".tmp" + with open(input_path, newline="", encoding=encoding) as inf, open( + temp_output, newline="", encoding=encoding, mode="w" + ) as outf: + + reader = csv.reader(inf) + writer = csv.writer(outf, quoting=csv.QUOTE_ALL) + + # Read and parse the header + headers = next(reader) + + # Build the list of headers to keep + their new names + headers_to_keep = [old for _, old in self.csv_entries] + headers_rename = [new for new, _ in self.csv_entries] + + # Check if all specified headers exist in the input file + missing = [h for h in headers_to_keep if h not in headers] + if missing: + raise ValueError( + f"The following headers are not in the input CSV: {missing}" + ) + + # Determine the indices of the headers to keep + indices = [headers.index(old) for old in headers_to_keep] + + # Write the renamed header + writer.writerow(headers_rename) + + # Stream through every data row and write out the filtered columns + for row in reader: + filtered = [row[i] for i in indices] + writer.writerow(filtered) + + # Atomically replace the old file + os.replace(temp_output, output_path) + logging.info(f"Core data written to CSV file at '{output_path}'") diff --git a/python/mrds_common/mrds/processors/xml_processor.py b/python/mrds_common/mrds/processors/xml_processor.py new file mode 100644 index 0000000..2e54eea --- /dev/null +++ b/python/mrds_common/mrds/processors/xml_processor.py @@ -0,0 +1,30 @@ +import logging + +from .base import TaskProcessor + +from mrds.utils import ( + xml_utils, + csv_utils, +) + + +class XMLTaskProcessor(TaskProcessor): + + def _extract(self): + # Extract data from XML + csv_data = xml_utils.extract_data( + self.global_config.source_filepath, + self.xpath_entries, + self.xml_position_entries, + self.task_conf.namespaces, + self.workflow_context, + self.global_config.encoding_type, + ) + logging.info(f"CSV data extracted for task '{self.task_conf.task_name}'") + + # Generate CSV + logging.info(f"Writing core data to CSV file at '{self.output_filepath}'") + csv_utils.write_data_to_csv_file( + self.output_filepath, csv_data, self.global_config.encoding_type + ) + logging.info(f"Core data written to CSV file at '{self.output_filepath}'") diff --git a/python/mrds_common/mrds/utils/__init__.py b/python/mrds_common/mrds/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/mrds_common/mrds/utils/csv_utils.py b/python/mrds_common/mrds/utils/csv_utils.py new file mode 100644 index 0000000..af68ede --- /dev/null +++ b/python/mrds_common/mrds/utils/csv_utils.py @@ -0,0 +1,69 @@ +import csv +import os + +TASK_HISTORY_MULTIPLIER = 1_000_000_000 + + +def read_csv_file(csv_filepath, encoding_type="utf-8"): + with open(csv_filepath, "r", newline="", encoding=encoding_type) as csvfile: + reader = list(csv.reader(csvfile)) + headers = reader[0] + data_rows = reader[1:] + return headers, data_rows + + +def write_data_to_csv_file(csv_filepath, data, encoding_type="utf-8"): + temp_csv_filepath = csv_filepath + ".tmp" + with open(temp_csv_filepath, "w", newline="", encoding=encoding_type) as csvfile: + writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL) + writer.writerow(data["headers"]) + writer.writerows(data["rows"]) + os.replace(temp_csv_filepath, csv_filepath) + + +def add_static_columns(data_rows, headers, static_entries): + for column_header, value in static_entries: + if column_header not in headers: + headers.append(column_header) + for row in data_rows: + row.append(value) + else: + idx = headers.index(column_header) + for row in data_rows: + row[idx] = value + + +def add_a_key_columns(data_rows, headers, a_key_entries, task_history_key): + for column_header in a_key_entries: + if column_header not in headers: + headers.append(column_header) + for i, row in enumerate(data_rows, start=1): + a_key_value = int(task_history_key) * TASK_HISTORY_MULTIPLIER + i + row.append(str(a_key_value)) + else: + idx = headers.index(column_header) + for i, row in enumerate(data_rows, start=1): + a_key_value = int(task_history_key) * TASK_HISTORY_MULTIPLIER + i + row[idx] = str(a_key_value) + + +def add_workflow_key_columns(data_rows, headers, workflow_key_entries, workflow_key): + for column_header in workflow_key_entries: + if column_header not in headers: + headers.append(column_header) + for row in data_rows: + row.append(workflow_key) + else: + idx = headers.index(column_header) + for row in data_rows: + row[idx] = workflow_key + + +def rearrange_columns(headers, data_rows, column_order): + header_to_index = {header: idx for idx, header in enumerate(headers)} + new_indices = [ + header_to_index[header] for header in column_order if header in header_to_index + ] + headers = [headers[idx] for idx in new_indices] + data_rows = [[row[idx] for idx in new_indices] for row in data_rows] + return headers, data_rows diff --git a/python/mrds_common/mrds/utils/manage_files.py b/python/mrds_common/mrds/utils/manage_files.py new file mode 100644 index 0000000..f4200b6 --- /dev/null +++ b/python/mrds_common/mrds/utils/manage_files.py @@ -0,0 +1,177 @@ +from . import oraconn +from . import sql_statements +from . import utils + +# Get the next load id from the sequence + +# +# Workflows +# + + +def process_source_file_from_event(resource_id: str): + # + # expects object uri in the form /n//b//o/ + # eg /n/frcnomajoc7v/b/dmarsdb1/o/sqlnet.log + # and calls process_source_file with prefix and file_name extracted from that uri + # + + _, _, prefix, file_name = utils.parse_uri_with_regex(resource_id) + process_source_file(prefix, file_name) + + +def process_source_file(prefix: str, filename: str): + + sourcefile = f"{prefix.rstrip('/')}/{filename}" # rstrip to cater for cases where the prefix is passed with a trailing slash + try: + conn = oraconn.connect("MRDS_LOADER") + + oraconn.run_proc(conn, "CT_MRDS.FILE_MANAGER.PROCESS_SOURCE_FILE", [sourcefile]) + conn.commit() + finally: + conn.close() + + +def execute_query(query, query_parameters=None, account_alias="MRDS_LOADER"): + query_result = None + try: + conn = oraconn.connect(account_alias) + curs = conn.cursor() + if query_parameters != None: + curs.execute(query, query_parameters) + else: + curs.execute(query) + query_result = curs.fetchall() + conn.commit() + finally: + conn.close() + return [t[0] for t in query_result] + + +def get_file_prefix(source_key, source_file_id, table_id): + query_result = None + try: + conn = oraconn.connect("MRDS_LOADER") + curs = conn.cursor() + + curs.execute( + sql_statements.get_sql("get_file_prefix"), + [source_key, source_file_id, table_id], + ) + query_result = curs.fetchone() + conn.commit() + finally: + conn.close() + return query_result[0] + + +def get_inbox_bucket(): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_INBOX_BUCKET", str, []) + conn.commit() + finally: + conn.close() + + return ret + + +def get_data_bucket(): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_func(conn, "CT_MRDS.FILE_MANAGER.GET_DATA_BUCKET", str, []) + conn.commit() + finally: + conn.close() + + return ret + + +def add_source_file_config( + source_key, + source_file_type, + source_file_id, + source_file_desc, + source_file_name_pattern, + table_id, + template_table_name, +): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_proc( + conn, + "CT_MRDS.FILE_MANAGER.ADD_SOURCE_FILE_CONFIG", + [ + source_key, + source_file_type, + source_file_id, + source_file_desc, + source_file_name_pattern, + table_id, + template_table_name, + ], + ) + conn.commit() + finally: + conn.close() + + return ret + + +def add_column_date_format(template_table_name, column_name, date_format): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_proc( + conn, + "CT_MRDS.FILE_MANAGER.ADD_column_date_format", + [template_table_name, column_name, date_format], + ) + conn.commit() + finally: + conn.close() + + return ret + + +def execute(stmt): + try: + conn = oraconn.connect("MRDS_LOADER") + curs = conn.cursor() + curs.execute(stmt) + conn.commit() + finally: + conn.close() + + +def create_external_table(table_name, template_table_name, prefix): + try: + conn = oraconn.connect("ODS_LOADER") + + ret = oraconn.run_proc( + conn, + "CT_MRDS.FILE_MANAGER.CREATE_EXTERNAL_TABLE", + [table_name, template_table_name, prefix, get_bucket("ODS")], + ) + conn.commit() + finally: + conn.close() + + return ret + + +def get_bucket(bucket): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_func( + conn, "CT_MRDS.FILE_MANAGER.GET_BUCKET_URI", str, [bucket] + ) + conn.commit() + finally: + conn.close() + + return ret diff --git a/python/mrds_common/mrds/utils/manage_runs.py b/python/mrds_common/mrds/utils/manage_runs.py new file mode 100644 index 0000000..eb49945 --- /dev/null +++ b/python/mrds_common/mrds/utils/manage_runs.py @@ -0,0 +1,97 @@ +from . import oraconn +from . import sql_statements +from . import static_vars +from . import manage_files + + +def init_workflow(database_name: str, workflow_name: str, workflow_run_id: str): + + try: + conn = oraconn.connect("MRDS_LOADER") + a_workflow_history_key = oraconn.run_func( + conn, + "CT_MRDS.WORKFLOW_MANAGER.INIT_WORKFLOW", + int, + [database_name, workflow_run_id, workflow_name], + ) + conn.commit() + finally: + conn.close() + + return a_workflow_history_key + + +def finalise_workflow(a_workflow_history_key: int, workflow_status: str): + + try: + conn = oraconn.connect("MRDS_LOADER") + + oraconn.run_proc( + conn, + "CT_MRDS.WORKFLOW_MANAGER.FINALISE_WORKFLOW", + [a_workflow_history_key, workflow_status], + ) + + conn.commit() + finally: + conn.close() + + +def init_task(task_name: str, task_run_id: str, a_workflow_history_key: int): + + a_task_history_key: int + + try: + conn = oraconn.connect("MRDS_LOADER") + a_task_history_key = oraconn.run_func( + conn, + "CT_MRDS.WORKFLOW_MANAGER.INIT_TASK", + int, + [task_run_id, task_name, a_workflow_history_key], + ) + + conn.commit() + finally: + conn.close() + + return a_task_history_key + + +def finalise_task(a_task_history_key: int, task_status: str): + + try: + conn = oraconn.connect("MRDS_LOADER") + curs = conn.cursor() + + curs.execute( + sql_statements.get_sql("finalise_task"), [task_status, a_task_history_key] + ) + + conn.commit() + finally: + conn.close() + + +def set_workflow_property( + wf_history_key: int, service_name: str, property: str, value: str +): + try: + conn = oraconn.connect("MRDS_LOADER") + + ret = oraconn.run_proc( + conn, + "CT_MRDS.WORKFLOW_MANAGER.SET_WORKFLOW_PROPERTY", + [wf_history_key, service_name, property, value], + ) + conn.commit() + finally: + conn.close() + + return ret + + +def select_ods_tab(table_name: str, value: str, condition="1 = 1"): + + query = "select %s from %s where %s" % (value, table_name, condition) + print("query = |%s|" % query) + return manage_files.execute_query(query=query, account_alias="ODS_LOADER") diff --git a/python/mrds_common/mrds/utils/objectstore.py b/python/mrds_common/mrds/utils/objectstore.py new file mode 100644 index 0000000..64b5bc6 --- /dev/null +++ b/python/mrds_common/mrds/utils/objectstore.py @@ -0,0 +1,53 @@ +import oci + + +def get_client(): + # + # Authentication is done using Instance Principals on VMs and Resouce Principal on OCI Container Instances + # The function first tries Resource Principal and fails back to Instance Principal in case of error + # + try: + signer = oci.auth.signers.get_resource_principals_signer() + except: + signer = signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + + # Create secret client and retrieve content + client = oci.object_storage.ObjectStorageClient( + {}, signer=signer + ) # the first empyty bracket is an empty config + return client + + +def list_bucket(client, namespace, bucket, prefix): + objects = client.list_objects(namespace, bucket, prefix=prefix) + # see https://docs.oracle.com/en-us/iaas/tools/python/2.135.0/api/request_and_response.html#oci.response.Response for all attrs + return objects.data + + +def upload_file(client, source_filename, namespace, bucket, prefix, target_filename): + with open(source_filename, "rb") as in_file: + client.put_object( + namespace, bucket, f"{prefix.rstrip('/')}/{target_filename}", in_file + ) + + +def clean_folder(client, namespace, bucket, prefix): + objects = client.list_objects(namespace, bucket, prefix=prefix) + for o in objects.data.objects: + print(f"Deleting {prefix.rstrip('/')}/{o.name}") + client.delete_object(namespace, bucket, f"{o.name}") + + +def delete_file(client, file, namespace, bucket, prefix): + client.delete_object(namespace, bucket, f"{prefix.rstrip('/')}/{file}") + + +def download_file(client, namespace, bucket, prefix, source_filename, target_filename): + # Retrieve the file, streaming it into another file in 1 MiB chunks + + get_obj = client.get_object( + namespace, bucket, f"{prefix.rstrip('/')}/{source_filename}" + ) + with open(target_filename, "wb") as f: + for chunk in get_obj.data.raw.stream(1024 * 1024, decode_content=False): + f.write(chunk) diff --git a/python/mrds_common/mrds/utils/oraconn.py b/python/mrds_common/mrds/utils/oraconn.py new file mode 100644 index 0000000..3d149f4 --- /dev/null +++ b/python/mrds_common/mrds/utils/oraconn.py @@ -0,0 +1,38 @@ +import oracledb +import os +import traceback +import sys + + +def connect(alias): + + username = os.getenv(alias + "_DB_USER") + password = os.getenv(alias + "_DB_PASS") + tnsalias = os.getenv(alias + "_DB_TNS") + connstr = username + "/" + password + "@" + tnsalias + + oracledb.init_oracle_client() + + try: + conn = oracledb.connect(connstr) + return conn + except oracledb.DatabaseError as db_err: + tb = traceback.format_exc() + print(f"DatabaseError connecting to '{alias}': {db_err}\n{tb}", file=sys.stderr) + sys.exit(1) + except Exception as exc: + tb = traceback.format_exc() + print(f"Unexpected error connecting to '{alias}': {exc}\n{tb}", file=sys.stderr) + sys.exit(1) + + +def run_proc(connection, proc: str, param: []): + curs = connection.cursor() + curs.callproc(proc, param) + + +def run_func(connection, proc: str, rettype, param: []): + curs = connection.cursor() + ret = curs.callfunc(proc, rettype, param) + + return ret diff --git a/python/mrds_common/mrds/utils/secrets.py b/python/mrds_common/mrds/utils/secrets.py new file mode 100644 index 0000000..be4764c --- /dev/null +++ b/python/mrds_common/mrds/utils/secrets.py @@ -0,0 +1,46 @@ +import oci +import ast +import base64 + +# Specify the OCID of the secret to retrieve + + +def get_secretcontents(ocid): + # + # Authentication is done using Instance Principals on VMs and Resouce Principal on OCI Container Instances + # The function first tries Resource Principal and fails back to Instance Principal in case of error + # + try: + signer = oci.auth.signers.get_resource_principals_signer() + except: + signer = signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + + # Create secret client and retrieve content + secretclient = oci.secrets.SecretsClient({}, signer=signer) + secretcontents = secretclient.get_secret_bundle(secret_id=ocid) + return secretcontents + + +def get_password(ocid): + + secretcontents = get_secretcontents(ocid) + + # Decode the secret from base64 and return password + keybase64 = secretcontents.data.secret_bundle_content.content + keybase64bytes = keybase64.encode("ascii") + keybytes = base64.b64decode(keybase64bytes) + key = keybytes.decode("ascii") + keydict = ast.literal_eval(key) + return keydict["password"] + + +def get_secret(ocid): + + # Create client + secretcontents = get_secretcontents(ocid) + + # Decode the secret from base64 and return it + certbase64 = secretcontents.data.secret_bundle_content.content + certbytes = base64.b64decode(certbase64) + cert = certbytes.decode("UTF-8") + return cert diff --git a/python/mrds_common/mrds/utils/security_utils.py b/python/mrds_common/mrds/utils/security_utils.py new file mode 100644 index 0000000..02d320a --- /dev/null +++ b/python/mrds_common/mrds/utils/security_utils.py @@ -0,0 +1,106 @@ +import re +import logging + + +def verify_run_id(run_id, context=None): + """ + Verify run_id for security compliance. + + Args: + run_id (str): The run_id to verify + context (dict, optional): Airflow context for logging + + Returns: + str: Verified run_id + + Raises: + ValueError: If run_id is invalid or suspicious + """ + try: + # Basic checks + if not run_id or not isinstance(run_id, str): + raise ValueError( + f"Invalid run_id: must be non-empty string, got: {type(run_id).__name__}" + ) + + run_id = run_id.strip() + + if len(run_id) < 1 or len(run_id) > 250: + raise ValueError( + f"Invalid run_id: length must be 1-250 chars, got: {len(run_id)}" + ) + + # Allow only safe characters + if not re.match(r"^[a-zA-Z0-9_\-:+.T]+$", run_id): + suspicious_chars = "".join( + set( + char for char in run_id if not re.match(r"[a-zA-Z0-9_\-:+.T]", char) + ) + ) + logging.warning(f"SECURITY: Invalid chars in run_id: '{suspicious_chars}'") + raise ValueError("Invalid run_id: contains unsafe characters") + + # Check for attack patterns + dangerous_patterns = [ + r"\.\./", + r"\.\.\\", + r"=8.0.0,<9.0.0", + "oci>=2.129.3,<3.0.0", + "oracledb>=2.5.1,<3.0.0", + "PyYAML>=6.0.0,<7.0.0", + "lxml>=5.0.0,<5.3.0", + "xmlschema>=3.4.0,<3.4.3", + "cryptography>=3.3.1,<42.0.0", + "PyJWT>=2.0.0,<3.0.0", + "requests>=2.25.0,<3.0.0", + ], + extras_require={ + "dev": [ + "black==24.10.0", + "tox==4.23.2", + "pytest==8.3.4", + ], + }, + entry_points={ + "console_scripts": [ + "mrds-cli=mrds.cli:cli_main", + ], + }, + author="", + author_email="", + description="MRDS module for MarS ETL POC", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="", + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ], + python_requires=">=3.11", +) diff --git a/python/mrds_common/tox.ini b/python/mrds_common/tox.ini new file mode 100644 index 0000000..7353c3c --- /dev/null +++ b/python/mrds_common/tox.ini @@ -0,0 +1,17 @@ +# tox.ini + +[tox] +envlist = py310, format + +[testenv] +deps = + pytest +commands = + pytest + +[testenv:format] +basepython = python3 +deps = + black +commands = + black --check --diff . \ No newline at end of file