admin/mars-elt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Grzegorz Michalski
2026-03-02 09:47:35 +01:00
commit 2c225d68ac
715 changed files with 130067 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1534
airflow/ods/c2d/uc/config/common_ucdb.xsd Normal file
View File

File diff suppressed because it is too large Load Diff

124
airflow/ods/c2d/uc/config/disseminationFile.xsd Normal file
View File

@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!-- ***************** Schema for UCDB DisseminationFile ******************* -->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:c2d="http://c2d.escb.eu/UseOfCollateralMessage"
targetNamespace="http://c2d.escb.eu/UseOfCollateralMessage" elementFormDefault="qualified" version="3.0">
<!-- ***************** Include Common types ******************* -->
<xs:include schemaLocation="common_ucdb.xsd"/>
<!-- Definition of the root element and its structure -->
<xs:element name="DisseminationFile">
<xs:annotation>
<xs:documentation>Use of Collateral message. Dissemination files hold all reported usages of a snapshot and a specific NCB.</xs:documentation>
</xs:annotation>
<xs:complexType>
<xs:sequence>
<xs:element name="MetaInformation" type="c2d:MetaInformationTypeDisseminationFile"/>
<xs:element name="MarketableAssets" type="c2d:DisseminationMarketableAssetsType" minOccurs="0"/>
<xs:element name="NonMarketableAssets" type="c2d:DisseminationNonMarketableAssetsType" minOccurs="0"/>
<xs:element name="NonMarketableDECCs" type="c2d:DisseminationNonMarketableDECCsType" minOccurs="0"/>
</xs:sequence>
<xs:attribute name="version" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:complexType name="MetaInformationTypeDisseminationFile">
<xs:complexContent>
<xs:extension base="c2d:AbstractMetaInformationType">
<xs:sequence>
<xs:element name="ReportingNCB" type="c2d:EurosystemISOCodeType">
<xs:annotation>
<xs:documentation>The two letter code identifying the NCB contained in the dissemination file.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="SnapshotDate" type="xs:date">
<xs:annotation>
<xs:documentation>The date the snapshot of the initial file data was taken.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="DateCreated" type="xs:dateTime">
<xs:annotation>
<xs:documentation>The date when the dissemination file has been created.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="NumberOfSuspectRecords" type="xs:unsignedInt">
<xs:annotation>
<xs:documentation>The number of records in the dissemination file still in status suspect.</xs:documentation>
</xs:annotation>
</xs:element>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="DisseminationMarketableAssetsType">
<xs:annotation>
<xs:documentation>Holds all marketable assets.</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="MarketableAsset" type="c2d:DisseminationMarketableAssetType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="DisseminationNonMarketableAssetsType">
<xs:annotation>
<xs:documentation>Holds all non marketable assets.</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="NonMarketableAsset" type="c2d:DisseminationNonMarketableAssetType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="DisseminationNonMarketableDECCsType">
<xs:annotation>
<xs:documentation>Holds all non marketable DECCs.</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="NonMarketableDECC" type="c2d:DisseminationNonMarketableDECCType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="DisseminationMarketableAssetType">
<xs:annotation>
<xs:documentation>Marketable Assets with suspect addon</xs:documentation>
</xs:annotation>
<xs:complexContent>
<xs:extension base="c2d:MarketableAssetType">
<xs:sequence>
<xs:element name="SuspectInformation" type="c2d:DisseminationSuspectType" minOccurs="0"/>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="DisseminationNonMarketableAssetType">
<xs:annotation>
<xs:documentation>Non Marketable Assets with suspect addon</xs:documentation>
</xs:annotation>
<xs:complexContent>
<xs:extension base="c2d:NonMarketableAssetType">
<xs:sequence>
<xs:element name="SuspectInformation" type="c2d:DisseminationSuspectType" minOccurs="0"/>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="DisseminationNonMarketableDECCType">
<xs:annotation>
<xs:documentation>Non Marketable DECCs with suspect addon</xs:documentation>
</xs:annotation>
<xs:complexContent>
<xs:extension base="c2d:NonMarketableDECCType">
<xs:sequence>
<xs:element name="SuspectInformation" type="c2d:DisseminationSuspectType" minOccurs="0"/>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="DisseminationSuspectType">
<xs:annotation>
<xs:documentation>Holds all suspect information of a reported usage.</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="SuspectId" type="c2d:IntegerGreaterThanZeroType">
<xs:annotation>
<xs:documentation>An ID created by the UCDB system identifying each single record that is stored in the UCDB system as a suspect record.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="SuspectReasons" type="c2d:SuspectsReasonsType"/>
</xs:sequence>
</xs:complexType>
</xs:schema>

45
airflow/ods/c2d/uc/config/m_ODS_C2D_UC_DISSEM_PARSE.yml Normal file
View File

@@ -0,0 +1,45 @@
# Global configurations
inbox_prefix: INBOX/C2D/CollateralDataDissemination
archive_prefix: ARCHIVE/C2D/CollateralDataDissemination
workflow_name: w_ODS_C2D_UC_DISSEMI
validation_schema_path: 'disseminationFile.xsd'
file_type: xml
# List of tasks
tasks:
# Task 1
- task_name: m_ODS_C2D_UC_DISSEM_METADATA_PARSE
ods_prefix: INBOX/C2D/CollateralDataDissemination/C2D_A_UC_DISSEM_METADATA_LOADS
output_table: C2D_A_UC_DISSEM_METADATA_LOADS
namespaces:
ns: 'http://c2d.escb.eu/UseOfCollateralMessage'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '//ns:DisseminationFile/@version'
column_header: 'C2D_VERSION'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:DateCreated'
column_header: 'FILE_CREATION_DATE'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:NumberOfSuspectRecords'
column_header: 'NO_OF_SUSPECT_RECORDS'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:ReportingNCB'
column_header: 'REPORTING_NCB'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:SnapshotDate'
column_header: 'SNAPSHOT_DATE'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'PROCESSED_TO_DWH'

968
airflow/ods/c2d/uc/config/ucdb_cl.xsd Normal file
View File

@@ -0,0 +1,968 @@
<?xml version="1.0" encoding="UTF-8"?><schema targetNamespace="http://c2d.escb.eu/UseOfCollateralMessage" elementFormDefault="qualified" attributeFormDefault="qualified" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.w3.org/2001/XMLSchema">
<simpleType name="cl_issuer_csd">
<restriction base="string">
<enumeration value="CLAT01"/>
<enumeration value="CLBE01"/>
<enumeration value="CLBE02"/>
<enumeration value="CLBG01"/>
<enumeration value="CLBL01"/>
<enumeration value="CLCY01"/>
<enumeration value="CLCZ01"/>
<enumeration value="CLDE01"/>
<enumeration value="CLDE02"/>
<enumeration value="CLDK01"/>
<enumeration value="CLDL01"/>
<enumeration value="CLEE01"/>
<enumeration value="CLES01"/>
<enumeration value="CLEU01"/>
<enumeration value="CLFI01"/>
<enumeration value="CLFR01"/>
<enumeration value="CLGR01"/>
<enumeration value="CLHR01"/>
<enumeration value="CLIT01"/>
<enumeration value="CLLD01"/>
<enumeration value="CLLT02"/>
<enumeration value="CLLU01"/>
<enumeration value="CLLU03"/>
<enumeration value="CLLV02"/>
<enumeration value="CLMT01"/>
<enumeration value="CLNL01"/>
<enumeration value="CLPT02"/>
<enumeration value="CLSI01"/>
<enumeration value="CLSK01"/>
<enumeration value="CLSK02"/>
</restriction>
</simpleType>
<simpleType name="cl_ccb">
<restriction base="string">
<enumeration value="AT"/>
<enumeration value="BE"/>
<enumeration value="BG"/>
<enumeration value="CY"/>
<enumeration value="DE"/>
<enumeration value="EE"/>
<enumeration value="ES"/>
<enumeration value="FI"/>
<enumeration value="FR"/>
<enumeration value="GR"/>
<enumeration value="HR"/>
<enumeration value="IE"/>
<enumeration value="IT"/>
<enumeration value="LT"/>
<enumeration value="LU"/>
<enumeration value="LV"/>
<enumeration value="MT"/>
<enumeration value="NL"/>
<enumeration value="PT"/>
<enumeration value="SI"/>
<enumeration value="SK"/>
</restriction>
</simpleType>
<simpleType name="cl_mobilisation_channel">
<restriction base="string">
<enumeration value="CCBM mkt"/>
<enumeration value="CCBM mkt with links"/>
<enumeration value="CCBM nonmkt"/>
<enumeration value="Direct access"/>
<enumeration value="Direct access with links"/>
<enumeration value="Local CSD"/>
<enumeration value="Local CSD with links"/>
<enumeration value="Local cb nonmkt"/>
<enumeration value="Local dom nonmkt"/>
</restriction>
</simpleType>
<simpleType name="cl_triparty_agent">
<restriction base="string">
<enumeration value="CLBE02"/>
<enumeration value="CLDE01"/>
<enumeration value="CLFR01"/>
<enumeration value="CLIT01"/>
<enumeration value="CLLU01"/>
<enumeration value="CLNL01"/>
</restriction>
</simpleType>
<simpleType name="cl_la_rating">
<restriction base="string">
<enumeration value="LADB01"/>
<enumeration value="LADB02"/>
<enumeration value="LADB03"/>
<enumeration value="LADB04"/>
<enumeration value="LADB05"/>
<enumeration value="LADB06"/>
<enumeration value="LADB07"/>
<enumeration value="LADB08"/>
<enumeration value="LADB09"/>
<enumeration value="LADB10"/>
<enumeration value="LADB11"/>
<enumeration value="LADB12"/>
<enumeration value="LADB13"/>
<enumeration value="LADB14"/>
<enumeration value="LADB15"/>
<enumeration value="LADB16"/>
<enumeration value="LADB17"/>
<enumeration value="LADB18"/>
<enumeration value="LADB19"/>
<enumeration value="LADB20"/>
<enumeration value="LADB21"/>
<enumeration value="LADB22"/>
<enumeration value="LADB23"/>
<enumeration value="LADB24"/>
<enumeration value="LADB25"/>
<enumeration value="LADB26"/>
<enumeration value="LAIA01"/>
<enumeration value="LAIA02"/>
<enumeration value="LAIA03"/>
<enumeration value="LAIA04"/>
<enumeration value="LAIA05"/>
<enumeration value="LAIA06"/>
<enumeration value="LAIA07"/>
<enumeration value="LAIA08"/>
<enumeration value="LAIA09"/>
<enumeration value="LAIA10"/>
<enumeration value="LAIA11"/>
<enumeration value="LAIA12"/>
<enumeration value="LAIA13"/>
<enumeration value="LAIA14"/>
<enumeration value="LAIA15"/>
<enumeration value="LAIA16"/>
<enumeration value="LAIA17"/>
<enumeration value="LAIA18"/>
<enumeration value="LAIA19"/>
<enumeration value="LAIA20"/>
<enumeration value="LAIA21"/>
<enumeration value="LAMY01"/>
<enumeration value="LAMY02"/>
<enumeration value="LAMY03"/>
<enumeration value="LAMY04"/>
<enumeration value="LAMY05"/>
<enumeration value="LAMY06"/>
<enumeration value="LAMY07"/>
<enumeration value="LAMY08"/>
<enumeration value="LAMY09"/>
<enumeration value="LAMY10"/>
<enumeration value="LAMY11"/>
<enumeration value="LAMY12"/>
<enumeration value="LAMY13"/>
<enumeration value="LAMY14"/>
<enumeration value="LAMY15"/>
<enumeration value="LAMY16"/>
<enumeration value="LAMY17"/>
<enumeration value="LAMY18"/>
<enumeration value="LAMY19"/>
<enumeration value="LAMY20"/>
<enumeration value="LAMY21"/>
<enumeration value="LASC01"/>
<enumeration value="LASC02"/>
<enumeration value="LASC03"/>
<enumeration value="LASC04"/>
<enumeration value="LASC05"/>
<enumeration value="LASC06"/>
<enumeration value="LASC07"/>
<enumeration value="LASC08"/>
<enumeration value="LASC09"/>
<enumeration value="LASC10"/>
<enumeration value="LASC11"/>
<enumeration value="LASC12"/>
<enumeration value="LASC13"/>
<enumeration value="LASC14"/>
<enumeration value="LASC15"/>
<enumeration value="LASC16"/>
<enumeration value="LASC17"/>
<enumeration value="LASC18"/>
<enumeration value="LASC19"/>
<enumeration value="LASC20"/>
<enumeration value="LASC21"/>
<enumeration value="LASP01"/>
<enumeration value="LASP02"/>
<enumeration value="LASP03"/>
<enumeration value="LASP04"/>
<enumeration value="LASP05"/>
<enumeration value="LASP06"/>
<enumeration value="LASP07"/>
<enumeration value="LASP08"/>
<enumeration value="LASP09"/>
<enumeration value="LASP10"/>
<enumeration value="LASP11"/>
<enumeration value="LASP12"/>
<enumeration value="LASP13"/>
<enumeration value="LASP14"/>
<enumeration value="LASP15"/>
<enumeration value="LASP16"/>
<enumeration value="LASP17"/>
<enumeration value="LASP18"/>
<enumeration value="LASP19"/>
<enumeration value="LASP20"/>
<enumeration value="LASP21"/>
<enumeration value="LASP22"/>
<enumeration value="LPDB01"/>
<enumeration value="LPDB02"/>
<enumeration value="LPDB03"/>
<enumeration value="LPDB04"/>
<enumeration value="LPDB05"/>
<enumeration value="LPDB06"/>
<enumeration value="LPDB07"/>
<enumeration value="LPDB08"/>
<enumeration value="LPDB09"/>
<enumeration value="LPDB10"/>
<enumeration value="LPDB11"/>
<enumeration value="LPDB12"/>
<enumeration value="LPDB13"/>
<enumeration value="LPDB14"/>
<enumeration value="LPDB15"/>
<enumeration value="LPDB16"/>
<enumeration value="LPDB17"/>
<enumeration value="LPDB18"/>
<enumeration value="LPDB19"/>
<enumeration value="LPDB20"/>
<enumeration value="LPDB21"/>
<enumeration value="LPDB22"/>
<enumeration value="LPDB23"/>
<enumeration value="LPDB24"/>
<enumeration value="LPDB25"/>
<enumeration value="LPDB26"/>
<enumeration value="LPIA01"/>
<enumeration value="LPIA02"/>
<enumeration value="LPIA03"/>
<enumeration value="LPIA04"/>
<enumeration value="LPIA05"/>
<enumeration value="LPIA06"/>
<enumeration value="LPIA07"/>
<enumeration value="LPIA08"/>
<enumeration value="LPIA09"/>
<enumeration value="LPIA10"/>
<enumeration value="LPIA11"/>
<enumeration value="LPIA12"/>
<enumeration value="LPIA13"/>
<enumeration value="LPIA14"/>
<enumeration value="LPIA15"/>
<enumeration value="LPIA16"/>
<enumeration value="LPIA17"/>
<enumeration value="LPIA18"/>
<enumeration value="LPIA19"/>
<enumeration value="LPIA20"/>
<enumeration value="LPIA21"/>
<enumeration value="LPMY01"/>
<enumeration value="LPMY02"/>
<enumeration value="LPMY03"/>
<enumeration value="LPMY04"/>
<enumeration value="LPMY05"/>
<enumeration value="LPMY06"/>
<enumeration value="LPMY07"/>
<enumeration value="LPMY08"/>
<enumeration value="LPMY09"/>
<enumeration value="LPMY10"/>
<enumeration value="LPMY11"/>
<enumeration value="LPMY12"/>
<enumeration value="LPMY13"/>
<enumeration value="LPMY14"/>
<enumeration value="LPMY15"/>
<enumeration value="LPMY16"/>
<enumeration value="LPMY17"/>
<enumeration value="LPMY18"/>
<enumeration value="LPMY19"/>
<enumeration value="LPMY20"/>
<enumeration value="LPMY21"/>
<enumeration value="LPSC01"/>
<enumeration value="LPSC02"/>
<enumeration value="LPSC03"/>
<enumeration value="LPSC04"/>
<enumeration value="LPSC05"/>
<enumeration value="LPSC06"/>
<enumeration value="LPSC07"/>
<enumeration value="LPSC08"/>
<enumeration value="LPSC09"/>
<enumeration value="LPSC10"/>
<enumeration value="LPSC11"/>
<enumeration value="LPSC12"/>
<enumeration value="LPSC13"/>
<enumeration value="LPSC14"/>
<enumeration value="LPSC15"/>
<enumeration value="LPSC16"/>
<enumeration value="LPSC17"/>
<enumeration value="LPSC18"/>
<enumeration value="LPSC19"/>
<enumeration value="LPSC20"/>
<enumeration value="LPSC21"/>
<enumeration value="LPSP01"/>
<enumeration value="LPSP02"/>
<enumeration value="LPSP03"/>
<enumeration value="LPSP04"/>
<enumeration value="LPSP05"/>
<enumeration value="LPSP06"/>
<enumeration value="LPSP07"/>
<enumeration value="LPSP08"/>
<enumeration value="LPSP09"/>
<enumeration value="LPSP10"/>
<enumeration value="LPSP11"/>
<enumeration value="LPSP12"/>
<enumeration value="LPSP13"/>
<enumeration value="LPSP14"/>
<enumeration value="LPSP15"/>
<enumeration value="LPSP16"/>
<enumeration value="LPSP17"/>
<enumeration value="LPSP18"/>
<enumeration value="LPSP19"/>
<enumeration value="LPSP20"/>
<enumeration value="LPSP21"/>
<enumeration value="LPSP22"/>
</restriction>
</simpleType>
<simpleType name="cl_si_rating">
<restriction base="string">
<enumeration value="SIDB01"/>
<enumeration value="SIDB02"/>
<enumeration value="SIDB03"/>
<enumeration value="SIDB04"/>
<enumeration value="SIDB05"/>
<enumeration value="SIDB06"/>
<enumeration value="SIDB07"/>
<enumeration value="SIDB08"/>
<enumeration value="SIDB09"/>
<enumeration value="SIIA01"/>
<enumeration value="SIIA02"/>
<enumeration value="SIIA03"/>
<enumeration value="SIIA04"/>
<enumeration value="SIIA05"/>
<enumeration value="SIIA06"/>
<enumeration value="SIIA07"/>
<enumeration value="SIMY01"/>
<enumeration value="SIMY02"/>
<enumeration value="SIMY03"/>
<enumeration value="SIMY04"/>
<enumeration value="SISC01"/>
<enumeration value="SISC02"/>
<enumeration value="SISC03"/>
<enumeration value="SISC04"/>
<enumeration value="SISC05"/>
<enumeration value="SISC06"/>
<enumeration value="SISC07"/>
<enumeration value="SISP01"/>
<enumeration value="SISP02"/>
<enumeration value="SISP03"/>
<enumeration value="SISP04"/>
<enumeration value="SISP05"/>
<enumeration value="SISP06"/>
<enumeration value="SISP07"/>
</restriction>
</simpleType>
<simpleType name="cl_li_rating">
<restriction base="string">
<enumeration value="LIDB01"/>
<enumeration value="LIDB02"/>
<enumeration value="LIDB03"/>
<enumeration value="LIDB04"/>
<enumeration value="LIDB05"/>
<enumeration value="LIDB06"/>
<enumeration value="LIDB07"/>
<enumeration value="LIDB08"/>
<enumeration value="LIDB09"/>
<enumeration value="LIDB10"/>
<enumeration value="LIDB11"/>
<enumeration value="LIDB12"/>
<enumeration value="LIDB13"/>
<enumeration value="LIDB14"/>
<enumeration value="LIDB15"/>
<enumeration value="LIDB16"/>
<enumeration value="LIDB17"/>
<enumeration value="LIDB18"/>
<enumeration value="LIDB19"/>
<enumeration value="LIDB20"/>
<enumeration value="LIDB21"/>
<enumeration value="LIDB22"/>
<enumeration value="LIDB23"/>
<enumeration value="LIDB24"/>
<enumeration value="LIDB25"/>
<enumeration value="LIDB26"/>
<enumeration value="LIIA01"/>
<enumeration value="LIIA02"/>
<enumeration value="LIIA03"/>
<enumeration value="LIIA04"/>
<enumeration value="LIIA05"/>
<enumeration value="LIIA06"/>
<enumeration value="LIIA07"/>
<enumeration value="LIIA08"/>
<enumeration value="LIIA09"/>
<enumeration value="LIIA10"/>
<enumeration value="LIIA11"/>
<enumeration value="LIIA12"/>
<enumeration value="LIIA13"/>
<enumeration value="LIIA14"/>
<enumeration value="LIIA15"/>
<enumeration value="LIIA16"/>
<enumeration value="LIIA17"/>
<enumeration value="LIIA18"/>
<enumeration value="LIIA19"/>
<enumeration value="LIIA20"/>
<enumeration value="LIIA21"/>
<enumeration value="LIIA22"/>
<enumeration value="LIMY01"/>
<enumeration value="LIMY02"/>
<enumeration value="LIMY03"/>
<enumeration value="LIMY04"/>
<enumeration value="LIMY05"/>
<enumeration value="LIMY06"/>
<enumeration value="LIMY07"/>
<enumeration value="LIMY08"/>
<enumeration value="LIMY09"/>
<enumeration value="LIMY10"/>
<enumeration value="LIMY11"/>
<enumeration value="LIMY12"/>
<enumeration value="LIMY13"/>
<enumeration value="LIMY14"/>
<enumeration value="LIMY15"/>
<enumeration value="LIMY16"/>
<enumeration value="LIMY17"/>
<enumeration value="LIMY18"/>
<enumeration value="LIMY19"/>
<enumeration value="LIMY20"/>
<enumeration value="LIMY21"/>
<enumeration value="LISC01"/>
<enumeration value="LISC02"/>
<enumeration value="LISC03"/>
<enumeration value="LISC04"/>
<enumeration value="LISC05"/>
<enumeration value="LISC06"/>
<enumeration value="LISC07"/>
<enumeration value="LISC08"/>
<enumeration value="LISC09"/>
<enumeration value="LISC10"/>
<enumeration value="LISC11"/>
<enumeration value="LISC12"/>
<enumeration value="LISC13"/>
<enumeration value="LISC14"/>
<enumeration value="LISC15"/>
<enumeration value="LISC16"/>
<enumeration value="LISC17"/>
<enumeration value="LISC18"/>
<enumeration value="LISC19"/>
<enumeration value="LISC20"/>
<enumeration value="LISC21"/>
<enumeration value="LISP01"/>
<enumeration value="LISP02"/>
<enumeration value="LISP03"/>
<enumeration value="LISP04"/>
<enumeration value="LISP05"/>
<enumeration value="LISP06"/>
<enumeration value="LISP07"/>
<enumeration value="LISP08"/>
<enumeration value="LISP09"/>
<enumeration value="LISP10"/>
<enumeration value="LISP11"/>
<enumeration value="LISP12"/>
<enumeration value="LISP13"/>
<enumeration value="LISP14"/>
<enumeration value="LISP15"/>
<enumeration value="LISP16"/>
<enumeration value="LISP17"/>
<enumeration value="LISP18"/>
<enumeration value="LISP19"/>
<enumeration value="LISP20"/>
<enumeration value="LISP21"/>
<enumeration value="LISP22"/>
</restriction>
</simpleType>
<simpleType name="cl_lg_rating">
<restriction base="string">
<enumeration value="LGDB01"/>
<enumeration value="LGDB02"/>
<enumeration value="LGDB03"/>
<enumeration value="LGDB04"/>
<enumeration value="LGDB05"/>
<enumeration value="LGDB06"/>
<enumeration value="LGDB07"/>
<enumeration value="LGDB08"/>
<enumeration value="LGDB09"/>
<enumeration value="LGDB10"/>
<enumeration value="LGDB11"/>
<enumeration value="LGDB12"/>
<enumeration value="LGDB13"/>
<enumeration value="LGDB14"/>
<enumeration value="LGDB15"/>
<enumeration value="LGDB16"/>
<enumeration value="LGDB17"/>
<enumeration value="LGDB18"/>
<enumeration value="LGDB19"/>
<enumeration value="LGDB20"/>
<enumeration value="LGDB21"/>
<enumeration value="LGDB22"/>
<enumeration value="LGDB23"/>
<enumeration value="LGDB24"/>
<enumeration value="LGDB25"/>
<enumeration value="LGDB26"/>
<enumeration value="LGIA01"/>
<enumeration value="LGIA02"/>
<enumeration value="LGIA03"/>
<enumeration value="LGIA04"/>
<enumeration value="LGIA05"/>
<enumeration value="LGIA06"/>
<enumeration value="LGIA07"/>
<enumeration value="LGIA08"/>
<enumeration value="LGIA09"/>
<enumeration value="LGIA10"/>
<enumeration value="LGIA11"/>
<enumeration value="LGIA12"/>
<enumeration value="LGIA13"/>
<enumeration value="LGIA14"/>
<enumeration value="LGIA15"/>
<enumeration value="LGIA16"/>
<enumeration value="LGIA17"/>
<enumeration value="LGIA18"/>
<enumeration value="LGIA19"/>
<enumeration value="LGIA20"/>
<enumeration value="LGIA21"/>
<enumeration value="LGIA22"/>
<enumeration value="LGMY01"/>
<enumeration value="LGMY02"/>
<enumeration value="LGMY03"/>
<enumeration value="LGMY04"/>
<enumeration value="LGMY05"/>
<enumeration value="LGMY06"/>
<enumeration value="LGMY07"/>
<enumeration value="LGMY08"/>
<enumeration value="LGMY09"/>
<enumeration value="LGMY10"/>
<enumeration value="LGMY11"/>
<enumeration value="LGMY12"/>
<enumeration value="LGMY13"/>
<enumeration value="LGMY14"/>
<enumeration value="LGMY15"/>
<enumeration value="LGMY16"/>
<enumeration value="LGMY17"/>
<enumeration value="LGMY18"/>
<enumeration value="LGMY19"/>
<enumeration value="LGMY20"/>
<enumeration value="LGMY21"/>
<enumeration value="LGSC01"/>
<enumeration value="LGSC02"/>
<enumeration value="LGSC03"/>
<enumeration value="LGSC04"/>
<enumeration value="LGSC05"/>
<enumeration value="LGSC06"/>
<enumeration value="LGSC07"/>
<enumeration value="LGSC08"/>
<enumeration value="LGSC09"/>
<enumeration value="LGSC10"/>
<enumeration value="LGSC11"/>
<enumeration value="LGSC12"/>
<enumeration value="LGSC13"/>
<enumeration value="LGSC14"/>
<enumeration value="LGSC15"/>
<enumeration value="LGSC16"/>
<enumeration value="LGSC17"/>
<enumeration value="LGSC18"/>
<enumeration value="LGSC19"/>
<enumeration value="LGSC20"/>
<enumeration value="LGSC21"/>
<enumeration value="LGSP01"/>
<enumeration value="LGSP02"/>
<enumeration value="LGSP03"/>
<enumeration value="LGSP04"/>
<enumeration value="LGSP05"/>
<enumeration value="LGSP06"/>
<enumeration value="LGSP07"/>
<enumeration value="LGSP08"/>
<enumeration value="LGSP09"/>
<enumeration value="LGSP10"/>
<enumeration value="LGSP11"/>
<enumeration value="LGSP12"/>
<enumeration value="LGSP13"/>
<enumeration value="LGSP14"/>
<enumeration value="LGSP15"/>
<enumeration value="LGSP16"/>
<enumeration value="LGSP17"/>
<enumeration value="LGSP18"/>
<enumeration value="LGSP19"/>
<enumeration value="LGSP20"/>
<enumeration value="LGSP21"/>
<enumeration value="LGSP22"/>
</restriction>
</simpleType>
<simpleType name="cl_sa_rating">
<restriction base="string">
<enumeration value="SADB01"/>
<enumeration value="SADB02"/>
<enumeration value="SADB03"/>
<enumeration value="SADB04"/>
<enumeration value="SADB05"/>
<enumeration value="SADB06"/>
<enumeration value="SADB07"/>
<enumeration value="SADB08"/>
<enumeration value="SADB09"/>
<enumeration value="SAIA01"/>
<enumeration value="SAIA02"/>
<enumeration value="SAIA03"/>
<enumeration value="SAIA04"/>
<enumeration value="SAIA05"/>
<enumeration value="SAIA06"/>
<enumeration value="SAIA07"/>
<enumeration value="SAMY01"/>
<enumeration value="SAMY02"/>
<enumeration value="SAMY03"/>
<enumeration value="SAMY04"/>
<enumeration value="SASC01"/>
<enumeration value="SASC02"/>
<enumeration value="SASC03"/>
<enumeration value="SASC04"/>
<enumeration value="SASC05"/>
<enumeration value="SASC06"/>
<enumeration value="SASC07"/>
<enumeration value="SASP01"/>
<enumeration value="SASP02"/>
<enumeration value="SASP03"/>
<enumeration value="SASP04"/>
<enumeration value="SASP05"/>
<enumeration value="SASP06"/>
<enumeration value="SASP07"/>
<enumeration value="SPDB01"/>
<enumeration value="SPDB02"/>
<enumeration value="SPDB03"/>
<enumeration value="SPDB04"/>
<enumeration value="SPDB05"/>
<enumeration value="SPDB06"/>
<enumeration value="SPDB07"/>
<enumeration value="SPDB08"/>
<enumeration value="SPDB09"/>
<enumeration value="SPIA01"/>
<enumeration value="SPIA02"/>
<enumeration value="SPIA03"/>
<enumeration value="SPIA04"/>
<enumeration value="SPIA05"/>
<enumeration value="SPIA06"/>
<enumeration value="SPIA07"/>
<enumeration value="SPMY01"/>
<enumeration value="SPMY02"/>
<enumeration value="SPMY03"/>
<enumeration value="SPMY04"/>
<enumeration value="SPSC01"/>
<enumeration value="SPSC02"/>
<enumeration value="SPSC03"/>
<enumeration value="SPSC04"/>
<enumeration value="SPSC05"/>
<enumeration value="SPSC06"/>
<enumeration value="SPSC07"/>
<enumeration value="SPSP01"/>
<enumeration value="SPSP02"/>
<enumeration value="SPSP03"/>
<enumeration value="SPSP04"/>
<enumeration value="SPSP05"/>
<enumeration value="SPSP06"/>
<enumeration value="SPSP07"/>
</restriction>
</simpleType>
<simpleType name="cl_cgr_rating">
<restriction base="string">
<enumeration value="10"/>
<enumeration value="109"/>
<enumeration value="20"/>
<enumeration value="30"/>
<enumeration value="40"/>
<enumeration value="99"/>
</restriction>
</simpleType>
<simpleType name="cl_reference_rate">
<restriction base="string">
<enumeration value="10MEUBOR"/>
<enumeration value="10YEUIRS"/>
<enumeration value="10YGOTTEX"/>
<enumeration value="10YICAP"/>
<enumeration value="10YICES"/>
<enumeration value="10YOLO"/>
<enumeration value="11MEUBOR"/>
<enumeration value="11YEUIRS"/>
<enumeration value="11YICAP"/>
<enumeration value="11YOLO"/>
<enumeration value="12YEUIRS"/>
<enumeration value="12YGOTTEX"/>
<enumeration value="12YICAP"/>
<enumeration value="12YICES"/>
<enumeration value="12YOLO"/>
<enumeration value="13YEUIRS"/>
<enumeration value="13YICAP"/>
<enumeration value="13YOLO"/>
<enumeration value="14YEUIRS"/>
<enumeration value="14YICAP"/>
<enumeration value="14YOLO"/>
<enumeration value="15YEUIRS"/>
<enumeration value="15YGOTTEX"/>
<enumeration value="15YICAP"/>
<enumeration value="15YICES"/>
<enumeration value="15YOLO"/>
<enumeration value="16YICAP"/>
<enumeration value="16YOLO"/>
<enumeration value="17YICAP"/>
<enumeration value="17YOLO"/>
<enumeration value="18YICAP"/>
<enumeration value="18YOLO"/>
<enumeration value="19YICAP"/>
<enumeration value="19YOLO"/>
<enumeration value="1MEUBOR"/>
<enumeration value="1MEUCMS"/>
<enumeration value="1MLIBOR"/>
<enumeration value="1MLICMS"/>
<enumeration value="1WEUBOR"/>
<enumeration value="1WEUCMS"/>
<enumeration value="1WLIBOR"/>
<enumeration value="1WLICMS"/>
<enumeration value="1YEUBOR"/>
<enumeration value="1YEUCMS"/>
<enumeration value="1YEUIRS"/>
<enumeration value="1YICAP"/>
<enumeration value="1YICES"/>
<enumeration value="1YLIBOR"/>
<enumeration value="1YLICMS"/>
<enumeration value="1YOLO"/>
<enumeration value="20YEUIRS"/>
<enumeration value="20YGOTTEX"/>
<enumeration value="20YICAP"/>
<enumeration value="20YICES"/>
<enumeration value="20YOLO"/>
<enumeration value="21YICAP"/>
<enumeration value="21YOLO"/>
<enumeration value="22YICAP"/>
<enumeration value="22YOLO"/>
<enumeration value="23YICAP"/>
<enumeration value="23YOLO"/>
<enumeration value="24YICAP"/>
<enumeration value="24YOLO"/>
<enumeration value="25YEUIRS"/>
<enumeration value="25YICAP"/>
<enumeration value="25YICES"/>
<enumeration value="25YOLO"/>
<enumeration value="26YICAP"/>
<enumeration value="26YOLO"/>
<enumeration value="27YICAP"/>
<enumeration value="27YOLO"/>
<enumeration value="28YICAP"/>
<enumeration value="28YOLO"/>
<enumeration value="29YICAP"/>
<enumeration value="29YOLO"/>
<enumeration value="2MEUBOR"/>
<enumeration value="2MEUCMS"/>
<enumeration value="2MLIBOR"/>
<enumeration value="2MLICMS"/>
<enumeration value="2WEUBOR"/>
<enumeration value="2WEUCMS"/>
<enumeration value="2WLIBOR"/>
<enumeration value="2WLICMS"/>
<enumeration value="2YEUIRS"/>
<enumeration value="2YGOTTEX"/>
<enumeration value="2YICAP"/>
<enumeration value="2YICES"/>
<enumeration value="2YOLO"/>
<enumeration value="30YEUIRS"/>
<enumeration value="30YGOTTEX"/>
<enumeration value="30YICAP"/>
<enumeration value="30YICES"/>
<enumeration value="30YOLO"/>
<enumeration value="35YICAP"/>
<enumeration value="3MEUBOR"/>
<enumeration value="3MEUCMS"/>
<enumeration value="3MLIBOR"/>
<enumeration value="3MLICMS"/>
<enumeration value="3WEUBOR"/>
<enumeration value="3YEUIRS"/>
<enumeration value="3YGOTTEX"/>
<enumeration value="3YICAP"/>
<enumeration value="3YICES"/>
<enumeration value="3YOLO"/>
<enumeration value="40YICAP"/>
<enumeration value="4MEUBOR"/>
<enumeration value="4YEUIRS"/>
<enumeration value="4YGOTTEX"/>
<enumeration value="4YICAP"/>
<enumeration value="4YICES"/>
<enumeration value="4YOLO"/>
<enumeration value="50YICAP"/>
<enumeration value="5MEUBOR"/>
<enumeration value="5YEUIRS"/>
<enumeration value="5YGOTTEX"/>
<enumeration value="5YICAP"/>
<enumeration value="5YICES"/>
<enumeration value="5YOLO"/>
<enumeration value="6MEUBOR"/>
<enumeration value="6MEUCMS"/>
<enumeration value="6MLIBOR"/>
<enumeration value="6MLICMS"/>
<enumeration value="6YEUIRS"/>
<enumeration value="6YGOTTEX"/>
<enumeration value="6YICAP"/>
<enumeration value="6YICES"/>
<enumeration value="6YOLO"/>
<enumeration value="7MEUBOR"/>
<enumeration value="7YEUIRS"/>
<enumeration value="7YGOTTEX"/>
<enumeration value="7YICAP"/>
<enumeration value="7YICES"/>
<enumeration value="7YOLO"/>
<enumeration value="8MEUBOR"/>
<enumeration value="8YEUIRS"/>
<enumeration value="8YGOTTEX"/>
<enumeration value="8YICAP"/>
<enumeration value="8YICES"/>
<enumeration value="8YOLO"/>
<enumeration value="9MEUBOR"/>
<enumeration value="9MEUCMS"/>
<enumeration value="9MLIBOR"/>
<enumeration value="9MLICMS"/>
<enumeration value="9YEUIRS"/>
<enumeration value="9YGOTTEX"/>
<enumeration value="9YICAP"/>
<enumeration value="9YICES"/>
<enumeration value="9YOLO"/>
<enumeration value="A10YEUIRS"/>
<enumeration value="A11YEUIRS"/>
<enumeration value="A12YEUIRS"/>
<enumeration value="A13YEUIRS"/>
<enumeration value="A14YEUIRS"/>
<enumeration value="A15YEUIRS"/>
<enumeration value="A1MEUBOR"/>
<enumeration value="A1MEUCMS"/>
<enumeration value="A1MLIBOR"/>
<enumeration value="A1MLICMS"/>
<enumeration value="A1WEUBOR"/>
<enumeration value="A1WEUCMS"/>
<enumeration value="A1WLIBOR"/>
<enumeration value="A1WLICMS"/>
<enumeration value="A1YEUBOR"/>
<enumeration value="A1YEUCMS"/>
<enumeration value="A1YEUIRS"/>
<enumeration value="A1YLIBOR"/>
<enumeration value="A1YLICMS"/>
<enumeration value="A20YEUIRS"/>
<enumeration value="A25YEUIRS"/>
<enumeration value="A2MEUBOR"/>
<enumeration value="A2MEUCMS"/>
<enumeration value="A2MLIBOR"/>
<enumeration value="A2MLICMS"/>
<enumeration value="A2WEUBOR"/>
<enumeration value="A2WEUCMS"/>
<enumeration value="A2WLIBOR"/>
<enumeration value="A2WLICMS"/>
<enumeration value="A2YEUIRS"/>
<enumeration value="A30YEUIRS"/>
<enumeration value="A3MEUBOR"/>
<enumeration value="A3MEUCMS"/>
<enumeration value="A3MLIBOR"/>
<enumeration value="A3MLICMS"/>
<enumeration value="A3YEUIRS"/>
<enumeration value="A4YEUIRS"/>
<enumeration value="A5YEUIRS"/>
<enumeration value="A6MEUBOR"/>
<enumeration value="A6MEUCMS"/>
<enumeration value="A6MLIBOR"/>
<enumeration value="A6MLICMS"/>
<enumeration value="A6YEUIRS"/>
<enumeration value="A7YEUIRS"/>
<enumeration value="A8YEUIRS"/>
<enumeration value="A9MEUBOR"/>
<enumeration value="A9MEUCMS"/>
<enumeration value="A9MLIBOR"/>
<enumeration value="A9MLICMS"/>
<enumeration value="A9YEUIRS"/>
<enumeration value="ATG815"/>
<enumeration value="ATG8WBG"/>
<enumeration value="ATGMIN10"/>
<enumeration value="ATGMIN8"/>
<enumeration value="CNOTEC10"/>
<enumeration value="EONIA"/>
<enumeration value="ESBond"/>
<enumeration value="ESTR"/>
<enumeration value="EURR002W"/>
<enumeration value="EUSA10M"/>
<enumeration value="EUSA10Y"/>
<enumeration value="EUSA11M"/>
<enumeration value="EUSA11Y"/>
<enumeration value="EUSA12M"/>
<enumeration value="EUSA12Y"/>
<enumeration value="EUSA13Y"/>
<enumeration value="EUSA14Y"/>
<enumeration value="EUSA15M"/>
<enumeration value="EUSA15Y"/>
<enumeration value="EUSA16Y"/>
<enumeration value="EUSA17Y"/>
<enumeration value="EUSA18M"/>
<enumeration value="EUSA18Y"/>
<enumeration value="EUSA19Y"/>
<enumeration value="EUSA1D"/>
<enumeration value="EUSA1M"/>
<enumeration value="EUSA1W"/>
<enumeration value="EUSA20Y"/>
<enumeration value="EUSA21M"/>
<enumeration value="EUSA27M"/>
<enumeration value="EUSA2M"/>
<enumeration value="EUSA2Y"/>
<enumeration value="EUSA30M"/>
<enumeration value="EUSA33M"/>
<enumeration value="EUSA3M"/>
<enumeration value="EUSA3Y"/>
<enumeration value="EUSA4M"/>
<enumeration value="EUSA4Y"/>
<enumeration value="EUSA5M"/>
<enumeration value="EUSA5Y"/>
<enumeration value="EUSA6M"/>
<enumeration value="EUSA6Y"/>
<enumeration value="EUSA7M"/>
<enumeration value="EUSA7Y"/>
<enumeration value="EUSA8M"/>
<enumeration value="EUSA8Y"/>
<enumeration value="EUSA9M"/>
<enumeration value="EUSA9Y"/>
<enumeration value="MUDRB"/>
<enumeration value="OTHER"/>
<enumeration value="OTHER_NS"/>
<enumeration value="QMUDRB"/>
<enumeration value="QUDRB"/>
<enumeration value="RENDSTATO"/>
<enumeration value="SMUDRB"/>
<enumeration value="SUDRB"/>
<enumeration value="T4M"/>
<enumeration value="TAG"/>
<enumeration value="TAM"/>
<enumeration value="TME"/>
<enumeration value="UDRB"/>
<enumeration value="UDRBQWBG"/>
<enumeration value="YUDRB"/>
</restriction>
</simpleType>
<simpleType name="cl_eurosystem_iso_code">
<restriction base="string">
<enumeration value="AT"/>
<enumeration value="BE"/>
<enumeration value="BG"/>
<enumeration value="CY"/>
<enumeration value="DE"/>
<enumeration value="EE"/>
<enumeration value="ES"/>
<enumeration value="EU"/>
<enumeration value="FI"/>
<enumeration value="FR"/>
<enumeration value="GR"/>
<enumeration value="HR"/>
<enumeration value="IE"/>
<enumeration value="IT"/>
<enumeration value="LT"/>
<enumeration value="LU"/>
<enumeration value="LV"/>
<enumeration value="MT"/>
<enumeration value="NL"/>
<enumeration value="PT"/>
<enumeration value="SI"/>
<enumeration value="SK"/>
</restriction>
</simpleType>
</schema>

2412
airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd Normal file
View File

File diff suppressed because it is too large Load Diff

386
airflow/ods/c2d/uc/config/yaml/c2d_uc_dissem.yaml Normal file
View File

@@ -0,0 +1,386 @@
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/C2D/UC_DISSEM
workflow_name: w_OU_C2D_UC_DISSEM
validation_schema_path: '/opt/airflow/src/airflow/ods/c2d/uc/config/xsd/UseOfCollateralMessage.xsd'
bucket: mrds_inbox_tst
file_type: xml
# List of tasks
tasks:
- task_name: t_ODS_C2D_UC_DISSEM_create_metadata_file
ods_prefix: INBOX/C2D/UC_DISSEM/A_UC_DISSEM_METADATA_LOADS
output_table: A_UC_DISSEM_METADATA_LOADS
namespaces:
ns: 'http://c2d.escb.eu/UseOfCollateralMessage'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '//ns:DisseminationFile/@version'
column_header: 'C2D_VERSION'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:DateCreated'
column_header: 'FILE_CREATION_DATE'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:NumberOfSuspectRecords'
column_header: 'NO_OF_SUSPECT_RECORDS'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:ReportingNCB'
column_header: 'REPORTING_NCB'
is_key: 'N'
- type: 'xpath'
value: '//ns:MetaInformation/ns:SnapshotDate'
column_header: 'SNAPSHOT_DATE'
is_key: 'N'
- type: 'static'
value: 'N'
column_header: 'PROCESSED_TO_DWH'
- task_name: ou_C2D_UC_DISSEM_create_marketable_assets_file
ods_prefix: INBOX/C2D/UC_DISSEM/UC_MA_DISSEM
output_table: UC_MA_DISSEM
namespaces:
ns: 'http://c2d.escb.eu/UseOfCollateralMessage'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '//ns:MetaInformation/ns:ReportingNCB'
is_key: 'Y'
column_header: 'REPORTING_NCB'
- type: 'xpath'
value: '//ns:MetaInformation/ns:SnapshotDate'
is_key: 'Y'
column_header: 'SNAPSHOT_DATE'
- type: 'xpath'
value: '//ns:MetaInformation/ns:DateCreated'
column_header: 'FILE_CREATION_DATE'
is_key: 'Y'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:MFIId'
column_header: 'MFI_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:RegistrationCode/ns:ISINCode'
column_header: 'ISIN_CODE'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'OTHER_REG_NO'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:NominalAmountSubmitted'
column_header: 'NOM_AMT_SUBMITTED'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CollateralValueBeforeHaircuts'
column_header: 'COLL_BEFORE_HAIRCUTS'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CollateralValueAfterHaircuts'
column_header: 'COLL_AFTER_HAIRCUTS'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:TypeOfSystem'
column_header: 'TYPE_OF_SYSTEM'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'TYPE_OF_OPERATION'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:DomesticOrXborder'
column_header: 'DOM_OR_XBORDER'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'ISSUER_CAS'
- type: 'static'
value: ''
column_header: 'ISSUER_CRED_PROVIDER'
- type: 'static'
value: ''
column_header: 'ISSUER_CLASS'
- type: 'static'
value: ''
column_header: 'ISSUER_RATING_ENUM_VALUE'
- type: 'static'
value: ''
column_header: 'ISSUER_RATING_NUMBER_VALUE'
- type: 'static'
value: ''
column_header: 'NCB_COMMENT'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:MobilisationChannel'
column_header: 'MOBILISATION_CHANNEL'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:CCB'
column_header: 'CCB'
is_key: 'N'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:InvestorSSS'
column_header: 'INVESTOR_SSS'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'INTERMEDIARY_SSS'
- type: 'xpath'
value: '//ns:MarketableAssets/ns:MarketableAsset/ns:IssuerSSS'
column_header: 'ISSUER_SSS'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'TRIPARTY_AGENT'
- type: 'static'
value: ''
column_header: 'SUSPECT_ID'
- type: 'static'
value: ''
column_header: 'QUALITY_CHECK_STATUS'
- type: 'static'
value: ''
column_header: 'ERROR_CODE'
- type: 'static'
value: ''
column_header: 'ERROR_MESSAGE'
- type: 'static'
value: ''
column_header: 'ERROR_POSITION_IN_FILE'
- task_name: ou_C2D_UC_DISSEM_create_nonmarketable_assets_file
ods_prefix: INBOX/C2D/UC_DISSEM/UC_NMA_DISSEM
output_table: UC_NMA_DISSEM
namespaces:
ns: 'http://c2d.escb.eu/UseOfCollateralMessage'
xsi: 'http://www.w3.org/2001/XMLSchema-instance'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '//ns:MetaInformation/ns:ReportingNCB'
is_key: 'Y'
column_header: 'REPORTING_NCB'
- type: 'xpath'
value: '//ns:MetaInformation/ns:SnapshotDate'
is_key: 'Y'
column_header: 'SNAPSHOT_DATE'
- type: 'xpath'
value: '//ns:MetaInformation/ns:DateCreated'
column_header: 'FILE_CREATION_DATE'
is_key: 'Y'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:MFIId'
column_header: 'MFI_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:OtherRegistrationNumber'
column_header: 'OTHER_REG_NO'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:TypeOfSystem'
column_header: 'TYPE_OF_SYSTEM'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'TYPE_OF_OPERATION'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:DomesticOrXborder'
column_header: 'DOM_OR_XBORDER'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NonMktAssetType'
column_header: 'NON_MKT_ASSET_TYPE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:DateOfMaturity'
column_header: 'MATURITY_DATE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:InterestPaymentType'
column_header: 'INTEREST_PAYMENT_TYPE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Cap'
column_header: 'CAP'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ReferenceRate'
column_header: 'REFERENCE_RATE'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'REFERENCE_RATE_COMMENT'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CollateralValueBeforeHaircuts'
column_header: 'COLL_BEFORE_HAIRCUTS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CollateralValueAfterHaircuts'
column_header: 'COLL_AFTER_HAIRCUTS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NumberOfAggregatedDebtors'
column_header: 'NO_AGGR_DEBTORS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:EligibleViaGuarantor'
column_header: 'ELIGIBLE_VIA_GUAR'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/@xsi:type'
column_header: 'DEBTOR_TYPE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Name'
column_header: 'DEBTOR_NAME'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:ID/@xsi:type'
column_header: 'DEBTOR_ID_TYPE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:ID/ns:value'
column_header: 'DEBTOR_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Class'
column_header: 'DEBTOR_CLASS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Residence'
column_header: 'DEBTOR_RESIDENCE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:CreditAssessmentSource'
column_header: 'DEBTOR_CAS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:CredAssessSysProvider'
column_header: 'DEBTOR_CRED_PROV'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'DEBTOR_RATING_ENUM_VALUE'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Debtor/ns:Rating/ns:NumberValue'
column_header: 'DEBTOR_RATING_NUMBER_VALUE'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'GUAR_TYPE'
- type: 'static'
value: ''
column_header: 'GUAR_NAME'
- type: 'static'
value: ''
column_header: 'GUAR_ID_TYPE'
- type: 'static'
value: ''
column_header: 'GUAR_ID'
- type: 'static'
value: ''
column_header: 'GUAR_CLASS'
- type: 'static'
value: ''
column_header: 'GUAR_RESIDENCE'
- type: 'static'
value: ''
column_header: 'GUAR_CRED_CAS'
- type: 'static'
value: ''
column_header: 'GUAR_CRED_PROV'
- type: 'static'
value: ''
column_header: 'GUAR_RATING_ENUM_VALUE'
- type: 'static'
value: ''
column_header: 'GUAR_RATING_NUMBER_VALUE'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NumberOfAggregatedAssets'
column_header: 'NO_AGGR_ASSETS'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Denomination'
column_header: 'DENOMINATION'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:Secured'
column_header: 'SECURED_FLAG'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ResidualMaturity'
column_header: 'RESIDUAL_MATURITY'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:BucketSize'
column_header: 'BUCKET_SIZE'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NCBComment'
column_header: 'NCB_COMMENT'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ValuationMethodology'
column_header: 'VALUATION_METHODOLOGY'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:NominalAmountSubmitted'
column_header: 'NOM_AMT_SUBMITTED'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ResettingPeriodMoreThanOneYear'
column_header: 'RESET_PERIOD_MORE_ONE_YEAR'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:MobilisationChannel'
column_header: 'MOBILISATION_CHANNEL'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:CCB'
column_header: 'CCB'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'INVESTOR_SSS'
- type: 'static'
value: ''
column_header: 'INTERMEDIARY_SSS'
- type: 'static'
value: ''
column_header: 'ISSUER_SSS'
- type: 'static'
value: ''
column_header: 'SUSPECT_ID'
- type: 'static'
value: ''
column_header: 'QUALITY_CHECK_STATUS'
- type: 'static'
value: ''
column_header: 'ERROR_CODE'
- type: 'static'
value: ''
column_header: 'ERROR_MESSAGE'
- type: 'static'
value: ''
column_header: 'ERROR_POSITION_IN_FILE'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:OaId'
column_header: 'OA_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:ContractId'
column_header: 'CONTRACT_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns:NonMarketableAssets/ns:NonMarketableAsset/ns:InstrmntId'
column_header: 'INSTRMNT_ID'
is_key: 'N'

179
airflow/ods/c2d/uc/test_data/MOC-CollateralDataDissemination-UCDB_dissemination_DISS-FR-2023-02-16-34270_example.xml Normal file
View File

@@ -0,0 +1,179 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<DisseminationFile version="R14" xmlns="http://c2d.escb.eu/UseOfCollateralMessage">
<MetaInformation>
<ReportingNCB>FR</ReportingNCB>
<SnapshotDate>2023-02-16</SnapshotDate>
<DateCreated>2023-02-23T11:00:35</DateCreated>
<NumberOfSuspectRecords>0</NumberOfSuspectRecords>
</MetaInformation>
<MarketableAssets>
<MarketableAsset>
<MFIId>FR10107</MFIId>
<RegistrationCode xsi:type="isin" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ISINCode>BE6302866973</ISINCode>
</RegistrationCode>
<NominalAmountSubmitted>40</NominalAmountSubmitted>
<CollateralValueBeforeHaircuts>41.92566012</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>40.75174164</CollateralValueAfterHaircuts>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>cross-border</DomesticOrXborder>
<MobilisationChannel>CCBM mkt</MobilisationChannel>
<CCB>BE</CCB>
<InvestorSSS>CLBE01</InvestorSSS>
<IssuerSSS>CLBE01</IssuerSSS>
</MarketableAsset>
<MarketableAsset>
<MFIId>FR10107</MFIId>
<RegistrationCode xsi:type="isin" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ISINCode>DE000A1RQCP0</ISINCode>
</RegistrationCode>
<NominalAmountSubmitted>10</NominalAmountSubmitted>
<CollateralValueBeforeHaircuts>10.2664863</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>10.06115657</CollateralValueAfterHaircuts>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>cross-border</DomesticOrXborder>
<MobilisationChannel>CCBM mkt</MobilisationChannel>
<CCB>DE</CCB>
<InvestorSSS>CLDE01</InvestorSSS>
<IssuerSSS>CLDE01</IssuerSSS>
</MarketableAsset>
<MarketableAsset>
<MFIId>FR10107</MFIId>
<RegistrationCode xsi:type="isin" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ISINCode>ES0305248009</ISINCode>
</RegistrationCode>
<NominalAmountSubmitted>7.5753425</NominalAmountSubmitted>
<CollateralValueBeforeHaircuts>7.31191527</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>6.78545737</CollateralValueAfterHaircuts>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>cross-border</DomesticOrXborder>
<MobilisationChannel>CCBM mkt</MobilisationChannel>
<CCB>ES</CCB>
<InvestorSSS>CLES01</InvestorSSS>
<IssuerSSS>CLES01</IssuerSSS>
</MarketableAsset>
</MarketableAssets>
<NonMarketableAssets>
<NonMarketableAsset>
<MFIId>FR10107</MFIId>
<OtherRegistrationNumber>FRCPACAGGREGATE</OtherRegistrationNumber>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>domestic</DomesticOrXborder>
<NonMktAssetType>Credit claim</NonMktAssetType>
<DateOfMaturity>2023-07-08</DateOfMaturity>
<InterestPaymentType>Fixed</InterestPaymentType>
<CollateralValueBeforeHaircuts>100</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>88.5</CollateralValueAfterHaircuts>
<NumberOfAggregatedDebtors>2</NumberOfAggregatedDebtors>
<EligibleViaGuarantor>N</EligibleViaGuarantor>
<Debtor xsi:type="DG3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<Name>Aggregate</Name>
<ID xsi:type="NCB">
<value>Aggregate</value>
</ID>
<Class>NFC-PSE3</Class>
<Residence>FR</Residence>
<CreditAssessmentSource>ICAS</CreditAssessmentSource>
<CredAssessSysProvider>Banque de France</CredAssessSysProvider>
<Rating>
<NumberValue>0.0003</NumberValue>
</Rating>
</Debtor>
<NumberOfAggregatedAssets>2</NumberOfAggregatedAssets>
<Denomination>EUR</Denomination>
<Secured>N</Secured>
<ResidualMaturity>1-3</ResidualMaturity>
<BucketSize>0_to_100</BucketSize>
<NCBComment>10107</NCBComment>
<ValuationMethodology>Outstanding</ValuationMethodology>
<NominalAmountSubmitted>10</NominalAmountSubmitted>
<MobilisationChannel>Local dom nonmkt</MobilisationChannel>
<CCB>FR</CCB>
<OaId>10107</OaId>
<ContractId>1549493</ContractId>
<InstrmntId>1549493</InstrmntId>
</NonMarketableAsset>
<NonMarketableAsset>
<MFIId>FR11188</MFIId>
<OtherRegistrationNumber>FRC000748968616</OtherRegistrationNumber>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>domestic</DomesticOrXborder>
<NonMktAssetType>ACC</NonMktAssetType>
<DateOfMaturity>2023-09-30</DateOfMaturity>
<InterestPaymentType>Floating</InterestPaymentType>
<Cap>N</Cap>
<ReferenceRate>3MEUBOR</ReferenceRate>
<CollateralValueBeforeHaircuts>200</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>160</CollateralValueAfterHaircuts>
<NumberOfAggregatedDebtors>1</NumberOfAggregatedDebtors>
<EligibleViaGuarantor>N</EligibleViaGuarantor>
<Debtor xsi:type="DG3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<Name>SAS UNIP SAINT MALO AUTOMOBILES DISTRIBUTION</Name>
<ID xsi:type="NCB">
<value>FR895780419</value>
</ID>
<Class>Other</Class>
<Residence>FR</Residence>
<CreditAssessmentSource>ICAS</CreditAssessmentSource>
<CredAssessSysProvider>Banque de France</CredAssessSysProvider>
<Rating>
<NumberValue>0.0051</NumberValue>
</Rating>
</Debtor>
<NumberOfAggregatedAssets>1</NumberOfAggregatedAssets>
<Denomination>EUR</Denomination>
<Secured>N</Secured>
<ResidualMaturity>10-15</ResidualMaturity>
<NCBComment>11188</NCBComment>
<ValuationMethodology>Outstanding</ValuationMethodology>
<NominalAmountSubmitted>999999.99999999</NominalAmountSubmitted>
<ResettingPeriodMoreThanOneYear>N</ResettingPeriodMoreThanOneYear>
<MobilisationChannel>Local dom nonmkt</MobilisationChannel>
<CCB>FR</CCB>
<OaId>11188</OaId>
<ContractId>R05020ETC</ContractId>
<InstrmntId>202095459110</InstrmntId>
</NonMarketableAsset>
<NonMarketableAsset>
<MFIId>FR11188</MFIId>
<OtherRegistrationNumber>FRC000748968732</OtherRegistrationNumber>
<TypeOfSystem>pool</TypeOfSystem>
<DomesticOrXborder>domestic</DomesticOrXborder>
<NonMktAssetType>ACC</NonMktAssetType>
<DateOfMaturity>2023-09-30</DateOfMaturity>
<InterestPaymentType>Floating</InterestPaymentType>
<Cap>N</Cap>
<ReferenceRate>3MEUBOR</ReferenceRate>
<CollateralValueBeforeHaircuts>300</CollateralValueBeforeHaircuts>
<CollateralValueAfterHaircuts>201</CollateralValueAfterHaircuts>
<NumberOfAggregatedDebtors>1</NumberOfAggregatedDebtors>
<EligibleViaGuarantor>N</EligibleViaGuarantor>
<Debtor xsi:type="DG3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<Name>ALLIANCE E.S.D.B</Name>
<ID xsi:type="NCB">
<value>FR347861981</value>
</ID>
<Class>Other</Class>
<Residence>FR</Residence>
<CreditAssessmentSource>ICAS</CreditAssessmentSource>
<CredAssessSysProvider>Banque de France</CredAssessSysProvider>
<Rating>
<NumberValue>0.0051</NumberValue>
</Rating>
</Debtor>
<NumberOfAggregatedAssets>1</NumberOfAggregatedAssets>
<Denomination>EUR</Denomination>
<Secured>N</Secured>
<ResidualMaturity>+30</ResidualMaturity>
<NCBComment>11188</NCBComment>
<ValuationMethodology>Outstanding</ValuationMethodology>
<NominalAmountSubmitted>0</NominalAmountSubmitted>
<ResettingPeriodMoreThanOneYear>N</ResettingPeriodMoreThanOneYear>
<MobilisationChannel>Local dom nonmkt</MobilisationChannel>
<CCB>FR</CCB>
<OaId>11188</OaId>
<ContractId>R05320ETC</ContractId>
<InstrmntId>202095459010</InstrmntId>
</NonMarketableAsset>
</NonMarketableAssets>
</DisseminationFile>

0
airflow/ods/csdb/debt/.gitkeep Normal file
View File

0
airflow/ods/csdb/debt/config/.gitkeep Normal file
View File

398
airflow/ods/csdb/debt/config/m_ODS_CSDB_DEBT_PARSE.yaml Normal file
View File

@@ -0,0 +1,398 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/CentralizedSecuritiesDissemination
archive_prefix: ARCHIVE/CSDB/CentralizedSecuritiesDissemination
workflow_name: w_ODS_CSDB_DEBT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_DEBT_PARSE
ods_prefix: INBOX/CSDB/CentralizedSecuritiesDissemination/CSDB_DEBT
output_table: CSDB_DEBT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'Date last modified'
column_header: 'NEWUPDATED'
- type: 'csv_header'
value: 'Extraction date'
column_header: 'IDLOADDATE_DIM'
- type: 'csv_header'
value: 'ISIN code'
column_header: 'EXTERNALCODE_ISIN'
- type: 'csv_header'
value: 'National instrument code type'
column_header: 'EXTERNALCODETYPE_NC'
- type: 'csv_header'
value: 'National instrument code'
column_header: 'EXTERNALCODE_NATIONAL'
- type: 'csv_header'
value: 'Internal instrument code'
column_header: 'IDIRINSTRUMENT'
- type: 'csv_header'
value: 'Short name'
column_header: 'SHORTNAME'
- type: 'csv_header'
value: 'Bond duration'
column_header: 'VA_BONDDURATION'
- type: 'csv_header'
value: 'Debt type'
column_header: 'IDIRDEBTTYPE'
- type: 'csv_header'
value: 'Asset securitisation type'
column_header: 'IDIRASSETSECTYPE'
- type: 'csv_header'
value: 'CFI classification'
column_header: 'IDIRCLASSIFICATIONCODE_CFI'
- type: 'csv_header'
value: 'Instrument ESA 95 class'
column_header: 'IDIRCLASSIFICATIONCODE_ESAI'
- type: 'csv_header'
value: 'Instrument ESA 95 class - value type'
column_header: 'IDIRCLASSIFICATIONCODE_ESAI_DM'
- type: 'csv_header'
value: 'Nominal currency'
column_header: 'IDIRCURRENCY_NOMINAL'
- type: 'csv_header'
value: 'Amount issued'
column_header: 'AMOUNTISSUED'
- type: 'csv_header'
value: 'Amount outstanding'
column_header: 'AMOUNTOUTSTANDING'
- type: 'csv_header'
value: 'Amount outstanding in EUR'
column_header: 'AMOUNTOUTSTANDING_EUR'
- type: 'csv_header'
value: 'Pool factor'
column_header: 'POOLFACTOR'
- type: 'csv_header'
value: 'Issue price'
column_header: 'ISSUEPRICE'
- type: 'csv_header'
value: 'Issue date'
column_header: 'IDISSUEDATE'
- type: 'csv_header'
value: 'Coupon type'
column_header: 'IDIRCOUPONTYPE'
- type: 'csv_header'
value: 'Last Coupon frequency'
column_header: 'IDIRCOUPONFREQUENCY'
- type: 'csv_header'
value: 'Coupon currency'
column_header: 'IDIRCURRENCY_COUPON'
- type: 'csv_header'
value: 'Last Coupon rate'
column_header: 'COUPONRATE'
- type: 'csv_header'
value: 'Last Coupon date'
column_header: 'COUPONDATE'
- type: 'csv_header'
value: 'Redemption type'
column_header: 'IDIRREDEMPTIONTYPE'
- type: 'csv_header'
value: 'Redemption frequency'
column_header: 'IDIRREDEMPTIONFREQUENCY'
- type: 'csv_header'
value: 'Redemption currency'
column_header: 'IDIRCURRENCY_REDEMPTION'
- type: 'csv_header'
value: 'Redemption price'
column_header: 'REDEMPTIONPRICE'
- type: 'csv_header'
value: 'Maturity date'
column_header: 'IDMATURITYDATE'
- type: 'csv_header'
value: 'Issuer organisation alias type'
column_header: 'IDIRORGANISATIONALIASTYPE_IS'
- type: 'csv_header'
value: 'Issuer organisation alias code'
column_header: 'ISSUERSOURCECODE'
- type: 'csv_header'
value: 'Issuer MFI code'
column_header: 'ISSUEREXTERNALCODE_MFI'
- type: 'csv_header'
value: 'Issuer BIC code'
column_header: 'ISSUEREXTERNALCODE_BIC'
- type: 'csv_header'
value: 'Issuer BEI code'
column_header: 'ISSUEREXTERNALCODE_BEI'
- type: 'csv_header'
value: 'Internal organisation code'
column_header: 'IDIRORGANISATION_ISSUER'
- type: 'csv_header'
value: 'Issuer name'
column_header: 'ISSUERNAME'
- type: 'csv_header'
value: 'Issuer domicile country'
column_header: 'IDIRCOUNTRY'
- type: 'csv_header'
value: 'Issuer domicile country - value type'
column_header: 'IDIRCOUNTRY_DM'
- type: 'csv_header'
value: 'Issuer ESA 95 sector'
column_header: 'IDIRCLASSIFICATIONCODE_ESAO'
- type: 'csv_header'
value: 'Issuer ESA 95 sector - value type'
column_header: 'IDIRCLASSIFICATIONCODE_ESAO_DM'
- type: 'csv_header'
value: 'Issuer NACE sector'
column_header: 'IDIRCLASSIFICATIONCODE_NACE'
- type: 'csv_header'
value: 'Price date'
column_header: 'PUBLICATIONPRICEDATE'
- type: 'csv_header'
value: 'Price value'
column_header: 'PUBLICATIONPRICE'
- type: 'csv_header'
value: 'Price value - type'
column_header: 'PUBLICATIONPRICETYPE'
- type: 'csv_header'
value: 'Quotation basis'
column_header: 'PUBLICATIONPRICEQUOTATIONBASIS'
- type: 'csv_header'
value: 'Monthly average price'
column_header: 'MONTHLYAVERAGEPRICE'
- type: 'csv_header'
value: 'Accrual start date'
column_header: 'ACCRUALSTARTDATE'
- type: 'csv_header'
value: 'Accrued income factor'
column_header: 'DEBTACCRUALDEBTOR'
- type: 'csv_header'
value: 'Accrued income factor - value type'
column_header: 'DEBTACCRUALDEBTOR_DM'
- type: 'csv_header'
value: 'Accrued income (Creditor)'
column_header: 'DEBTACCRUALCREDITOR'
- type: 'csv_header'
value: 'Accrued income (Creditor) - value type'
column_header: 'DEBTACCRUALCREDITOR_TYP'
- type: 'csv_header'
value: 'Accrued interest'
column_header: 'ACCRUEDINTEREST'
- type: 'csv_header'
value: 'Yield to maturity'
column_header: 'YTMNONOPTIONADJUSTED'
- type: 'csv_header'
value: 'ESCB issuer identifier'
column_header: 'ESCB_ISSUER_IDENT'
- type: 'csv_header'
value: 'ESCB issuer identifier type'
column_header: 'VA_ESCBCODETYPE'
- type: 'csv_header'
value: 'Issuer compound ID'
column_header: 'IDUDCMPPARTY'
- type: 'csv_header'
value: 'Amount Oustanding type'
column_header: 'AMOUNTOUTSTANDINGTYPE'
- type: 'csv_header'
value: 'Market Capitalisation'
column_header: 'MARKETCAPITALISATION'
- type: 'csv_header'
value: 'Market Capitalisation in euro'
column_header: 'MARKETCAPITALISATION_EUR'
- type: 'csv_header'
value: 'Security Status'
column_header: 'VA_SECURITYSTATUS'
- type: 'csv_header'
value: 'Instrument suppl class'
column_header: 'VA_INSTRSUPPLEMENTARYCLASS'
- type: 'csv_header'
value: 'Residual maturity class'
column_header: 'VA_RESIDUALMATURITYCLASS'
- type: 'csv_header'
value: 'Is In SEC'
column_header: 'VA_ISINSEC'
- type: 'csv_header'
value: 'Is In EADB'
column_header: 'VA_ISELIGIBLEFOREADB'
- type: 'csv_header'
value: 'Instrument ESA 2010 class'
column_header: 'IDIRCLASSIFICATIONCODE_ESAI10'
- type: 'csv_header'
value: 'Issuer ESA 2010 sector'
column_header: 'IDIRCLASSIFICATIONCODE_ESAO10'
- type: 'csv_header'
value: 'Primary asset classification 2'
column_header: 'IDIRDEBTTYPE_N'
- type: 'csv_header'
value: 'Instruments seniority type'
column_header: 'SENIORITY'
- type: 'csv_header'
value: 'Issuer LEI code'
column_header: 'ISSUEREXTERNALCODE_LEI'
- type: 'csv_header'
value: 'Instrument ESA 2010 class - value type'
column_header: 'INSTR_ESA2010_CLASS_VALUETYPE'
- type: 'csv_header'
value: 'Issuer ESA 2010 class - value type'
column_header: 'ISS_ESA2010_CLASS_VALUETYPE'
- type: 'csv_header'
value: 'Security status date'
column_header: 'VA_SECURITYSTATUSDATE'
- type: 'csv_header'
value: 'Group type'
column_header: 'GROUP_TYPE'
- type: 'csv_header'
value: 'Has embedded option'
column_header: 'HASEMBEDDEDOPTION'
- type: 'csv_header'
value: 'Volume traded'
column_header: 'VOLUMETRADED'
- type: 'csv_header'
value: 'Primary listing name'
column_header: 'PRIMARYLISTINGNAME'
- type: 'csv_header'
value: 'Primary listing residency country'
column_header: 'PRIMARYLISTINGCOUNTRY'
- type: 'csv_header'
value: 'Instrument portfolio flags'
column_header: 'VA_INSTRPORTFLAGS'
- type: 'csv_header'
value: 'Residual maturity'
column_header: 'RESIDUALMATURITY'
- type: 'csv_header'
value: 'Original maturity'
column_header: 'ORIGINAL_MATURITY'
- type: 'csv_header'
value: 'CFIN classification'
column_header: 'IDIRCLASSIFICATIONCODE_CFIN'
- type: 'csv_header'
value: 'First scheduled Coupon date'
column_header: 'COUPONFIRSTPAYMENTDATE'
- type: 'csv_header'
value: 'Last scheduled Coupon date'
column_header: 'COUPONLASTPAYMENTDATE'
- type: 'csv_header'
value: 'Coupon rate underlying ISIN'
column_header: 'COUPONRATEUNDERLYINGCODE_ISIN'
- type: 'csv_header'
value: 'Coupon rate spread'
column_header: 'COUPONRATESPREAD'
- type: 'csv_header'
value: 'Coupon rate multiplier'
column_header: 'COUPONRATEMULTIPLIER'
- type: 'csv_header'
value: 'Coupon rate cap'
column_header: 'COUPONRATECAP'
- type: 'csv_header'
value: 'Coupon rate floor'
column_header: 'COUPONRATEFLOOR'
- type: 'csv_header'
value: 'Issue date tranche'
column_header: 'IDISSUEDATE_TRANCHE'
- type: 'csv_header'
value: 'Issue price tranche'
column_header: 'ISSUEPRICE_TRANCHE'
- type: 'csv_header'
value: 'Is private placement'
column_header: 'VA_ISPRIVATEPLACEMENT'
- type: 'csv_header'
value: 'RIAD code'
column_header: 'RIAD_CODE'
- type: 'csv_header'
value: 'RIAD OUID'
column_header: 'RIAD_OUID'
- type: 'csv_header'
value: 'ESG Flag 1'
column_header: 'ESG1'
- type: 'csv_header'
value: 'ESG Flag 2'
column_header: 'ESG2'
- type: 'csv_header'
value: 'ESG Flag 3'
column_header: 'ESG3'
- type: 'csv_header'
value: 'Strip'
column_header: 'STRIP'
- type: 'csv_header'
value: 'Depository receipt'
column_header: 'DEPOSITORY_RECEIPT'
- type: 'csv_header'
value: 'Rule 144A'
column_header: 'RULE_144A'
- type: 'csv_header'
value: 'Reg S'
column_header: 'REG_S'
- type: 'csv_header'
value: 'Warrant'
column_header: 'WARRANT'
- type: 'csv_header'
value: 'CSEC Relevance - stock'
column_header: 'CSEC_RELEVANCE_STOCK'
- type: 'csv_header'
value: 'CSEC relevance - gross issuance'
column_header: 'CSEC_RELEVANCE_GROSS_ISSUANCE'
- type: 'csv_header'
value: 'CSEC relevance - redemption'
column_header: 'CSEC_RELEVANCE_REDEMPTION'
- type: 'csv_header'
value: 'Accruing coupon'
column_header: 'ACCRUING_COUPON'
- type: 'csv_header'
value: 'Accruing discount'
column_header: 'ACCRUING_DISCOUNT'
- type: 'csv_header'
value: 'STEP Id'
column_header: 'STEPID'
- type: 'csv_header'
value: 'Program Name'
column_header: 'PROGRAMNAME'
- type: 'csv_header'
value: 'Program Ceiling'
column_header: 'PROGRAMCEILING'
- type: 'csv_header'
value: 'Program Status'
column_header: 'PROGRAMSTATUS'
- type: 'csv_header'
value: 'Issuer NACE21 sector'
column_header: 'ISSUERNACE21SECTOR'
- type: 'csv_header'
value: 'Instrument quotation basis'
column_header: 'INSTRUMENTQUOTATIONBASIS'
- type: 'csv_header'
value: 'placeholder 38'
column_header: 'PLACEHOLDER38'
- type: 'csv_header'
value: 'placeholder 39'
column_header: 'PLACEHOLDER39'
- type: 'csv_header'
value: 'placeholder 40'
column_header: 'PLACEHOLDER40'
- type: 'csv_header'
value: 'placeholder 41'
column_header: 'PLACEHOLDER41'
- type: 'csv_header'
value: 'placeholder 42'
column_header: 'PLACEHOLDER42'
- type: 'csv_header'
value: 'placeholder 43'
column_header: 'PLACEHOLDER43'
- type: 'csv_header'
value: 'placeholder 44'
column_header: 'PLACEHOLDER44'
- type: 'csv_header'
value: 'placeholder 45'
column_header: 'PLACEHOLDER45'
- type: 'csv_header'
value: 'placeholder 46'
column_header: 'PLACEHOLDER46'
- type: 'csv_header'
value: 'placeholder 47'
column_header: 'PLACEHOLDER47'
- type: 'csv_header'
value: 'placeholder 48'
column_header: 'PLACEHOLDER48'
- type: 'csv_header'
value: 'placeholder 49'
column_header: 'PLACEHOLDER49'
- type: 'csv_header'
value: 'placeholder 50'
column_header: 'PLACEHOLDER50'

400
airflow/ods/csdb/debt_daily/config/yaml/m_ODS_CSDB_DEBT_DAILY_PARSE.yaml Normal file
View File

@@ -0,0 +1,400 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination
archive_prefix: ARCHIVE/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination
workflow_name: w_ODS_CSDB_DEBT_DAILY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_DEBT_DAILY_PARSE
ods_prefix: INBOX/CSDB/CentralizedSecuritiesDailyReferenceDataDissemination/CSDB_DEBT_DAILY
output_table: CSDB_DEBT_DAILY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'Date last modified'
column_header: 'Date last modified'
- type: 'csv_header'
value: 'Extraction date'
column_header: 'Extraction date'
- type: 'csv_header'
value: 'ISIN code'
column_header: 'ISIN code'
- type: 'csv_header'
value: 'National instrument code type'
column_header: 'National instrument code type'
- type: 'csv_header'
value: 'National instrument code'
column_header: 'National instrument code'
- type: 'csv_header'
value: 'Internal instrument code'
column_header: 'Internal instrument code'
- type: 'csv_header'
value: 'Short name'
column_header: 'Short name'
- type: 'csv_header'
value: 'Bond duration'
column_header: 'Bond duration'
- type: 'csv_header'
value: 'Debt type'
column_header: 'Debt type'
- type: 'csv_header'
value: 'Asset securitisation type'
column_header: 'Asset securitisation type'
- type: 'csv_header'
value: 'CFI classification'
column_header: 'CFI classification'
- type: 'csv_header'
value: 'Instrument ESA 95 class'
column_header: 'Instrument ESA 95 class'
- type: 'csv_header'
value: 'Instrument ESA 95 class - value type'
column_header: 'Instrument ESA 95 class - value type'
- type: 'csv_header'
value: 'Nominal currency'
column_header: 'Nominal currency'
- type: 'csv_header'
value: 'Amount issued'
column_header: 'Amount issued'
- type: 'csv_header'
value: 'Amount outstanding'
column_header: 'Amount outstanding'
- type: 'csv_header'
value: 'Amount outstanding in EUR'
column_header: 'Amount outstanding in EUR'
- type: 'csv_header'
value: 'Pool factor'
column_header: 'Pool factor'
- type: 'csv_header'
value: 'Issue price'
column_header: 'Issue price'
- type: 'csv_header'
value: 'Issue date'
column_header: 'Issue date'
- type: 'csv_header'
value: 'Coupon type'
column_header: 'Coupon type'
- type: 'csv_header'
value: 'Last Coupon frequency'
column_header: 'Last Coupon frequency'
- type: 'csv_header'
value: 'Coupon currency'
column_header: 'Coupon currency'
- type: 'csv_header'
value: 'Last Coupon rate'
column_header: 'Last Coupon rate'
- type: 'csv_header'
value: 'Last Coupon date'
column_header: 'Last Coupon date'
- type: 'csv_header'
value: 'Redemption type'
column_header: 'Redemption type'
- type: 'csv_header'
value: 'Redemption frequency'
column_header: 'Redemption frequency'
- type: 'csv_header'
value: 'Redemption currency'
column_header: 'Redemption currency'
- type: 'csv_header'
value: 'Redemption price'
column_header: 'Redemption price'
- type: 'csv_header'
value: 'Maturity date'
column_header: 'Maturity date'
- type: 'csv_header'
value: 'Issuer organisation alias type'
column_header: 'Issuer organisation alias type'
- type: 'csv_header'
value: 'Issuer organisation alias code'
column_header: 'Issuer organisation alias code'
- type: 'csv_header'
value: 'Issuer MFI code'
column_header: 'Issuer MFI code'
- type: 'csv_header'
value: 'Issuer BIC code'
column_header: 'Issuer BIC code'
- type: 'csv_header'
value: 'Issuer BEI code'
column_header: 'Issuer BEI code'
- type: 'csv_header'
value: 'Internal organisation code'
column_header: 'Internal organisation code'
- type: 'csv_header'
value: 'Issuer name'
column_header: 'Issuer name'
- type: 'csv_header'
value: 'Issuer domicile country'
column_header: 'Issuer domicile country'
- type: 'csv_header'
value: 'Issuer domicile country - value type'
column_header: 'Issuer domicile country - value type'
- type: 'csv_header'
value: 'Issuer ESA 95 sector'
column_header: 'Issuer ESA 95 sector'
- type: 'csv_header'
value: 'Issuer ESA 95 sector - value type'
column_header: 'Issuer ESA 95 sector - value type'
- type: 'csv_header'
value: 'Issuer NACE sector'
column_header: 'Issuer NACE sector'
- type: 'csv_header'
value: 'Price date'
column_header: 'Price date'
- type: 'csv_header'
value: 'Price value'
column_header: 'Price value'
- type: 'csv_header'
value: 'Price value - type'
column_header: 'Price value - type'
- type: 'csv_header'
value: 'Quotation basis'
column_header: 'Quotation basis'
- type: 'csv_header'
value: 'Monthly average price'
column_header: 'Monthly average price'
- type: 'csv_header'
value: 'Accrual start date'
column_header: 'Accrual start date'
- type: 'csv_header'
value: 'Accrued income factor'
column_header: 'Accrued income factor'
- type: 'csv_header'
value: 'Accrued income factor - value type'
column_header: 'Accrued income factor - value type'
- type: 'csv_header'
value: 'Accrued income (Creditor)'
column_header: 'Accrued income (Creditor)'
- type: 'csv_header'
value: 'Accrued income (Creditor) - value type'
column_header: 'Accrued income (Creditor) - value type'
- type: 'csv_header'
value: 'Accrued interest'
column_header: 'Accrued interest'
- type: 'csv_header'
value: 'Yield to maturity'
column_header: 'Yield to maturity'
- type: 'csv_header'
value: 'ESCB issuer identifier'
column_header: 'ESCB issuer identifier'
- type: 'csv_header'
value: 'ESCB issuer identifier type'
column_header: 'ESCB issuer identifier type'
- type: 'csv_header'
value: 'Issuer compound ID'
column_header: 'Issuer compound ID'
- type: 'csv_header'
value: 'Amount Oustanding type'
column_header: 'Amount Oustanding type'
- type: 'csv_header'
value: 'Market Capitalisation'
column_header: 'Market Capitalisation'
- type: 'csv_header'
value: 'Market Capitalisation in euro'
column_header: 'Market Capitalisation in euro'
- type: 'csv_header'
value: 'Security Status'
column_header: 'Security Status'
- type: 'csv_header'
value: 'Instrument suppl class'
column_header: 'Instrument suppl class'
- type: 'csv_header'
value: 'Residual maturity class'
column_header: 'Residual maturity class'
- type: 'csv_header'
value: 'Is In SEC'
column_header: 'Is In SEC'
- type: 'csv_header'
value: 'Is In EADB'
column_header: 'Is In EADB'
- type: 'csv_header'
value: 'Instrument ESA 2010 class'
column_header: 'Instrument ESA 2010 class'
- type: 'csv_header'
value: 'Issuer ESA 2010 sector'
column_header: 'Issuer ESA 2010 sector'
- type: 'csv_header'
value: 'Primary asset classification 2'
column_header: 'Primary asset classification 2'
- type: 'csv_header'
value: 'Instruments seniority type'
column_header: 'Instruments seniority type'
- type: 'csv_header'
value: 'Issuer LEI code'
column_header: 'Issuer LEI code'
- type: 'csv_header'
value: 'Instrument ESA 2010 class - value type'
column_header: 'Instrument ESA 2010 class - value type'
- type: 'csv_header'
value: 'Issuer ESA 2010 class - value type'
column_header: 'Issuer ESA 2010 class - value type'
- type: 'csv_header'
value: 'Security status date'
column_header: 'Security status date'
- type: 'csv_header'
value: 'Group type'
column_header: 'Group type'
- type: 'csv_header'
value: 'Has embedded option'
column_header: 'Has embedded option'
- type: 'csv_header'
value: 'Volume traded'
column_header: 'Volume traded'
- type: 'csv_header'
value: 'Primary listing name'
column_header: 'Primary listing name'
- type: 'csv_header'
value: 'Primary listing residency country'
column_header: 'Primary listing residency country'
- type: 'csv_header'
value: 'Instrument portfolio flags'
column_header: 'Instrument portfolio flags'
- type: 'static'
value: ''
column_header: 'BOND_DURATION'
- type: 'csv_header'
value: 'Residual maturity'
column_header: 'Residual maturity'
- type: 'csv_header'
value: 'Original maturity'
column_header: 'Original maturity'
- type: 'csv_header'
value: 'CFIN classification'
column_header: 'CFIN classification'
- type: 'csv_header'
value: 'First scheduled Coupon date'
column_header: 'First scheduled Coupon date'
- type: 'csv_header'
value: 'Last scheduled Coupon date'
column_header: 'Last scheduled Coupon date'
- type: 'csv_header'
value: 'Coupon rate underlying ISIN'
column_header: 'Coupon rate underlying ISIN'
- type: 'csv_header'
value: 'Coupon rate spread'
column_header: 'Coupon rate spread'
- type: 'csv_header'
value: 'Coupon rate multiplier'
column_header: 'Coupon rate multiplier'
- type: 'csv_header'
value: 'Coupon rate cap'
column_header: 'Coupon rate cap'
- type: 'csv_header'
value: 'Coupon rate floor'
column_header: 'Coupon rate floor'
- type: 'csv_header'
value: 'Issue date tranche'
column_header: 'Issue date tranche'
- type: 'csv_header'
value: 'Issue price tranche'
column_header: 'Issue price tranche'
- type: 'csv_header'
value: 'Is private placement'
column_header: 'Is private placement'
- type: 'csv_header'
value: 'RIAD code'
column_header: 'RIAD code'
- type: 'csv_header'
value: 'RIAD OUID'
column_header: 'RIAD OUID'
- type: 'csv_header'
value: 'ESG Flag 1'
column_header: 'ESG Flag 1'
- type: 'csv_header'
value: 'ESG Flag 2'
column_header: 'ESG Flag 2'
- type: 'csv_header'
value: 'ESG Flag 3'
column_header: 'ESG Flag 3'
- type: 'csv_header'
value: 'Strip'
column_header: 'Strip'
- type: 'csv_header'
value: 'Depository receipt'
column_header: 'Depository receipt'
- type: 'csv_header'
value: 'Rule 144A'
column_header: 'Rule 144A'
- type: 'csv_header'
value: 'Reg S'
column_header: 'Reg S'
- type: 'csv_header'
value: 'Warrant'
column_header: 'Warrant'
- type: 'csv_header'
value: 'CSEC Relevance - stock'
column_header: 'CSEC Relevance - stock'
- type: 'csv_header'
value: 'CSEC relevance - gross issuance'
column_header: 'CSEC relevance - gross issuance'
- type: 'csv_header'
value: 'CSEC relevance - redemption'
column_header: 'CSEC relevance - redemption'
- type: 'csv_header'
value: 'Accruing coupon'
column_header: 'Accruing coupon'
- type: 'csv_header'
value: 'Accruing discount'
column_header: 'Accruing discount'
- type: 'csv_header'
value: 'STEP Id'
column_header: 'STEP Id'
- type: 'csv_header'
value: 'Program Name'
column_header: 'Program Name'
- type: 'csv_header'
value: 'Program Ceiling'
column_header: 'Program Ceiling'
- type: 'csv_header'
value: 'Program Status'
column_header: 'Program Status'
- type: 'csv_header'
value: 'Issuer NACE21 sector'
column_header: 'Issuer NACE21 sector'
- type: 'csv_header'
value: 'Instrument quotation basis'
column_header: 'Instrument quotation basis'
- type: 'csv_header'
value: 'placeholder 38'
column_header: 'placeholder 38'
- type: 'csv_header'
value: 'placeholder 39'
column_header: 'placeholder 39'
- type: 'csv_header'
value: 'placeholder 40'
column_header: 'placeholder 40'
- type: 'csv_header'
value: 'placeholder 41'
column_header: 'placeholder 41'
- type: 'csv_header'
value: 'placeholder 42'
column_header: 'placeholder 42'
- type: 'csv_header'
value: 'placeholder 43'
column_header: 'placeholder 43'
- type: 'csv_header'
value: 'placeholder 44'
column_header: 'placeholder 44'
- type: 'csv_header'
value: 'placeholder 45'
column_header: 'placeholder 45'
- type: 'csv_header'
value: 'placeholder 46'
column_header: 'placeholder 46'
- type: 'csv_header'
value: 'placeholder 47'
column_header: 'placeholder 47'
- type: 'csv_header'
value: 'placeholder 48'
column_header: 'placeholder 48'
- type: 'csv_header'
value: 'placeholder 49'
column_header: 'placeholder 49'
- type: 'csv_header'
value: 'placeholder 50'
column_header: 'placeholder 50'

0
airflow/ods/csdb/full_ratings/.gitkeep Normal file
View File

0
airflow/ods/csdb/full_ratings/config/.gitkeep Normal file
View File

103
airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml Normal file
View File

@@ -0,0 +1,103 @@
encoding_type: latin1
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/FullRatingsDissemination
archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination
workflow_name: w_ODS_CSDB_RATINGS_FULL
##file format
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_INSTR_DESC_FULL_PARSE
ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_INSTR_DESC_FULL
output_table: CSDB_INSTR_DESC_FULL
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'RDB_INSTR_ID'
column_header: 'IDIRINSTRUMENT'
- type: 'csv_header'
value: 'ISIN'
column_header: 'ISIN'
- type: 'csv_header'
value: 'MOO_INSTR_ID'
column_header: 'MOO_INSTR_ID'
- type: 'csv_header'
value: 'SNP_INSTR_ID'
column_header: 'SNP_INSTR_ID'
- type: 'csv_header'
value: 'FTC_INSTR_ID'
column_header: 'FITCH_IDENTIFIER'
- type: 'csv_header'
value: 'DBR_INSTR_ID'
column_header: 'DBRS_IDENTIFIER'
- type: 'csv_header'
value: 'EA_STATUS'
column_header: 'EA_STATUS'
- type: 'csv_header'
value: 'IS_TMS'
column_header: 'IS_TMS'
- type: 'csv_header'
value: 'DBRS_COVERED_BOND_PROGRAM_ID'
column_header: 'DBRS_COVERED_BOND_PROGRAM'
- type: 'csv_header'
value: 'FITCH_PROGRAM_ID'
column_header: 'FITCH_PRG_IDENTIFIER'
- type: 'csv_header'
value: 'MOO_DEAL_NUMBER'
column_header: 'MOO_DEAL_NUMBER'
- type: 'csv_header'
value: 'SNP_PROGRAM_ID'
column_header: 'SNP_PROGRAM_ID'
- type: 'csv_header'
value: 'DBRS_DEBT_TYPE'
column_header: 'IDIRDEBTTYPE'
- type: 'csv_header'
value: 'SNP_DEBT_TYPE'
column_header: 'SNP_DEBT_TYPE'
- type: 'csv_header'
value: 'MOODY_SENIORITY'
column_header: 'MOODY_SENIORITY'
- type: 'csv_header'
value: 'FITCH_DEBT_LEVEL_CODE'
column_header: 'FITCH_DEBT_LEVEL_CODE'
- type: 'csv_header'
value: 'DBRS_RANK_TYPE'
column_header: 'DBRS_RANK_TYPE'
- type: 'csv_header'
value: 'DBRS_SECURITY_TYPE'
column_header: 'DBRS_SECURITY_TYPE'
- type: 'csv_header'
value: 'SCO_DEBT_TYPE'
column_header: 'SCO_DEBT_TYPE'
- type: 'csv_header'
value: 'SCO_INSTR_ID'
column_header: 'SCO_INSTR_ID'
- type: 'csv_header'
value: 'SCO_COVERED_BOND_PROGRAM'
column_header: 'SCO_COVERED_BOND_PROGRAM'
- type: 'csv_header'
value: 'SCO_CATEGORY'
column_header: 'SCO_CATEGORY'
- type: 'csv_header'
value: 'PLACEHOLDER15'
column_header: 'PLACEHOLDER15'
- type: 'csv_header'
value: 'PLACEHOLDER16'
column_header: 'PLACEHOLDER16'
- type: 'csv_header'
value: 'PLACEHOLDER17'
column_header: 'PLACEHOLDER17'
- type: 'csv_header'
value: 'PLACEHOLDER18'
column_header: 'PLACEHOLDER18'
- type: 'csv_header'
value: 'PLACEHOLDER19'
column_header: 'PLACEHOLDER19'
- type: 'csv_header'
value: 'PLACEHOLDER20'
column_header: 'PLACEHOLDER20'

130
airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml Normal file
View File

@@ -0,0 +1,130 @@
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/FullRatingsDissemination
archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination
workflow_name: w_ODS_CSDB_RATINGS_FULL
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_INSTR_RAT_FULL_PARSE
ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_INSTR_RAT_FULL
output_table: CSDB_INSTR_RAT_FULL
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'RDB_INSTR_ID'
column_header: 'RDB_INSTR_ID'
- type: 'csv_header'
value: 'SOURCE'
column_header: 'SOURCE'
- type: 'csv_header'
value: 'RATING_SCHEME'
column_header: 'RATING_SCHEME'
- type: 'csv_header'
value: 'RATING'
column_header: 'RATING'
- type: 'csv_header'
value: 'RATING_DATE'
column_header: 'RATING_DATE'
- type: 'csv_header'
value: 'TIME_HORIZON'
column_header: 'TIME_HORIZON'
- type: 'csv_header'
value: 'CURRENCY_TYPE'
column_header: 'CURRENCY_TYPE'
- type: 'csv_header'
value: 'NOTES'
column_header: 'NOTES'
- type: 'csv_header'
value: 'VALID_FROM'
column_header: 'VALID_FROM'
- type: 'csv_header'
value: 'VALID_UNTIL'
column_header: 'VALID_UNTIL'
- type: 'csv_header'
value: 'RDB_RATINGS_ID'
column_header: 'RDB_RATINGS_ID'
- type: 'csv_header'
value: 'OUTLOOK'
column_header: 'WATCHLIST'
- type: 'csv_header'
value: 'OUTLOOK_DATE'
column_header: 'WATCHLIST_DATE'
- type: 'csv_header'
value: 'WATCHLIST'
column_header: 'OUTLOOK'
- type: 'csv_header'
value: 'WATCHLIST_DATE'
column_header: 'OUTLOOK_DATE'
- type: 'csv_header'
value: 'RATING_ACTION'
column_header: 'RATING_ACTION'
- type: 'csv_header'
value: 'RATING_ACTION_DATE'
column_header: 'RATING_ACTION_DATE'
- type: 'csv_header'
value: 'IS_PRELIMINARY'
column_header: 'IS_PRELIMINARY'
- type: 'csv_header'
value: 'RATING_RAW'
column_header: 'RATING_RAW'
- type: 'csv_header'
value: 'RATING_TYPE'
column_header: 'RATING_TYPE'
- type: 'csv_header'
value: 'ENDORSEMENT_INDICATOR'
column_header: 'ENDORSEMENT_INDICATOR'
- type: 'csv_header'
value: 'LAST_REVIEW_DATE'
column_header: 'LAST_REVIEW_DATE'
- type: 'csv_header'
value: 'PLACEHOLDER6'
column_header: 'PLACEHOLDER6'
- type: 'csv_header'
value: 'PLACEHOLDER7'
column_header: 'PLACEHOLDER7'
- type: 'csv_header'
value: 'PLACEHOLDER8'
column_header: 'PLACEHOLDER8'
- type: 'csv_header'
value: 'PLACEHOLDER9'
column_header: 'PLACEHOLDER9'
- type: 'csv_header'
value: 'PLACEHOLDER10'
column_header: 'PLACEHOLDER10'
- type: 'csv_header'
value: 'PLACEHOLDER11'
column_header: 'PLACEHOLDER11'
- type: 'csv_header'
value: 'PLACEHOLDER12'
column_header: 'PLACEHOLDER12'
- type: 'csv_header'
value: 'PLACEHOLDER13'
column_header: 'PLACEHOLDER13'
- type: 'csv_header'
value: 'PLACEHOLDER14'
column_header: 'PLACEHOLDER14'
- type: 'csv_header'
value: 'PLACEHOLDER15'
column_header: 'PLACEHOLDER15'
- type: 'csv_header'
value: 'PLACEHOLDER16'
column_header: 'PLACEHOLDER16'
- type: 'csv_header'
value: 'PLACEHOLDER17'
column_header: 'PLACEHOLDER17'
- type: 'csv_header'
value: 'PLACEHOLDER18'
column_header: 'PLACEHOLDER18'
- type: 'csv_header'
value: 'PLACEHOLDER19'
column_header: 'PLACEHOLDER19'
- type: 'csv_header'
value: 'PLACEHOLDER20'
column_header: 'PLACEHOLDER20'

106
airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml Normal file
View File

@@ -0,0 +1,106 @@
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/FullRatingsDissemination
archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination
workflow_name: w_ODS_CSDB_RATINGS_FULL
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_ISSUER_DESC_FULL_PARSE
ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_ISSUER_DESC_FULL
output_table: CSDB_ISSUER_DESC_FULL
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'RDB_ISSUER_ID'
column_header: 'RDB_ISSUER_ID'
- type: 'csv_header'
value: 'ISSUER_NAME'
column_header: 'ISSUERNAME'
- type: 'csv_header'
value: 'COUNTRY_DOMICILE'
column_header: 'COUNTRY_DOMICILE'
- type: 'csv_header'
value: 'IS_SOVEREIGN'
column_header: 'IS_SOVEREIGN'
- type: 'csv_header'
value: 'MOO_ISSUER_ID'
column_header: 'MOODY_IDENTIFIER'
- type: 'csv_header'
value: 'SNP_ISSUER_ID'
column_header: 'SNP_ISSUER_ID'
- type: 'csv_header'
value: 'FTC_ISSUER_ID'
column_header: 'FITCH_IDENTIFIER'
- type: 'csv_header'
value: 'DBR_ISSUER_ID'
column_header: 'DBRS_IDENTIFIER'
- type: 'csv_header'
value: 'LEI_ISSUER_ID'
column_header: 'LEI_ISSUER_ID'
- type: 'csv_header'
value: 'RIAD_CODE'
column_header: 'RIAD_CODE'
- type: 'csv_header'
value: 'RIAD_OUID'
column_header: 'RIAD_OUID'
- type: 'csv_header'
value: 'CLASH_GROUP_STATUS'
column_header: 'CLASH_GROUP_STATUS'
- type: 'csv_header'
value: 'SCO_ISSUER_ID'
column_header: 'SCO_ISSUER_ID'
- type: 'csv_header'
value: 'PLACEHOLDER5'
column_header: 'PLACEHOLDER5'
- type: 'csv_header'
value: 'PLACEHOLDER6'
column_header: 'PLACEHOLDER6'
- type: 'csv_header'
value: 'PLACEHOLDER7'
column_header: 'PLACEHOLDER7'
- type: 'csv_header'
value: 'PLACEHOLDER8'
column_header: 'PLACEHOLDER8'
- type: 'csv_header'
value: 'PLACEHOLDER9'
column_header: 'PLACEHOLDER9'
- type: 'csv_header'
value: 'PLACEHOLDER10'
column_header: 'PLACEHOLDER10'
- type: 'csv_header'
value: 'PLACEHOLDER11'
column_header: 'PLACEHOLDER11'
- type: 'csv_header'
value: 'PLACEHOLDER12'
column_header: 'PLACEHOLDER12'
- type: 'csv_header'
value: 'PLACEHOLDER13'
column_header: 'PLACEHOLDER13'
- type: 'csv_header'
value: 'PLACEHOLDER14'
column_header: 'PLACEHOLDER14'
- type: 'csv_header'
value: 'PLACEHOLDER15'
column_header: 'PLACEHOLDER15'
- type: 'csv_header'
value: 'PLACEHOLDER16'
column_header: 'PLACEHOLDER16'
- type: 'csv_header'
value: 'PLACEHOLDER17'
column_header: 'PLACEHOLDER17'
- type: 'csv_header'
value: 'PLACEHOLDER18'
column_header: 'PLACEHOLDER18'
- type: 'csv_header'
value: 'PLACEHOLDER19'
column_header: 'PLACEHOLDER19'
- type: 'csv_header'
value: 'PLACEHOLDER20'
column_header: 'PLACEHOLDER20'

131
airflow/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml Normal file
View File

@@ -0,0 +1,131 @@
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/CSDB/FullRatingsDissemination
archive_prefix: ARCHIVE/CSDB/FullRatingsDissemination
workflow_name: w_ODS_CSDB_RATINGS_FULL
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_CSDB_ISSUER_RAT_FULL_PARSE
ods_prefix: INBOX/CSDB/FullRatingsDissemination/CSDB_ISSUER_RAT_FULL
output_table: CSDB_ISSUER_RAT_FULL
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'RDB_ISSUER_ID'
column_header: 'RDB_ISSUER_ID'
- type: 'csv_header'
value: 'SOURCE'
column_header: 'SOURCE'
- type: 'csv_header'
value: 'RATING_SCHEME'
column_header: 'RATING_SCHEME'
- type: 'csv_header'
value: 'RATING'
column_header: 'RATING'
- type: 'csv_header'
value: 'RATING_DATE'
column_header: 'RATING_DATE'
- type: 'csv_header'
value: 'TIME_HORIZON'
column_header: 'TIME_HORIZON'
- type: 'csv_header'
value: 'CURRENCY_TYPE'
column_header: 'CURRENCY_TYPE'
- type: 'csv_header'
value: 'NOTES'
column_header: 'NOTES'
- type: 'csv_header'
value: 'VALID_FROM'
column_header: 'VALID_FROM'
- type: 'csv_header'
value: 'VALID_UNTIL'
column_header: 'VALID_UNTIL'
- type: 'csv_header'
value: 'RDB_RATINGS_ID'
column_header: 'RDB_RATINGS_ID'
- type: 'csv_header'
value: 'OUTLOOK'
column_header: 'OUTLOOK'
- type: 'csv_header'
value: 'OUTLOOK_DATE'
column_header: 'OUTLOOK_DATE'
- type: 'csv_header'
value: 'WATCHLIST'
column_header: 'WATCHLIST'
- type: 'csv_header'
value: 'WATCHLIST_DATE'
column_header: 'WATCHLIST_DATE'
- type: 'csv_header'
value: 'RATING_ACTION'
column_header: 'RATING_ACTION'
- type: 'csv_header'
value: 'RATING_ACTION_DATE'
column_header: 'RATING_ACTION_DATE'
- type: 'csv_header'
value: 'IS_PRELIMINARY'
column_header: 'IS_PRELIMINARY'
- type: 'csv_header'
value: 'RATING_RAW'
column_header: 'RATING_RAW'
- type: 'csv_header'
value: 'RATING_TYPE'
column_header: 'RATING_TYPE'
- type: 'csv_header'
value: 'ENDORSEMENT_INDICATOR'
column_header: 'ENDORSEMENT_INDICATOR'
- type: 'csv_header'
value: 'LAST_REVIEW_DATE'
column_header: 'LAST_REVIEW_DATE'
- type: 'csv_header'
value: 'PLACEHOLDER6'
column_header: 'PLACEHOLDER6'
- type: 'csv_header'
value: 'PLACEHOLDER7'
column_header: 'PLACEHOLDER7'
- type: 'csv_header'
value: 'PLACEHOLDER8'
column_header: 'PLACEHOLDER8'
- type: 'csv_header'
value: 'PLACEHOLDER9'
column_header: 'PLACEHOLDER9'
- type: 'csv_header'
value: 'PLACEHOLDER10'
column_header: 'PLACEHOLDER10'
- type: 'csv_header'
value: 'PLACEHOLDER11'
column_header: 'PLACEHOLDER11'
- type: 'csv_header'
value: 'PLACEHOLDER12'
column_header: 'PLACEHOLDER12'
- type: 'csv_header'
value: 'PLACEHOLDER13'
column_header: 'PLACEHOLDER13'
- type: 'csv_header'
value: 'PLACEHOLDER14'
column_header: 'PLACEHOLDER14'
- type: 'csv_header'
value: 'PLACEHOLDER15'
column_header: 'PLACEHOLDER15'
- type: 'csv_header'
value: 'PLACEHOLDER16'
column_header: 'PLACEHOLDER16'
- type: 'csv_header'
value: 'PLACEHOLDER17'
column_header: 'PLACEHOLDER17'
- type: 'csv_header'
value: 'PLACEHOLDER18'
column_header: 'PLACEHOLDER18'
- type: 'csv_header'
value: 'PLACEHOLDER19'
column_header: 'PLACEHOLDER19'
- type: 'csv_header'
value: 'PLACEHOLDER20'
column_header: 'PLACEHOLDER20'

420
airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_COORDINATOR.py Normal file
View File

@@ -0,0 +1,420 @@
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.python import get_current_context
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
dag_id = "w_ODS_CSDB_RATINGS_FULL_COORDINATOR"
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
OBJECT_PREFIX = os.getenv("OBJECT_PREFIX", "csdb/ratings/full/")
REPROCESS = (os.getenv("CSDB_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts"
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts"
def _oci_client():
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_processed_map() -> dict[str, float]:
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
return {}
def _list_all_zip_objects(include_processed: bool = False) -> list[dict]:
"""List all zip files in the bucket"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map() if not include_processed else {}
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=OBJECT_PREFIX)
all_items: list[dict] = []
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
if not name or name.endswith('/') or not base:
continue
if not ("STC-FullRatingsDissemination" in base and base.lower().endswith(".zip")):
continue
# Get timestamp
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
# Check if already processed
last_proc_ts = float(processed_map.get(name, 0.0))
is_processed = (ts <= last_proc_ts) if processed_map else False
item = {
"name": name,
"base": base,
"mtime": ts,
"is_processed": is_processed
}
all_items.append(item)
# Sort by timestamp (oldest first)
all_items.sort(key=lambda x: x["mtime"])
return all_items
def _list_new_zip_objects() -> list[dict]:
"""List only new/unprocessed zip files"""
all_items = _list_all_zip_objects(include_processed=False)
# Filter out processed items
new_items = [item for item in all_items if not item.get("is_processed", False)]
logging.info("Found %d new STC-FullRatingsDissemination zip file(s) (sorted oldest to newest)", len(new_items))
return new_items
def _find_specific_zip(filename_pattern: str) -> dict:
"""Find a specific zip file by name pattern"""
all_items = _list_all_zip_objects(include_processed=True)
# Try exact match first
for item in all_items:
if item["base"] == filename_pattern or item["name"] == filename_pattern:
logging.info("Found exact match: %s", item["base"])
return item
# Try partial match
for item in all_items:
if filename_pattern.lower() in item["base"].lower():
logging.info("Found partial match: %s", item["base"])
return item
raise AirflowFailException(f"No zip file found matching pattern: {filename_pattern}")
with DAG(
dag_id=dag_id,
default_args=default_args,
description='CSDB Ratings Full Coordinator: Lists and triggers processing for zip files',
schedule_interval="0 */6 * * *", # Every 6 hours, adjust as needed
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["CSDB", "COORDINATOR", "ODS", "OCI", "RATINGS"],
) as dag:
@af_task(task_id="determine_processing_mode")
def determine_processing_mode(**context):
"""
Determine what to process based on dag_run configuration.
Configuration options:
1. No config or mode='all': Process all new zip files
2. mode='specific' + filename='xxx': Process specific zip file
3. mode='reprocess_all': Reprocess all zip files (including already processed)
4. mode='list_only': Just list available files without processing
5. filenames=['file1.zip', 'file2.zip']: Process specific list of files
"""
conf = context.get('dag_run').conf or {}
mode = conf.get('mode', 'all')
filename = conf.get('filename')
filenames = conf.get('filenames', [])
force_reprocess = conf.get('force_reprocess', False)
limit = conf.get('limit') # Limit number of files to process
logging.info("Processing mode: %s", mode)
logging.info("Configuration: %s", json.dumps(conf, indent=2))
result = {
"mode": mode,
"filename": filename,
"filenames": filenames,
"force_reprocess": force_reprocess,
"limit": limit
}
return result
@af_task(task_id="list_zip_files")
def list_zip_files(mode_config: dict):
"""List zip files based on the processing mode"""
mode = mode_config.get("mode", "all")
filename = mode_config.get("filename")
filenames = mode_config.get("filenames", [])
force_reprocess = mode_config.get("force_reprocess", False)
limit = mode_config.get("limit")
zip_files = []
if mode == "list_only":
# Just list all files for information
all_files = _list_all_zip_objects(include_processed=True)
logging.info("=== Available ZIP Files ===")
for idx, f in enumerate(all_files, 1):
status = "PROCESSED" if f.get("is_processed") else "NEW"
logging.info("%d. [%s] %s (mtime: %s)",
idx, status, f["base"],
datetime.fromtimestamp(f["mtime"]).isoformat())
raise AirflowSkipException("List only mode - no processing triggered")
elif mode == "specific":
# Process a specific file
if not filename:
raise AirflowFailException("mode='specific' requires 'filename' parameter")
zip_file = _find_specific_zip(filename)
zip_files = [zip_file]
logging.info("Processing specific file: %s", zip_file["base"])
elif mode == "specific_list":
# Process a list of specific files
if not filenames:
raise AirflowFailException("mode='specific_list' requires 'filenames' parameter")
for fn in filenames:
try:
zip_file = _find_specific_zip(fn)
zip_files.append(zip_file)
except Exception as e:
logging.warning("Could not find file %s: %s", fn, e)
if not zip_files:
raise AirflowFailException("None of the specified files were found")
logging.info("Processing %d specific files", len(zip_files))
elif mode == "reprocess_all":
# Reprocess all files (including already processed)
all_files = _list_all_zip_objects(include_processed=True)
zip_files = all_files
logging.info("Reprocessing all %d files", len(zip_files))
elif mode == "date_range":
# Process files within a date range
start_date = mode_config.get("start_date")
end_date = mode_config.get("end_date")
if not start_date or not end_date:
raise AirflowFailException("mode='date_range' requires 'start_date' and 'end_date'")
start_ts = datetime.fromisoformat(start_date).timestamp()
end_ts = datetime.fromisoformat(end_date).timestamp()
all_files = _list_all_zip_objects(include_processed=True)
zip_files = [f for f in all_files if start_ts <= f["mtime"] <= end_ts]
logging.info("Found %d files in date range %s to %s",
len(zip_files), start_date, end_date)
else: # mode == "all" or default
# Process all new files
zip_files = _list_new_zip_objects()
if not zip_files:
logging.info("No new zip files to process")
raise AirflowSkipException("No new zip files found")
# Apply limit if specified
if limit and isinstance(limit, int) and limit > 0:
original_count = len(zip_files)
zip_files = zip_files[:limit]
logging.info("Limited processing from %d to %d files", original_count, len(zip_files))
# Sort by timestamp (oldest first)
zip_files.sort(key=lambda x: x["mtime"])
logging.info("Selected %d zip file(s) for processing:", len(zip_files))
for idx, f in enumerate(zip_files, 1):
logging.info("%d. %s (mtime: %s)",
idx, f["base"],
datetime.fromtimestamp(f["mtime"]).isoformat())
return {
"zip_files": zip_files,
"mode": mode,
"force_reprocess": force_reprocess
}
@af_task(task_id="trigger_processing_dags")
def trigger_processing_dags(list_result: dict):
"""Trigger the processing DAG for each zip file sequentially"""
from airflow.api.common.trigger_dag import trigger_dag
from time import sleep
zip_files = list_result.get("zip_files", [])
mode = list_result.get("mode", "all")
force_reprocess = list_result.get("force_reprocess", False)
if not zip_files:
logging.info("No zip files to process")
return []
triggered_runs = []
for idx, zip_file in enumerate(zip_files):
conf = {
"zip_object_name": zip_file["name"],
"zip_base_name": zip_file["base"],
"zip_mtime": zip_file["mtime"],
"sequence_number": idx + 1,
"total_files": len(zip_files),
"processing_mode": mode,
"force_reprocess": force_reprocess,
"is_processed": zip_file.get("is_processed", False)
}
logging.info(f"Triggering processing DAG for file {idx + 1}/{len(zip_files)}: {zip_file['base']}")
try:
run_id = trigger_dag(
dag_id="w_ODS_CSDB_RATINGS_FULL_CORE",
run_id=f"coordinator__{datetime.now().strftime('%Y%m%d_%H%M%S')}__{idx}",
conf=conf,
execution_date=None,
replace_microseconds=False,
)
triggered_runs.append({
"run_id": str(run_id),
"zip_file": zip_file["base"],
"sequence": idx + 1,
"status": "triggered"
})
logging.info(f"Successfully triggered run: {run_id}")
except Exception as e:
logging.error(f"Failed to trigger processing for {zip_file['base']}: {e}")
triggered_runs.append({
"zip_file": zip_file["base"],
"sequence": idx + 1,
"status": "failed",
"error": str(e)
})
# Small delay between triggers to avoid overwhelming the system
sleep(2)
logging.info(f"Triggered {len([r for r in triggered_runs if r.get('status') == 'triggered'])} processing DAG runs")
logging.info(f"Failed to trigger {len([r for r in triggered_runs if r.get('status') == 'failed'])} runs")
return triggered_runs
@af_task(task_id="summary_report")
def summary_report(trigger_result: list):
"""Generate a summary report of triggered runs"""
if not trigger_result:
logging.info("No runs were triggered")
return
successful = [r for r in trigger_result if r.get("status") == "triggered"]
failed = [r for r in trigger_result if r.get("status") == "failed"]
logging.info("=" * 80)
logging.info("PROCESSING SUMMARY")
logging.info("=" * 80)
logging.info(f"Total files: {len(trigger_result)}")
logging.info(f"Successfully triggered: {len(successful)}")
logging.info(f"Failed to trigger: {len(failed)}")
if successful:
logging.info("\nSuccessfully triggered:")
for r in successful:
logging.info(f" - {r['zip_file']} (run_id: {r['run_id']})")
if failed:
logging.info("\nFailed to trigger:")
for r in failed:
logging.info(f" - {r['zip_file']} (error: {r.get('error', 'unknown')})")
logging.info("=" * 80)
return {
"total": len(trigger_result),
"successful": len(successful),
"failed": len(failed)
}
# Build DAG structure
mode_task = determine_processing_mode()
list_task = list_zip_files(mode_task)
trigger_task = trigger_processing_dags(list_task)
summary_task = summary_report(trigger_task)
mode_task >> list_task >> trigger_task >> summary_task
logging.info("CSDB Ratings Full Coordinator DAG ready")

388
airflow/ods/csdb/full_ratings/dags/w_ODS_CSDB_RATINGS_FULL_CORE.py Normal file
View File

@@ -0,0 +1,388 @@
import sys
import os
import json
import logging
import zipfile
from pathlib import Path
from datetime import timedelta, datetime, timezone
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.empty import EmptyOperator
from airflow.operators.python import get_current_context
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/csdb')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = "w_ODS_CSDB_RATINGS_FULL_CORE"
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
OBJECT_PREFIX = os.getenv("OBJECT_PREFIX", "csdb/ratings/full/")
TEMP_DIR = "/tmp/csdb_ratings"
PROCESSED_TS_VAR = "w_ODS_CSDB_RATINGS_FULL_COORDINATOR__processed_objects_ts"
# CSV configurations
CSV_CONFIGS = [
{
"source_filename": "FULL_INSTRUMENT_DESCRIPTION.csv",
"config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_DESC_FULL_PARSE.yaml",
"task_name": "m_ODS_CSDB_RATINGS_FULL_INSTRUMENT_DESCRIPTION"
},
{
"source_filename": "FULL_INSTRUMENT_RATINGS.csv",
"config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_INSTR_RAT_FULL_PARSE.yaml",
"task_name": "m_ODS_CSDB_RATINGS_FULL_INSTRUMENT_RATINGS"
},
{
"source_filename": "FULL_ISSUER_DESCRIPTION.csv",
"config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_DESC_FULL_PARSE.yaml",
"task_name": "m_ODS_CSDB_RATINGS_FULL_ISSUER_DESCRIPTION"
},
{
"source_filename": "FULL_ISSUER_RATINGS.csv",
"config_yaml": "/opt/airflow/src/airflow/dags/ods/csdb/full_ratings/config/m_ODS_CSDB_ISSUER_RAT_FULL_PARSE.yaml",
"task_name": "m_ODS_CSDB_RATINGS_FULL_ISSUER_RATINGS"
}
]
def _oci_client():
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_processed_map() -> dict[str, float]:
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed(zip_key: str, zip_mtime: float):
m = _load_processed_map()
m[zip_key] = float(zip_mtime)
_save_processed_map(m)
logging.info("Marked as processed: %s (mtime=%s)", zip_key, zip_mtime)
with DAG(
dag_id=dag_id,
default_args=default_args,
description='CSDB Ratings Full Processor: Processes one zip file with 4 CSV files in parallel',
schedule_interval=None, # Triggered by coordinator
catchup=False,
max_active_runs=3, # Allow some parallelism but controlled
render_template_as_native_obj=True,
tags=["CSDB", "PROCESSOR", "MRDS", "ODS", "OCI", "RATINGS"],
) as dag:
@af_task(task_id="get_zip_config")
def get_zip_config(**context):
"""Get the zip file configuration from dag_run conf"""
conf = context['dag_run'].conf or {}
zip_object_name = conf.get('zip_object_name')
zip_base_name = conf.get('zip_base_name')
zip_mtime = conf.get('zip_mtime')
sequence_number = conf.get('sequence_number', 0)
total_files = conf.get('total_files', 0)
if not all([zip_object_name, zip_base_name, zip_mtime]):
raise AirflowFailException("Missing required configuration: zip_object_name, zip_base_name, or zip_mtime")
logging.info(f"Processing zip file {sequence_number}/{total_files}: {zip_base_name}")
return {
"zip_object_name": zip_object_name,
"zip_base_name": zip_base_name,
"zip_mtime": zip_mtime,
"sequence_number": sequence_number,
"total_files": total_files
}
@af_task(task_id="download_and_unzip")
def download_and_unzip(config: dict):
"""Download and unzip the specific zip file"""
zip_key = config["zip_object_name"]
zip_base = config["zip_base_name"]
client = _oci_client()
os.makedirs(TEMP_DIR, exist_ok=True)
# Create unique temp directory for this run
run_temp_dir = os.path.join(TEMP_DIR, f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}")
os.makedirs(run_temp_dir, exist_ok=True)
local_zip = os.path.join(run_temp_dir, zip_base)
logging.info("Downloading %s to %s", zip_key, local_zip)
get_obj = client.get_object(OCI_NAMESPACE, OCI_BUCKET, zip_key)
with open(local_zip, 'wb') as f:
for chunk in get_obj.data.raw.stream(1024 * 1024, decode_content=False):
f.write(chunk)
logging.info("Unzipping %s", local_zip)
with zipfile.ZipFile(local_zip, 'r') as zip_ref:
zip_ref.extractall(run_temp_dir)
extracted_files = []
for root, dirs, files in os.walk(run_temp_dir):
for file in files:
if file.endswith('.csv'):
full_path = os.path.join(root, file)
extracted_files.append({"filename": file, "path": full_path})
logging.info("Extracted CSV: %s", file)
logging.info("Total CSV files extracted: %d", len(extracted_files))
return {
"extracted_files": extracted_files,
"zip_config": config,
"temp_dir": run_temp_dir
}
@af_task(task_id="init_workflow")
def init_workflow(unzipped: dict):
"""Initialize MRDS workflow"""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
extracted_files = unzipped.get("extracted_files", [])
zip_config = unzipped.get("zip_config", {})
temp_dir = unzipped.get("temp_dir")
task_configs = []
for csv_config in CSV_CONFIGS:
matching_file = next(
(ef for ef in extracted_files if ef["filename"] == csv_config["source_filename"]),
None
)
if matching_file:
task_configs.append({
"task_name": csv_config["task_name"],
"source_filename": csv_config["source_filename"],
"source_path": matching_file["path"],
"config_file": csv_config["config_yaml"],
})
logging.info("Prepared task config for %s", csv_config["source_filename"])
else:
logging.warning("CSV file %s not found in extracted files", csv_config["source_filename"])
return {
"workflow_history_key": a_workflow_history_key,
"task_configs": task_configs,
"zip_config": zip_config,
"temp_dir": temp_dir
}
def run_mrds_task(task_config: dict, **context):
"""Run MRDS processing for a single CSV file"""
ti = context['ti']
task_name = task_config["task_name"]
source_path = task_config["source_path"]
config_file = task_config["config_file"]
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
if not os.path.exists(source_path):
raise FileNotFoundError(f"Source CSV file not found: {source_path}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_history_key = init_bundle.get('workflow_history_key')
if not workflow_history_key:
raise AirflowFailException("No workflow_history_key from init_workflow")
try:
logging.info(f"{task_name}: Starting MRDS processing for {source_path}")
mrds_main(workflow_history_key, source_path, config_file, generate_workflow_context=False)
logging.info(f"{task_name}: MRDS processing completed successfully")
except Exception as e:
logging.exception(f"{task_name}: MRDS failed on {source_path}")
raise
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize the workflow and mark zip as processed"""
ti = context['ti']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
zip_config = init_bundle.get('zip_config', {})
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
# Check if any CSV task failed
csv_task_ids = [cfg["task_name"] for cfg in CSV_CONFIGS]
dag_run = context['dag_run']
tis = [t for t in dag_run.get_task_instances() if t.task_id in csv_task_ids]
from airflow.utils.state import State
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
# Mark zip as processed
zip_key = zip_config.get("zip_object_name")
zip_mtime = zip_config.get("zip_mtime")
if zip_key and zip_mtime:
_mark_processed(zip_key, zip_mtime)
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
else:
failed_tasks = [ti_i.task_id for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed tasks=%s)",
a_workflow_history_key, failed_tasks)
raise AirflowFailException(f"Workflow failed for tasks: {failed_tasks}")
@af_task(task_id="cleanup_temp_files")
def cleanup_temp_files(**context):
"""Clean up temporary files for this run"""
import shutil
ti = context['ti']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
temp_dir = init_bundle.get('temp_dir')
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
logging.info("Cleaned up temp directory: %s", temp_dir)
@af_task(task_id="move_zip_to_archive")
def move_zip_to_archive(**context):
"""Move processed zip file to archive"""
ti = context['ti']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
zip_config = init_bundle.get('zip_config', {})
zip_key = zip_config.get("zip_object_name")
if not zip_key:
logging.warning("No zip key found, skipping archive")
return
client = _oci_client()
archive_key = zip_key.replace(OBJECT_PREFIX, f"{OBJECT_PREFIX}archive/", 1)
try:
client.copy_object(
OCI_NAMESPACE,
OCI_BUCKET,
{
"sourceObjectName": zip_key,
"destinationRegion": os.getenv("OCI_REGION", "eu-frankfurt-1"),
"destinationNamespace": OCI_NAMESPACE,
"destinationBucket": OCI_BUCKET,
"destinationObjectName": archive_key
}
)
logging.info("Copied to archive: %s -> %s", zip_key, archive_key)
client.delete_object(OCI_NAMESPACE, OCI_BUCKET, zip_key)
logging.info("Deleted from inbox: %s", zip_key)
except Exception as e:
logging.error("Failed to archive zip file %s: %s", zip_key, e)
raise
# Build the DAG structure
config_task = get_zip_config()
unzip_task = download_and_unzip(config_task)
init_task = init_workflow(unzip_task)
# Create CSV processing tasks dynamically
csv_tasks = []
for csv_config in CSV_CONFIGS:
task = PythonOperator(
task_id=csv_config["task_name"],
python_callable=run_mrds_task,
op_kwargs={
"task_config": {
"task_name": csv_config["task_name"],
"source_filename": csv_config["source_filename"],
"source_path": "{{ ti.xcom_pull(task_ids='init_workflow')['task_configs'] | selectattr('task_name', 'equalto', '" + csv_config["task_name"] + "') | map(attribute='source_path') | first }}",
"config_file": csv_config["config_yaml"],
}
},
provide_context=True,
)
csv_tasks.append(task)
finalize_task = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
cleanup_task = cleanup_temp_files()
archive_task = move_zip_to_archive()
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_SUCCESS,
)
# Define task dependencies
config_task >> unzip_task >> init_task >> csv_tasks >> finalize_task >> [cleanup_task, archive_task] >> all_good
logging.info("CSDB Ratings Full Processor DAG ready")

29
airflow/ods/fxcd/BRANCH/config/m_ODS_FXCD_F_BRANCH_PARSE.yml Normal file
View File

@@ -0,0 +1,29 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/BRANCH
archive_prefix: ARCHIVE/FXCD/BRANCH
workflow_name: w_ODS_FXCD_F_BRANCH
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_BRANCH_PARSE
ods_prefix: INBOX/FXCD/BRANCH/FXCD_F_BRANCH
output_table: FXCD_F_BRANCH
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'BRANCH_ID'
column_header: 'BRANCH_ID'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'CTP_ID'
column_header: 'CTP_ID'

53
airflow/ods/fxcd/CLEARER/config/m_ODS_FXCD_F_CLEARER_PARSE.yml Normal file
View File

@@ -0,0 +1,53 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CLEARER
archive_prefix: ARCHIVE/FXCD/CLEARER
workflow_name: w_ODS_FXCD_F_CLEARER
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CLEARER_PARSE
ods_prefix: INBOX/FXCD/CLEARER/FXCD_F_CLEARER
output_table: FXCD_F_CLEARER
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CLEARER_ID'
column_header: 'CLEARER_ID'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_OF_FLAG'
column_header: 'ELIGIBILITY_OF_FLAG'
- type: 'csv_header'
value: 'ELIGIBILITY_FR_FLAG'
column_header: 'ELIGIBILITY_FR_FLAG'
- type: 'csv_header'
value: 'ACTIVE_FLAG'
column_header: 'ACTIVE_FLAG'
- type: 'csv_header'
value: 'OVERALL_OF_LIMIT_AMT'
column_header: 'OVERALL_OF_LIMIT_AMT'
- type: 'csv_header'
value: 'CASH_OF_LIMIT_AMT'
column_header: 'CASH_OF_LIMIT_AMT'
- type: 'csv_header'
value: 'SECURITIES_OF_LIMIT_AMT'
column_header: 'SECURITIES_OF_LIMIT_AMT'
- type: 'csv_header'
value: 'OVERALL_FR_LIMIT_AMT'
column_header: 'OVERALL_FR_LIMIT_AMT'
- type: 'csv_header'
value: 'CASH_FR_LIMIT_AMT'
column_header: 'CASH_FR_LIMIT_AMT'
- type: 'csv_header'
value: 'SECURITIES_FR_LIMIT_AMT'
column_header: 'SECURITIES_FR_LIMIT_AMT'

35
airflow/ods/fxcd/CLEARER_NCB_LIMIT/config/m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,35 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CLEARER_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/CLEARER_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_CLEARER_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CLEARER_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/CLEARER_NCB_LIMIT/FXCD_F_CLEARER_NCB_LIMIT
output_table: FXCD_F_CLEARER_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CLEARER_ID'
column_header: 'CLEARER_ID'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'OVERALL_LIMIT_AMT'
column_header: 'OVERALL_LIMIT_AMT'
- type: 'csv_header'
value: 'CASH_LIMIT_AMT'
column_header: 'CASH_LIMIT_AMT'
- type: 'csv_header'
value: 'SECURITIES_LIMIT_AMT'
column_header: 'SECURITIES_LIMIT_AMT'

29
airflow/ods/fxcd/CONSTANT/config/m_ODS_FXCD_F_CONSTANT_PARSE.yml Normal file
View File

@@ -0,0 +1,29 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CONSTANT
archive_prefix: ARCHIVE/FXCD/CONSTANT
workflow_name: w_ODS_FXCD_F_CONSTANT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CONSTANT_PARSE
ods_prefix: INBOX/FXCD/CONSTANT/FXCD_F_CONSTANT
output_table: FXCD_F_CONSTANT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CONSTANT_NAME'
column_header: 'CONSTANT_NAME'
- type: 'csv_header'
value: 'CONSTANT_VALUE'
column_header: 'CONSTANT_VALUE'
- type: 'csv_header'
value: 'CONSTANT_TYPE'
column_header: 'CONSTANT_TYPE'

70
airflow/ods/fxcd/COUNTERPARTY/config/m_ODS_FXCD_F_COUNTERPARTY_PARSE.yml Normal file
View File

@@ -0,0 +1,70 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/COUNTERPARTY
archive_prefix: ARCHIVE/FXCD/COUNTERPARTY
workflow_name: w_ODS_FXCD_F_COUNTERPARTY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_COUNTERPARTY_PARSE
ods_prefix: INBOX/FXCD/COUNTERPARTY/FXCD_F_COUNTERPARTY
output_table: FXCD_F_COUNTERPARTY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CTP_ID'
column_header: 'CTP_ID'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'CTP_GROUP_FKIT_CODE'
column_header: 'CTP_GROUP_FKIT_CODE'
- type: 'csv_header'
value: 'ACTIVE_FLAG'
column_header: 'ACTIVE_FLAG'
- type: 'csv_header'
value: 'CTP_COMMENT'
column_header: 'CTP_COMMENT'
- type: 'csv_header'
value: 'GUARANTOR_ID'
column_header: 'GUARANTOR_ID'
- type: 'csv_header'
value: 'OF_OVERALL_LMT_AMT'
column_header: 'OF_OVERALL_LMT_AMT'
- type: 'csv_header'
value: 'OF_MANUAL_FLAG'
column_header: 'OF_MANUAL_FLAG'
- type: 'csv_header'
value: 'FR_OVERALL_LMT_AMT'
column_header: 'FR_OVERALL_LMT_AMT'
- type: 'csv_header'
value: 'FR_MANUAL_FLAG'
column_header: 'FR_MANUAL_FLAG'
- type: 'csv_header'
value: 'MP_OVERALL_LMT_AMT'
column_header: 'MP_OVERALL_LMT_AMT'
- type: 'csv_header'
value: 'MP_MANUAL_FLAG'
column_header: 'MP_MANUAL_FLAG'
- type: 'csv_header'
value: 'EOI_CTP_GROUP'
column_header: 'EOI_CTP_GROUP'
- type: 'csv_header'
value: 'ART_101_FLA'
column_header: 'ART_101_FLA'
- type: 'csv_header'
value: 'MEDIAN_CAPITAL_FLAG'
column_header: 'MEDIAN_CAPITAL_FLAG'
- type: 'csv_header'
value: 'CHANGE_DESCRIPTION'
column_header: 'CHANGE_DESCRIPTION'

85
airflow/ods/fxcd/COUNTRY/config/m_ODS_FXCD_F_COUNTRY_PARSE.yml Normal file
View File

@@ -0,0 +1,85 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/COUNTRY
archive_prefix: ARCHIVE/FXCD/COUNTRY
workflow_name: w_ODS_FXCD_F_COUNTRY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_COUNTRY_PARSE
ods_prefix: INBOX/FXCD/COUNTRY/FXCD_F_COUNTRY
output_table: FXCD_F_COUNTRY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'COUNTRY_GDP'
column_header: 'COUNTRY_GDP'
- type: 'csv_header'
value: 'NCB_USD_LAMBDA'
column_header: 'NCB_USD_LAMBDA'
- type: 'csv_header'
value: 'OF_FLAG'
column_header: 'OF_FLAG'
- type: 'csv_header'
value: 'FR_FLAG'
column_header: 'FR_FLAG'
- type: 'csv_header'
value: 'EU_FLAG'
column_header: 'EU_FLAG'
- type: 'csv_header'
value: 'EUROSYSTEM_FLAG'
column_header: 'EUROSYSTEM_FLAG'
- type: 'csv_header'
value: 'FR_PORTF_SHARE_OPTOUT'
column_header: 'FR_PORTF_SHARE_OPTOUT'
- type: 'csv_header'
value: 'FR_LIMIT_CALC_OPTOUT'
column_header: 'FR_LIMIT_CALC_OPTOUT'
- type: 'csv_header'
value: 'COUNTRY_COMMENT'
column_header: 'COUNTRY_COMMENT'
- type: 'csv_header'
value: 'OF_LMT_AMNT'
column_header: 'OF_LMT_AMNT'
- type: 'csv_header'
value: 'OF_MANUAL_FLAG'
column_header: 'OF_MANUAL_FLAG'
- type: 'csv_header'
value: 'FR_LMT_AMNT'
column_header: 'FR_LMT_AMNT'
- type: 'csv_header'
value: 'FR_MANUAL_FLAG'
column_header: 'FR_MANUAL_FLAG'
- type: 'csv_header'
value: 'USD_PORTFOLIO_EUR_SIZE'
column_header: 'USD_PORTFOLIO_EUR_SIZE'
- type: 'csv_header'
value: 'JPY_PORTFOLIO_EUR_SIZE'
column_header: 'JPY_PORTFOLIO_EUR_SIZE'
- type: 'csv_header'
value: 'CAPITAL_KEY_AMNT'
column_header: 'CAPITAL_KEY_AMNT'
- type: 'csv_header'
value: 'LAMBDA_MANUAL_FLAG'
column_header: 'LAMBDA_MANUAL_FLAG'
- type: 'csv_header'
value: 'CNY_PORTFOLIO_EUR_SIZE'
column_header: 'CNY_PORTFOLIO_EUR_SIZE'
- type: 'csv_header'
value: 'CHANGE_DESCRIPTION'
column_header: 'CHANGE_DESCRIPTION'

35
airflow/ods/fxcd/COUNTRY_NCB_LIMIT/config/m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,35 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/COUNTRY_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/COUNTRY_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_COUNTRY_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_COUNTRY_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/COUNTRY_NCB_LIMIT/FXCD_F_COUNTRY_NCB_LIMIT
output_table: FXCD_F_COUNTRY_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'NCB_COUNTRY_ID'
column_header: 'NCB_COUNTRY_ID'
- type: 'csv_header'
value: 'USD_LIMIT_AMT'
column_header: 'USD_LIMIT_AMT'
- type: 'csv_header'
value: 'JPY_LIMIT_AMT'
column_header: 'JPY_LIMIT_AMT'
- type: 'csv_header'
value: 'TOTAL_LIMIT_AMT'
column_header: 'TOTAL_LIMIT_AMT'

35
airflow/ods/fxcd/CTP_GROUP/config/m_ODS_FXCD_F_CTP_GROUP_PARSE.yml Normal file
View File

@@ -0,0 +1,35 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CTP_GROUP
archive_prefix: ARCHIVE/FXCD/CTP_GROUP
workflow_name: w_ODS_FXCD_F_CTP_GROUP
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CTP_GROUP_PARSE
ods_prefix: INBOX/FXCD/CTP_GROUP/FXCD_F_CTP_GROUP
output_table: FXCD_F_CTP_GROUP
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CTP_GROUP_FKIT_CODE'
column_header: 'CTP_GROUP_FKIT_CODE'
- type: 'csv_header'
value: 'CTP_GROUP_NAME'
column_header: 'CTP_GROUP_NAME'
- type: 'csv_header'
value: 'OF_MAX_LIMIT_AMT'
column_header: 'OF_MAX_LIMIT_AMT'
- type: 'csv_header'
value: 'FR_MP_MAX_LIMIT_AMT'
column_header: 'FR_MP_MAX_LIMIT_AMT'
- type: 'csv_header'
value: 'ACTIVE'
column_header: 'ACTIVE'

32
airflow/ods/fxcd/CTP_GROUP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,32 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CTP_GROUP_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/CTP_GROUP_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CTP_GROUP_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/CTP_GROUP_NCB_LIMIT/FXCD_F_CTP_GROUP_NCB_LIMIT
output_table: FXCD_F_CTP_GROUP_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'CTP_GROUP_FKIT_CODE'
column_header: 'CTP_GROUP_FKIT_CODE'
- type: 'csv_header'
value: 'ELIGIBILITY_ID'
column_header: 'ELIGIBILITY_ID'
- type: 'csv_header'
value: 'LIMIT_AMT'
column_header: 'LIMIT_AMT'

32
airflow/ods/fxcd/CTP_NCB_LIMIT/config/m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,32 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/CTP_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/CTP_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_CTP_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_CTP_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/CTP_NCB_LIMIT/FXCD_F_CTP_NCB_LIMIT
output_table: FXCD_F_CTP_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_ID'
column_header: 'ELIGIBILITY_ID'
- type: 'csv_header'
value: 'CTP_ID'
column_header: 'CTP_ID'
- type: 'csv_header'
value: 'LIMIT_AMT'
column_header: 'LIMIT_AMT'

78
airflow/ods/fxcd/ELIGIBILITY/config/m_ODS_FXCD_F_ELIGIBILITY_PARSE.yml Normal file
View File

@@ -0,0 +1,78 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ELIGIBILITY
archive_prefix: ARCHIVE/FXCD/ELIGIBILITY
workflow_name: w_ODS_FXCD_F_ELIGIBILITY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ELIGIBILITY_PARSE
ods_prefix: INBOX/FXCD/ELIGIBILITY/FXCD_F_ELIGIBILITY
output_table: FXCD_F_ELIGIBILITY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ELIGIBILITY_ID'
column_header: 'ELIGIBILITY_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_TYPE'
column_header: 'ELIGIBILITY_TYPE'
- type: 'csv_header'
value: 'ELIGIBILITY_NAME'
column_header: 'ELIGIBILITY_NAME'
- type: 'csv_header'
value: 'LIMIT_PERCENTAGE'
column_header: 'LIMIT_PERCENTAGE'
- type: 'csv_header'
value: 'LIMIT_AMT_MANUAL_FLAG'
column_header: 'LIMIT_AMT_MANUAL_FLAG'
- type: 'csv_header'
value: 'LIMIT_AMT'
column_header: 'LIMIT_AMT'
- type: 'csv_header'
value: 'NCB_LMT_AMT_FLOOR'
column_header: 'NCB_LMT_AMT_FLOOR'
- type: 'csv_header'
value: 'RATING_THRESHOLD'
column_header: 'RATING_THRESHOLD'
- type: 'csv_header'
value: 'FKIT_SUBLIMIT_NAME'
column_header: 'FKIT_SUBLIMIT_NAME'
- type: 'csv_header'
value: 'ELIGIBILITY_ORDER'
column_header: 'ELIGIBILITY_ORDER'
- type: 'csv_header'
value: 'LIMIT_DISTRIBUTION'
column_header: 'LIMIT_DISTRIBUTION'
- type: 'csv_header'
value: 'CTP_MIN_REQ_RATINGS'
column_header: 'CTP_MIN_REQ_RATINGS'
- type: 'csv_header'
value: 'LIMITS_CALCULATION'
column_header: 'LIMITS_CALCULATION'
- type: 'csv_header'
value: 'ART_101_FLAG'
column_header: 'ART_101_FLAG'
- type: 'csv_header'
value: 'DEFINITION'
column_header: 'DEFINITION'
- type: 'csv_header'
value: 'FOR_CP_FLAG'
column_header: 'FOR_CP_FLAG'
- type: 'csv_header'
value: 'FOR_CPG_FLAG'
column_header: 'FOR_CPG_FLAG'
- type: 'csv_header'
value: 'OVERALL_LIMIT_FLAG'
column_header: 'OVERALL_LIMIT_FLAG'
- type: 'csv_header'
value: 'IDENTIFIER'
column_header: 'IDENTIFIER'

35
airflow/ods/fxcd/ELIGIBILITY_GROUP_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE.yml Normal file
View File

@@ -0,0 +1,35 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ELIGIBILITY_GROUP_MAP
archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_GROUP_MAP
workflow_name: w_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ELIGIBILITY_GROUP_MAP_PARSE
ods_prefix: INBOX/FXCD/ELIGIBILITY_GROUP_MAP/FXCD_F_ELIGIBILITY_GROUP_MAP
output_table: FXCD_F_ELIGIBILITY_GROUP_MAP
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CTP_GROUP_FKIT_CODE'
column_header: 'CTP_GROUP_FKIT_CODE'
- type: 'csv_header'
value: 'ELIGIBILITY_ID'
column_header: 'ELIGIBILITY_ID'
- type: 'csv_header'
value: 'OF_LIMIT_AMT'
column_header: 'OF_LIMIT_AMT'
- type: 'csv_header'
value: 'FR_MP_LIMIT_AMT'
column_header: 'FR_MP_LIMIT_AMT'
- type: 'csv_header'
value: 'LIMIT_AMT'
column_header: 'LIMIT_AMT'

44
airflow/ods/fxcd/ELIGIBILITY_ISSUER/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE.yml Normal file
View File

@@ -0,0 +1,44 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER
archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_ISSUER
workflow_name: w_ODS_FXCD_F_ELIGIBILITY_ISSUER
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ELIGIBILITY_ISSUER_PARSE
ods_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER/FXCD_F_ELIGIBILITY_ISSUER
output_table: FXCD_F_ELIGIBILITY_ISSUER
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ELIGIBILITY_ISSUER_ID'
column_header: 'ELIGIBILITY_ISSUER_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_ISSUER_NAME'
column_header: 'ELIGIBILITY_ISSUER_NAME'
- type: 'csv_header'
value: 'ART_101_FLAG'
column_header: 'ART_101_FLAG'
- type: 'csv_header'
value: 'RATING_THRESHOLD'
column_header: 'RATING_THRESHOLD'
- type: 'csv_header'
value: 'DEFINITION'
column_header: 'DEFINITION'
- type: 'csv_header'
value: 'RATING_TERM_TYPE'
column_header: 'RATING_TERM_TYPE'
- type: 'csv_header'
value: 'RATING_UPLIFT_FACTOR'
column_header: 'RATING_UPLIFT_FACTOR'
- type: 'csv_header'
value: 'ELIG_FLAG'
column_header: 'ELIG_FLAG'

32
airflow/ods/fxcd/ELIGIBILITY_ISSUER_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE.yml Normal file
View File

@@ -0,0 +1,32 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER_MAP
archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_ISSUER_MAP
workflow_name: w_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ELIGIBILITY_ISSUER_MAP_PARSE
ods_prefix: INBOX/FXCD/ELIGIBILITY_ISSUER_MAP/FXCD_F_ELIGIBILITY_ISSUER_MAP
output_table: FXCD_F_ELIGIBILITY_ISSUER_MAP
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ISSUER_ID'
column_header: 'ISSUER_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_ISSUER_ID'
column_header: 'ELIGIBILITY_ISSUER_ID'
- type: 'csv_header'
value: 'ELEGIBLE_FLAG'
column_header: 'ELEGIBLE_FLAG'
- type: 'csv_header'
value: 'MANUAL_ELIGIBLE_FLAG'
column_header: 'MANUAL_ELIGIBLE_FLAG'

41
airflow/ods/fxcd/ELIGIBILITY_MAP/config/m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE.yml Normal file
View File

@@ -0,0 +1,41 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ELIGIBILITY_MAP
archive_prefix: ARCHIVE/FXCD/ELIGIBILITY_MAP
workflow_name: w_ODS_FXCD_F_ELIGIBILITY_MAP
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ELIGIBILITY_MAP_PARSE
ods_prefix: INBOX/FXCD/ELIGIBILITY_MAP/FXCD_F_ELIGIBILITY_MAP
output_table: FXCD_F_ELIGIBILITY_MAP
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'CTP_ID'
column_header: 'CTP_ID'
- type: 'csv_header'
value: 'ELIGIBILITY_ID'
column_header: 'ELIGIBILITY_ID'
- type: 'csv_header'
value: 'OF_LIMIT_AMT'
column_header: 'OF_LIMIT_AMT'
- type: 'csv_header'
value: 'OF_FLAG'
column_header: 'OF_FLAG'
- type: 'csv_header'
value: 'FR_LIMIT_AMT'
column_header: 'FR_LIMIT_AMT'
- type: 'csv_header'
value: 'FR_MP_FLAG'
column_header: 'FR_MP_FLAG'
- type: 'csv_header'
value: 'LIMIT_AMT'
column_header: 'LIMIT_AMT'

56
airflow/ods/fxcd/ENTITY/config/m_ODS_FXCD_F_ENTITY_PARSE.yml Normal file
View File

@@ -0,0 +1,56 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ENTITY
archive_prefix: ARCHIVE/FXCD/ENTITY
workflow_name: w_ODS_FXCD_F_ENTITY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ENTITY_PARSE
ods_prefix: INBOX/FXCD/ENTITY/FXCD_F_ENTITY
output_table: FXCD_F_ENTITY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'FKIT_CODE'
column_header: 'FKIT_CODE'
- type: 'csv_header'
value: 'ENTITY_NAME'
column_header: 'ENTITY_NAME'
- type: 'csv_header'
value: 'BVD_BANK_INDEX_NUMBER'
column_header: 'BVD_BANK_INDEX_NUMBER'
- type: 'csv_header'
value: 'RISK_COUNTRY_ID'
column_header: 'RISK_COUNTRY_ID'
- type: 'csv_header'
value: 'ST_MANUAL_RATING_FLAG'
column_header: 'ST_MANUAL_RATING_FLAG'
- type: 'csv_header'
value: 'LT_MANUAL_RATING_FLAG'
column_header: 'LT_MANUAL_RATING_FLAG'
- type: 'csv_header'
value: 'ST_SECOND_BEST_RATING'
column_header: 'ST_SECOND_BEST_RATING'
- type: 'csv_header'
value: 'LT_SECOND_BEST_RATING'
column_header: 'LT_SECOND_BEST_RATING'
- type: 'csv_header'
value: 'CAP_TIER1_AMT'
column_header: 'CAP_TIER1_AMT'
- type: 'csv_header'
value: 'CAP_EQUITY_AMT'
column_header: 'CAP_EQUITY_AMT'
- type: 'csv_header'
value: 'BLOOMBERG_TICKER'
column_header: 'BLOOMBERG_TICKER'

35
airflow/ods/fxcd/EQUIVALENCE_RULE/config/m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE.yml Normal file
View File

@@ -0,0 +1,35 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/EQUIVALENCE_RULE
archive_prefix: ARCHIVE/FXCD/EQUIVALENCE_RULE
workflow_name: w_ODS_FXCD_F_EQUIVALENCE_RULE
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_EQUIVALENCE_RULE_PARSE
ods_prefix: INBOX/FXCD/EQUIVALENCE_RULE/FXCD_F_EQUIVALENCE_RULE
output_table: FXCD_F_EQUIVALENCE_RULE
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'RATING_TERM_TYPE'
column_header: 'RATING_TERM_TYPE'
- type: 'csv_header'
value: 'AGENCY_ID'
column_header: 'AGENCY_ID'
- type: 'csv_header'
value: 'AGENCY_RATING'
column_header: 'AGENCY_RATING'
- type: 'csv_header'
value: 'NUMERICAL_EQUIVALENCE'
column_header: 'NUMERICAL_EQUIVALENCE'
- type: 'csv_header'
value: 'RATING_FACTOR'
column_header: 'RATING_FACTOR'

77
airflow/ods/fxcd/ISSUER/config/m_ODS_FXCD_F_ISSUER_PARSE.yml Normal file
View File

@@ -0,0 +1,77 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ISSUER
archive_prefix: ARCHIVE/FXCD/ISSUER
workflow_name: w_ODS_FXCD_F_ISSUER
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ISSUER_PARSE
ods_prefix: INBOX/FXCD/ISSUER/FXCD_F_ISSUER
output_table: FXCD_F_ISSUER
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ISSUER_ID'
column_header: 'ISSUER_ID'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'ACTIVE_FLAG'
column_header: 'ACTIVE_FLAG'
- type: 'csv_header'
value: 'ISSUER_COMMENT'
column_header: 'ISSUER_COMMENT'
- type: 'csv_header'
value: 'OFM_ISSUER_LMT'
column_header: 'OFM_ISSUER_LMT'
- type: 'csv_header'
value: 'OF_MANUAL_FLAG'
column_header: 'OF_MANUAL_FLAG'
- type: 'csv_header'
value: 'GLOBAL_LMT_AMNT'
column_header: 'GLOBAL_LMT_AMNT'
- type: 'csv_header'
value: 'MANUAL_GLOBAL_LMT_FLAG'
column_header: 'MANUAL_GLOBAL_LMT_FLAG'
- type: 'csv_header'
value: 'OF_ISSUER_CATEGORY'
column_header: 'OF_ISSUER_CATEGORY'
- type: 'csv_header'
value: 'FR_ISSUER_CATEGORY'
column_header: 'FR_ISSUER_CATEGORY'
- type: 'csv_header'
value: 'ISSUER_LMT'
column_header: 'ISSUER_LMT'
- type: 'csv_header'
value: 'FR_MANUAL_FLAG'
column_header: 'FR_MANUAL_FLAG'
- type: 'csv_header'
value: 'NCB_USD_LAMBDA'
column_header: 'NCB_USD_LAMBDA'
- type: 'csv_header'
value: 'ART_101_FLAG'
column_header: 'ART_101_FLAG'
- type: 'csv_header'
value: 'UPLIFT_IMPLIED_RATING'
column_header: 'UPLIFT_IMPLIED_RATING'
- type: 'csv_header'
value: 'LAMBDA_MANUAL_FLAG'
column_header: 'LAMBDA_MANUAL_FLAG'
- type: 'csv_header'
value: 'CHANGE_DESCRIPTION'
column_header: 'CHANGE_DESCRIPTION'
- type: 'csv_header'
value: 'LT_FRM_LIMIT_RATIO'
column_header: 'LT_FRM_LIMIT_RATIO'
- type: 'csv_header'
value: 'LT_FRM_LIMIT_RATIO_MANUAL_FLAG'
column_header: 'LT_FRM_LIMIT_RATIO_MANUAL_FLAG'

38
airflow/ods/fxcd/ISSUER_NCB_LIMIT/config/m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,38 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/ISSUER_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/ISSUER_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_ISSUER_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_ISSUER_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/ISSUER_NCB_LIMIT/FXCD_F_ISSUER_NCB_LIMIT
output_table: FXCD_F_ISSUER_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'ISSUER_ID'
column_header: 'ISSUER_ID'
- type: 'csv_header'
value: 'USD_LIMIT_AMT'
column_header: 'USD_LIMIT_AMT'
- type: 'csv_header'
value: 'JPY_LIMIT_AMT'
column_header: 'JPY_LIMIT_AMT'
- type: 'csv_header'
value: 'TOTAL_LIMIT_AMT'
column_header: 'TOTAL_LIMIT_AMT'
- type: 'csv_header'
value: 'LIMIT_TYPE'
column_header: 'LIMIT_TYPE'

29
airflow/ods/fxcd/LIQUIDITY_NCB_LIMIT/config/m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE.yml Normal file
View File

@@ -0,0 +1,29 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/LIQUIDITY_NCB_LIMIT
archive_prefix: ARCHIVE/FXCD/LIQUIDITY_NCB_LIMIT
workflow_name: w_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_LIQUIDITY_NCB_LIMIT_PARSE
ods_prefix: INBOX/FXCD/LIQUIDITY_NCB_LIMIT/FXCD_F_LIQUIDITY_NCB_LIMIT
output_table: FXCD_F_LIQUIDITY_NCB_LIMIT
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'USD_NCB_MIN_LIMIT_AMT'
column_header: 'USD_NCB_MIN_LIMIT_AMT'
- type: 'csv_header'
value: 'USD_NCB_MAX_LIMIT_AMT'
column_header: 'USD_NCB_MAX_LIMIT_AMT'

47
airflow/ods/fxcd/NCB_PORTFOLIO_SHARE/config/m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE.yml Normal file
View File

@@ -0,0 +1,47 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/NCB_PORTFOLIO_SHARE
archive_prefix: ARCHIVE/FXCD/NCB_PORTFOLIO_SHARE
workflow_name: w_ODS_FXCD_F_NCB_PORTFOLIO_SHARE
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_NCB_PORTFOLIO_SHARE_PARSE
ods_prefix: INBOX/FXCD/NCB_PORTFOLIO_SHARE/FXCD_F_NCB_PORTFOLIO_SHARE
output_table: FXCD_F_NCB_PORTFOLIO_SHARE
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'COUNTRY_ID'
column_header: 'COUNTRY_ID'
- type: 'csv_header'
value: 'COUNTRY_SHARE'
column_header: 'COUNTRY_SHARE'
- type: 'csv_header'
value: 'USD_COUNTRY_SHARE'
column_header: 'USD_COUNTRY_SHARE'
- type: 'csv_header'
value: 'JPY_COUNTRY_SHARE'
column_header: 'JPY_COUNTRY_SHARE'
- type: 'csv_header'
value: 'SCALED_COUNTRY_SHARE'
column_header: 'SCALED_COUNTRY_SHARE'
- type: 'csv_header'
value: 'SCALED_USD_COUNTRY_SHARE'
column_header: 'SCALED_USD_COUNTRY_SHARE'
- type: 'csv_header'
value: 'SCALED_JPY_COUNTRY_SHARE'
column_header: 'SCALED_JPY_COUNTRY_SHARE'
- type: 'csv_header'
value: 'CNY_COUNTRY_SHARE'
column_header: 'CNY_COUNTRY_SHARE'
- type: 'csv_header'
value: 'SCALED_CNY_COUNTRY_SHARE'
column_header: 'SCALED_CNY_COUNTRY_SHARE'

32
airflow/ods/fxcd/RATING/config/m_ODS_FXCD_F_RATING_PARSE.yml Normal file
View File

@@ -0,0 +1,32 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/RATING
archive_prefix: ARCHIVE/FXCD/RATING
workflow_name: w_ODS_FXCD_F_RATING
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_RATING_PARSE
ods_prefix: INBOX/FXCD/RATING/FXCD_F_RATING
output_table: FXCD_F_RATING
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'ENTITY_ID'
column_header: 'ENTITY_ID'
- type: 'csv_header'
value: 'RATING_TERM_TYPE'
column_header: 'RATING_TERM_TYPE'
- type: 'csv_header'
value: 'AGENCY_ID'
column_header: 'AGENCY_ID'
- type: 'csv_header'
value: 'AGENCY_RATING'
column_header: 'AGENCY_RATING'

29
airflow/ods/fxcd/RATING_AGENCY/config/m_ODS_FXCD_F_RATING_AGENCY_PARSE.yml Normal file
View File

@@ -0,0 +1,29 @@
# App configurations
encoding_type: latin1
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/FXCD/RATING_AGENCY
archive_prefix: ARCHIVE/FXCD/RATING_AGENCY
workflow_name: w_ODS_FXCD_F_RATING_AGENCY
validation_schema_path: None
file_type: csv
tasks:
- task_name: m_ODS_FXCD_F_RATING_AGENCY_PARSE
ods_prefix: INBOX/FXCD/RATING_AGENCY/FXCD_F_RATING_AGENCY
output_table: FXCD_F_RATING_AGENCY
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'AGENCY_ID'
column_header: 'AGENCY_ID'
- type: 'csv_header'
value: 'AGENCY_NAME'
column_header: 'AGENCY_NAME'
- type: 'csv_header'
value: 'ENABLED_FLAG'
column_header: 'ENABLED_FLAG'

0
airflow/ods/lm/adhoc_adjustments/config/.gitkeep Normal file
View File

129
airflow/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd Normal file
View File

@@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/csm-adjustment"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/csm-adjustment"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="adjustmentMessages">
<xs:complexType>
<xs:choice>
<xs:element ref="csmAdjustmentMessage" />
<xs:element ref="quarterlyRevaluationAdjustmentMessage" />
<xs:element ref="adhocAdjustmentMessage" />
</xs:choice>
</xs:complexType>
</xs:element>
<xs:element name="csmAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="month" type="month" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentSingleForecast" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="quarterlyRevaluationAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="quarter" type="quarter" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adhocAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="date" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:complexType name="baseAdjustment">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="effectiveDate" type="xs:date" />
<xs:element name="lastDateNotInForecast" type="xs:date" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="adjustmentSingleForecast">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="adjustmentMultipleForecasts">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="forecastItem">
<xs:sequence>
<xs:element name="forecastName" type="lm:forecastName" />
<xs:element name="adjustmentAmount" type="extendedDecimalEuroValue" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="extendedDecimalEuroValue">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="8" />
<xs:minInclusive value="-999999999.99999999" />
<xs:maxInclusive value="999999999.99999999" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="month">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="12" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="quarter">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="4" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

78
airflow/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml Normal file
View File

@@ -0,0 +1,78 @@
tmpdir: /tmp
inbox_prefix: INBOX/LM/AdhocAdjustment
archive_prefix: ARCHIVE/LM/AdhocAdjustment
workflow_name: w_ODS_LM_ADHOC_ADJUSTMENT_MSG
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/adhoc_adjustments/config/adhoc_adjustments.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_HEADER_PARSE
ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_HEADER
output_table: LM_ADHOC_ADJUSTMENTS_HEADER
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:header/ns2:date'
column_header: 'ADJUSTMENT_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:header/ns2:version'
column_header: 'VERSION'
is_key: 'N'
- task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_ITEM_HEADER_PARSE
ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_ITEM_HEADER
output_table: LM_ADHOC_ADJUSTMENTS_ITEM_HEADER
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:effectiveDate'
column_header: 'EFFECTIVE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast'
column_header: 'LAST_DATE_NOT_FORECAST'
is_key: 'N'
- task_name: m_ODS_LM_ADHOC_ADJUSTMENTS_ITEM_PARSE
ods_prefix: INBOX/LM/AdhocAdjustment/LM_ADHOC_ADJUSTMENTS_ITEM
output_table: LM_ADHOC_ADJUSTMENTS_ITEM
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName'
column_header: 'FORECAST_NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:adhocAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount'
column_header: 'ADJUSTMENT_AMOUNT'
is_key: 'N'

520
airflow/ods/lm/adhoc_adjustments/dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/w_ODS_LM_ADHOC_ADJUSTMENT_MSG.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/adhoc_adjustments/config/m_ODS_LM_ADHOC_ADJUSTMENT_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "ADHOC_ADJUSTMENT"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_ADHOC_ADJUSTMENT"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_ADHOC_ADJUSTMENT"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_ADHOC_ADJUSTMENT"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_ADHOC_ADJUSTMENT", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_ADHOC_ADJUSTMENT",
trigger_dag_id="w_MOPDB_LM_ADHOC_ADJUSTMENT",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

0
airflow/ods/lm/balancesheet/config/.gitkeep Normal file
View File

102
airflow/ods/lm/balancesheet/config/balancesheet.xsd Normal file
View File

@@ -0,0 +1,102 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/balancesheet"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/balancesheet"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="balanceSheetMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="cbBalanceSheet" type="cbBalanceSheet" minOccurs="1" maxOccurs="unbounded" />
<xs:element name="eurosystemBalanceSheet" type="eurosystemBalanceSheet" minOccurs="0" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:complexType name="cbHeader">
<xs:complexContent>
<xs:extension base="eurosystemHeader">
<xs:sequence>
<xs:element name="status" type="status" />
<xs:element name="freeText" type="lm:freeText" minOccurs="0" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="eurosystemHeader">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="referenceDate" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
<xs:element name="dateOfTransmission" type="xs:date" minOccurs="0" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="cbBalanceSheet">
<xs:annotation>
<xs:documentation>Represents a balanceSheet for a CB</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="cbHeader" />
<xs:element name="assets" type="balanceSheetItem" minOccurs="1" />
<xs:element name="liabilities" type="balanceSheetItem" minOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="eurosystemBalanceSheet">
<xs:annotation>
<xs:documentation>Represents an aggregated balanceSheet of all CBs</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="eurosystemHeader" />
<xs:element name="assets" type="balanceSheetItem" minOccurs="1" />
<xs:element name="liabilities" type="balanceSheetItem" minOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="balanceSheetItem">
<xs:sequence>
<xs:element name="level" type="lm:positiveInt" />
<xs:element name="position" type="lm:positiveInt" />
<xs:element name="itemType" type="itemType" minOccurs="0" />
<xs:element name="fullyQualifiedPosition" type="fullyQualifiedPosition" minOccurs="0" />
<xs:element name="name" type="itemName" minOccurs="0" />
<xs:element name="amount" type="lm:amountInEuro" />
<xs:element name="item" type="balanceSheetItem" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="itemName">
<xs:restriction base="xs:string">
<xs:maxLength value="200" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="itemType">
<xs:restriction base="xs:string">
<xs:enumeration value="Asset" />
<xs:enumeration value="Liability" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="fullyQualifiedPosition">
<xs:restriction base="xs:string">
<xs:pattern value="(A|L)(\.([1-9][0-9]*))+" />
<xs:maxLength value="200" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="status">
<xs:restriction base="xs:string">
<xs:enumeration value="U" />
<xs:enumeration value="B" />
<xs:enumeration value="R" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

82
airflow/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml Normal file
View File

@@ -0,0 +1,82 @@
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem
archive_prefix: ARCHIVE/LM/AggregatedDailyFinancialStatementOfTheEurosystem
workflow_name: w_ODS_LM_BALANCESHEET
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/balancesheet/config/balancesheet.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_BALANCESHEET_HEADER_PARSE
ods_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem/LM_BALANCESHEET_HEADER
output_table: LM_BALANCESHEET_HEADER
namespaces:
ns2: 'http://escb.ecb.int/balancesheet'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:balanceSheetMessage/ns2:*'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:referenceDate'
column_header: 'REFERENCE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:version'
column_header: 'VERSION'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:status'
column_header: 'STATUS'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage/ns2:*/ns2:header/ns2:freeText'
column_header: 'FREE_TEXT'
is_key: 'N'
- task_name: m_ODS_LM_BALANCESHEET_ITEM_PARSE
ods_prefix: INBOX/LM/AggregatedDailyFinancialStatementOfTheEurosystem/LM_BALANCESHEET_ITEM
output_table: LM_BALANCESHEET_ITEM
namespaces:
ns2: 'http://escb.ecb.int/balancesheet'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:balanceSheetMessage/ns2:*'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:level'
is_key: 'N'
column_header: 'ITEM_LEVEL'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:position'
is_key: 'N'
column_header: 'POSITION'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:itemType'
is_key: 'N'
column_header: 'ITEM_TYPE'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:fullyQualifiedPosition'
column_header: 'FULLY_QUALIFIED_POSITION'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:name'
column_header: 'NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:balanceSheetMessage//ns2:amount'
column_header: 'AMOUNT'
is_key: 'N'

520
airflow/ods/lm/balancesheet/dags/w_ODS_LM_BALANCESHEET.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/m_ODS_LM_BALANCESHEET.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/balancesheet/config/m_ODS_LM_BALANCESHEET_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "BALANCESHEET"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_BALANCESHEET"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_BALANCESHEET"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_BALANCESHEET"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_BALANCESHEET", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_BALANCESHEET",
trigger_dag_id="w_MOPDB_LM_BALANCESHEET",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

0
airflow/ods/lm/csm_adjustment/config/.gitkeep Normal file
View File

129
airflow/ods/lm/csm_adjustment/config/csm_adjustment.xsd Normal file
View File

@@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/csm-adjustment"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/csm-adjustment"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="adjustmentMessages">
<xs:complexType>
<xs:choice>
<xs:element ref="csmAdjustmentMessage" />
<xs:element ref="quarterlyRevaluationAdjustmentMessage" />
<xs:element ref="adhocAdjustmentMessage" />
</xs:choice>
</xs:complexType>
</xs:element>
<xs:element name="csmAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="month" type="month" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentSingleForecast" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="quarterlyRevaluationAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="quarter" type="quarter" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adhocAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="date" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:complexType name="baseAdjustment">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="effectiveDate" type="xs:date" />
<xs:element name="lastDateNotInForecast" type="xs:date" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="adjustmentSingleForecast">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="adjustmentMultipleForecasts">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="forecastItem">
<xs:sequence>
<xs:element name="forecastName" type="lm:forecastName" />
<xs:element name="adjustmentAmount" type="extendedDecimalEuroValue" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="extendedDecimalEuroValue">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="8" />
<xs:minInclusive value="-999999999.99999999" />
<xs:maxInclusive value="999999999.99999999" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="month">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="12" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="quarter">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="4" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

83
airflow/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml Normal file
View File

@@ -0,0 +1,83 @@
tmpdir: /tmp
inbox_prefix: INBOX/LM/MonthlyCSMAdjustment
archive_prefix: ARCHIVE/LM/MonthlyCSMAdjustment
workflow_name: w_ODS_LM_CSM_ADJUSTMENT_MSG
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/csm_adjustment/config/csm_adjustment.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_CSM_ADJUSTMENT_HEADER_PARSE
ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_HEADER
output_table: LM_CSM_ADJUSTMENTS_HEADER
namespaces:
ns2: http://escb.ecb.int/csm-adjustment
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:year'
column_header: 'YEAR'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:month'
column_header: 'MONTH'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:header/ns2:version'
column_header: 'VERSION'
is_key: 'N'
- task_name: m_ODS_CSM_ADJUSTMENT_ITEM_HEADER_PARSE
ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_ITEM_HEADER
output_table: LM_CSM_ADJUSTMENTS_ITEM_HEADER
namespaces:
ns2: http://escb.ecb.int/csm-adjustment
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:effectiveDate'
column_header: 'EFFECTIVE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast'
column_header: 'LAST_DATE_NOT_FORECAST'
is_key: 'N'
- task_name: m_ODS_CSM_ADJUSTMENT_ITEM_PARSE
ods_prefix: INBOX/LM/MonthlyCSMAdjustment/LM_CSM_ADJUSTMENTS_ITEM
output_table: LM_CSM_ADJUSTMENTS_ITEM
namespaces:
ns2: http://escb.ecb.int/csm-adjustment
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName'
column_header: 'FORECAST_NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:csmAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount'
column_header: 'ADJUSTMENT_AMOUNT'
is_key: 'N'

520
airflow/ods/lm/csm_adjustment/dags/w_ODS_LM_CSM_ADJUSTMENT_MSG.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/m_ODS_LM_CSM_ADJUSTMENT.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/csm_adjustment/config/m_ODS_LM_CSM_ADJUSTMENT_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "CSM_ADJUTMENT"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_CSM_ADJUSTMENT"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_CSM_ADJUSTMENT"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_CSM_ADJUSTMENT"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_CSM_ADJUSTMENT", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_CSM_ADJUSTMENT",
trigger_dag_id="w_MOPDB_LM_CSM_ADJUSTMENT",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

0
airflow/ods/lm/current_accounts/config/.gitkeep Normal file
View File

106
airflow/ods/lm/current_accounts/config/current_accounts.xsd Normal file
View File

@@ -0,0 +1,106 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/ca"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/ca"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="currentAccountMrrMessage">
<xs:complexType>
<xs:choice>
<xs:element name="cbCurrentAccountMrr" type="cbCurrentAccountMrr" minOccurs="1" maxOccurs="unbounded" />
<xs:element name="disaggregatedCurrentAccountMrr" type="disaggregatedCurrentAccountMrr" />
</xs:choice>
</xs:complexType>
</xs:element>
<xs:complexType name="cbCurrentAccountMrr">
<xs:annotation>
<xs:documentation>Represents current accounts of a CB</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="cbHeader" />
<xs:element name="currentAccountMrrs" type="currentAccountMrrs" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="cbHeader">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="referenceDate" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
<xs:element name="periodicity" type="periodicity" />
<xs:element name="freeText" type="lm:freeText" minOccurs="0" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="currentAccountMrrs">
<xs:sequence>
<xs:element name="currentAccountMrr" type="currentAccountMrr" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:group name="currentAccountMrrBase">
<xs:sequence>
<xs:element name="mfiCode" type="lm:mfiCode" />
<xs:element name="bankName" type="lm:bankName" />
<xs:element name="currentAccount" type="lm:decimalEuroValue" />
<xs:element name="minimumReserveRequirement" type="lm:decimalEuroValue" />
<xs:element name="comment" type="lm:comment" minOccurs="0" />
</xs:sequence>
</xs:group>
<xs:complexType name="currentAccountMrr">
<xs:group ref="currentAccountMrrBase" />
</xs:complexType>
<xs:complexType name="disaggregatedCurrentAccountMrr">
<xs:annotation>
<xs:documentation>Represents the disaggregated current accounts</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="disaggregatedHeader" />
<xs:element name="currentAccountMrrs" type="disaggregatedCurrentAccountMrrs" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="disaggregatedHeader">
<xs:sequence>
<xs:element name="referenceDate" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
<xs:element name="periodicity" type="periodicity" />
<xs:element name="currentAccountBSTotal" type="lm:decimalEuroValue" />
<xs:element name="mrrForecastTotal" type="lm:decimalEuroValue" />
<xs:element name="currentAccountMessageTotal" type="lm:decimalEuroValue" />
<xs:element name="mrrMessageTotal" type="lm:decimalEuroValue" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="disaggregatedCurrentAccountMrrs">
<xs:sequence>
<xs:element name="currentAccountMrr" type="currentAccountMrrDisaggregated" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="currentAccountMrrDisaggregated">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:group ref="currentAccountMrrBase" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="periodicity">
<xs:restriction base="xs:string">
<xs:enumeration value="Annual" />
<xs:enumeration value="Quarterly" />
<xs:enumeration value="MaintenancePeriod" />
<xs:enumeration value="Monthly" />
<xs:enumeration value="Weekly" />
<xs:enumeration value="Daily" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

91
airflow/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml Normal file
View File

@@ -0,0 +1,91 @@
tmpdir: /tmp
inbox_prefix: INBOX/LM/MinimumReservesRequirements
archive_prefix: ARCHIVE/LM/MinimumReservesRequirements
workflow_name: w_ODS_LM_CURRENT_ACCOUNTS
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/current_accounts/config/current_accounts.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_CURRENT_ACCOUNTS_HEADER_PARSE
ods_prefix: INBOX/LM/MinimumReservesRequirements/LM_CURRENT_ACCOUNTS_HEADER
output_table: LM_CURRENT_ACCOUNTS_HEADER
namespaces:
ns2: http://escb.ecb.int/ca
output_columns:
- type: 'xpath_element_id'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:version'
column_header: 'REVISION_NUMBER'
is_key: 'N'
- type: 'xpath'
value: 'ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:referenceDate'
column_header: 'REFERENCE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:freeText'
column_header: 'FREE_TEXT'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:currentAccountBSTotal'
column_header: 'CURRENT_ACCOUNT_BS_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:mrrForecastTotal'
column_header: 'MRR_FORECAST_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:currentAccountMessageTotal'
column_header: 'CURRENT_ACCOUNT_MESSAGE_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:mrrMessageTotal'
column_header: 'MRR_MESSAGE_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:header/ns2:periodicity'
column_header: 'PERIODICITY'
is_key: 'N'
- task_name: m_ODS_LM_CURRENT_ACCOUNTS_ITEM_PARSE
ods_prefix: INBOX/LM/MinimumReservesRequirements/LM_CURRENT_ACCOUNTS_ITEM
output_table: LM_CURRENT_ACCOUNTS_ITEM
namespaces:
ns2: http://escb.ecb.int/ca
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:mfiCode'
column_header: 'MFI_CODE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:bankName'
column_header: 'BANK_NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:currentAccount'
column_header: 'CURRENT_ACCOUNT'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:minimumReserveRequirement'
column_header: 'MINIMUM_RESERVE_REQUIREMENT'
is_key: 'N'
- type: 'xpath'
value: '/ns2:currentAccountMrrMessage/ns2:disaggregatedCurrentAccountMrr/ns2:currentAccountMrrs/ns2:currentAccountMrr/ns2:comment'
column_header: 'COMMENT_'
is_key: 'N'

520
airflow/ods/lm/current_accounts/dags/w_ODS_LM_CURRENT_ACCOUNT.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/w_ODS_LM_ADHOC_CURRENT_ACCOUNTS.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/current_accounts/config/m_ODS_LM_CURRENT_ACCOUNTS_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "CURRENT_ACCOUNTS"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_CURRENT_ACCOUNTS"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_CURRENT_ACCOUNTS"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_CURRENT_ACCOUNTS"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_CURRENT_ACCOUNTS", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_CURRENT_ACCOUNTS",
trigger_dag_id="w_MOPDB_LM_CURRENT_ACCOUNTS",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

91
airflow/ods/lm/forecast/config/forecast.xsd Normal file
View File

@@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/forecast"
xmlns:fc="http://escb.ecb.int/forecast"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/forecast"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="forecastMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="cbForecast" type="cbForecast" minOccurs="1" maxOccurs="unbounded" />
<xs:element name="eurosystemForecast" type="eurosystemForecast" minOccurs="0" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:complexType name="cbHeader">
<xs:complexContent>
<xs:extension base="eurosystemHeader">
<xs:sequence>
<xs:element name="freeText" type="lm:freeText" minOccurs="0" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="eurosystemHeader">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="referenceDate" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="cbForecast">
<xs:annotation>
<xs:documentation>Represents an autonomous factor forecast for a CB
</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="cbHeader" />
<xs:element ref="forecastItems" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="eurosystemForecast">
<xs:annotation>
<xs:documentation>Represents an aggregated autonomous factor forecast of all CBs
</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="header" type="eurosystemHeader" />
<xs:element ref="forecastItems" />
</xs:sequence>
</xs:complexType>
<xs:element name="forecastItems">
<xs:complexType>
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:unique name="nameUnique">
<xs:selector xpath="fc:forecastItem" />
<xs:field xpath="./fc:forecastName" />
<xs:field xpath="./fc:forecastDate" />
</xs:unique>
</xs:element>
<xs:complexType name="forecastItem">
<xs:sequence>
<xs:element name="forecastDate" type="xs:date" />
<xs:element name="forecastName" type="lm:forecastName" />
<xs:element name="forecastValue" type="forecastValue" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="forecastValue">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="8" />
<xs:minInclusive value="-999999999.99999999" />
<xs:maxInclusive value="999999999.99999999" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

63
airflow/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml Normal file
View File

@@ -0,0 +1,63 @@
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/LM/EurosystemAutonomousFactorForecast
archive_prefix: ARCHIVE/LM/EurosystemAutonomousFactorForecast
workflow_name: w_ODS_LM_FORECAST
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/forecast/config/forecast.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_FORECAST_HEADER_PARSE
ods_prefix: INBOX/LM/EurosystemAutonomousFactorForecast/LM_FORECAST_HEADER
output_table: LM_FORECAST_HEADER
namespaces:
ns2: 'http://escb.ecb.int/forecast'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:forecastMessage/ns2:*'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:referenceDate'
column_header: 'REFERENCE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:version'
column_header: 'REVISION'
is_key: 'N'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:header/ns2:freeText'
column_header: 'FREE_TEXT'
is_key: 'N'
- task_name: m_ODS_LM_FORECAST_ITEM_PARSE
ods_prefix: INBOX/LM/EurosystemAutonomousFactorForecast/LM_FORECAST_ITEM
output_table: LM_FORECAST_ITEM
namespaces:
ns2: 'http://escb.ecb.int/forecast'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:forecastMessage/ns2:*'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastDate'
column_header: 'FORECAST_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastName'
column_header: 'FORECAST_NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:forecastMessage/ns2:*/ns2:forecastItems/ns2:forecastItem/ns2:forecastValue'
column_header: 'FORECAST_VALUE'
is_key: 'N'

520
airflow/ods/lm/forecast/dags/w_ODS_LM_FORECAST.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/w_ODS_LM_FORECAST.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/forecast/config/m_ODS_LM_FORECAST_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "FORECAST"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_FORECAST"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_FORECAST"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_FORECAST"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_FORECAST", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_FORECAST",
trigger_dag_id="w_MOPDB_LM_FORECAST",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

0
airflow/ods/lm/lm_common/.gitkeep Normal file
View File

95
airflow/ods/lm/lm_common/lm.xsd Normal file
View File

@@ -0,0 +1,95 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://exdi.ecb.int/lm"
targetNamespace="http://exdi.ecb.int/lm"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:simpleType name="isoCode">
<xs:restriction base="xs:string">
<xs:pattern value="[A-Z0-9]*" />
<xs:minLength value="2" />
<xs:maxLength value="3" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="positiveInt">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="2147483647" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="freeText">
<xs:restriction base="xs:string">
<xs:maxLength value="4000" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="sendingRound">
<xs:restriction base="xs:nonNegativeInteger">
<xs:minInclusive value="0" />
<xs:maxInclusive value="99" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="maintenancePeriod">
<xs:restriction base="xs:string">
<xs:pattern value="[1-9][0-9]{3}[0-9]{2}" />
<xs:maxLength value="6" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="percent">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="5" />
<xs:fractionDigits value="2" />
<xs:minInclusive value="-100.00" />
<xs:maxInclusive value="100.00" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="amountInEuro">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="2" />
<xs:minInclusive value="-999999999999999.99" />
<xs:maxInclusive value="999999999999999.99" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="decimalEuroValue">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="8" />
<xs:minInclusive value="0" />
<xs:maxInclusive value="999999999.99999999" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="mfiCode">
<xs:restriction base="xs:string">
<xs:maxLength value="255"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="bankName">
<xs:restriction base="xs:string">
<xs:minLength value="1"/>
<xs:maxLength value="500" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="comment">
<xs:restriction base="xs:string">
<xs:maxLength value="4000"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="forecastName">
<xs:restriction base="xs:string">
<xs:maxLength value="50" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

0
airflow/ods/lm/quarterly_adjustment/config/.gitkeep Normal file
View File

82
airflow/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml Normal file
View File

@@ -0,0 +1,82 @@
tmpdir: /tmp
inbox_prefix: INBOX/LM/QuarterlyRevaluationAdjustment
archive_prefix: ARCHIVE/LM/QuarterlyRevaluationAdjustment
workflow_name: w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_HEADER_PARSE
ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_HEADER
output_table: LM_QRE_ADJUSTMENTS_HEADER
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:year'
column_header: 'YEAR'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:quarter'
column_header: 'QUARTER'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:header/ns2:version'
column_header: 'VERSION'
is_key: 'N'
- task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_ITEM_HEADER_PARSE
ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_ITEM_HEADER
output_table: LM_QRE_ADJUSTMENTS_ITEM_HEADER
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:effectiveDate'
column_header: 'EFFECTIVE_DATE'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:lastDateNotInForecast'
column_header: 'LAST_DATE_NOT_FORECAST'
is_key: 'N'
- task_name: m_ODS_LM_QUARTERLY_ADJUSTMENT_ITEM_PARSE
ods_prefix: INBOX/LM/QuarterlyRevaluationAdjustment/LM_QRE_ADJUSTMENTS_ITEM
output_table: LM_QRE_ADJUSTMENTS_ITEM
namespaces:
ns2: 'http://escb.ecb.int/csm-adjustment'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment'
column_header: 'A_HEADER_FK'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:forecastName'
column_header: 'FORECAST_NAME'
is_key: 'N'
- type: 'xpath'
value: '/ns2:adjustmentMessages/ns2:quarterlyRevaluationAdjustmentMessage/ns2:adjustment/ns2:forecastItem/ns2:adjustmentAmount'
column_header: 'ADJUSTMENT_AMOUNT'
is_key: 'N'

129
airflow/ods/lm/quarterly_adjustment/config/quarterly_adjustment.xsd Normal file
View File

@@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns="http://escb.ecb.int/csm-adjustment"
xmlns:lm="http://exdi.ecb.int/lm"
targetNamespace="http://escb.ecb.int/csm-adjustment"
elementFormDefault="qualified"
attributeFormDefault="unqualified">
<xs:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd" />
<xs:element name="adjustmentMessages">
<xs:complexType>
<xs:choice>
<xs:element ref="csmAdjustmentMessage" />
<xs:element ref="quarterlyRevaluationAdjustmentMessage" />
<xs:element ref="adhocAdjustmentMessage" />
</xs:choice>
</xs:complexType>
</xs:element>
<xs:element name="csmAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="month" type="month" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentSingleForecast" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="quarterlyRevaluationAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="year" type="xs:gYear" />
<xs:element name="quarter" type="quarter" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adhocAdjustmentMessage">
<xs:complexType>
<xs:sequence>
<xs:element name="header">
<xs:complexType>
<xs:sequence>
<xs:element name="date" type="xs:date" />
<xs:element name="version" type="lm:positiveInt" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="adjustment" type="adjustmentMultipleForecasts" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:complexType name="baseAdjustment">
<xs:sequence>
<xs:element name="country" type="lm:isoCode" />
<xs:element name="effectiveDate" type="xs:date" />
<xs:element name="lastDateNotInForecast" type="xs:date" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="adjustmentSingleForecast">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="adjustmentMultipleForecasts">
<xs:complexContent>
<xs:extension base="baseAdjustment">
<xs:sequence>
<xs:element name="forecastItem" type="forecastItem" minOccurs="1" maxOccurs="unbounded" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="forecastItem">
<xs:sequence>
<xs:element name="forecastName" type="lm:forecastName" />
<xs:element name="adjustmentAmount" type="extendedDecimalEuroValue" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="extendedDecimalEuroValue">
<xs:restriction base="xs:decimal">
<xs:totalDigits value="17" />
<xs:fractionDigits value="8" />
<xs:minInclusive value="-999999999.99999999" />
<xs:maxInclusive value="999999999.99999999" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="month">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="12" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="quarter">
<xs:restriction base="xs:int">
<xs:minInclusive value="1" />
<xs:maxInclusive value="4" />
</xs:restriction>
</xs:simpleType>
</xs:schema>

520
airflow/ods/lm/quarterly_adjustment/dags/w_ODS_LM_QUARTERLY_ADJUSTMENT_MSG.py Normal file
View File

@@ -0,0 +1,520 @@
# dags/m_ODS_LM_QUARTERLY_ADJUSTMENT.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/quarterly_adjustment/config/m_ODS_LM_QUARTERLY_ADJUSTMENT_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "QUARTERLY_ADJUTMENT"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_QUARTERLY_ADJUSTMENT"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_QUARTERLY_ADJUSTMENT"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_QUARTERLY_ADJUSTMENT"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_QUARTERLY_ADJUSTMENT", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_QRE_ADJUSTMENT",
trigger_dag_id="w_MOPDB_LM_QRE_ADJUSTMENT",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

85
airflow/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml Normal file
View File

@@ -0,0 +1,85 @@
# Global configurations
tmpdir: /tmp
inbox_prefix: INBOX/LM/DistributeStandingFacilities
archive_prefix: ARCHIVE/LM/DistributeStandingFacilities
workflow_name: w_ODS_LM_Standing_Facilities
validation_schema_path: '/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/sf.xsd'
file_type: xml
# List of tasks
tasks:
- task_name: m_ODS_LM_Standing_Facilities_HEADER_PARSE
ods_prefix: INBOX/LM/DistributeStandingFacilities/LM_STANDING_FACILITIES_HEADER
output_table: LM_STANDING_FACILITIES_HEADER
namespaces:
ns2: 'http://escb.ecb.int/sf'
output_columns:
- type: 'xpath_element_id'
value: '//ns2:header'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath'
value: '//ns2:header/ns2:version'
column_header: 'REV_NUMBER'
is_key: 'N'
- type: 'xpath'
value: '//ns2:header/ns2:referenceDate'
column_header: 'REF_DATE'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'FREE_TEXT'
- type: 'xpath'
value: '//ns2:header/ns2:marginalLendingBSTotal'
column_header: 'MLF_BS_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '//ns2:header/ns2:depositFacilityBSTotal'
column_header: 'DF_BS_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '//ns2:header/ns2:marginalLendingSFTotal'
column_header: 'MLF_SF_TOTAL'
is_key: 'N'
- type: 'xpath'
value: '//ns2:header/ns2:depositFacilitySFTotal'
column_header: 'DF_SF_TOTAL'
is_key: 'N'
- task_name: m_ODS_LM_Standing_Facilities_ITEM_PARSE
ods_prefix: INBOX/LM/DistributeStandingFacilities/LM_STANDING_FACILITIES
output_table: LM_STANDING_FACILITIES
namespaces:
ns2: 'http://escb.ecb.int/sf'
output_columns:
- type: 'a_key'
column_header: 'A_KEY'
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'xpath_element_id'
value: '//ns2:header'
column_header: 'A_SFH_FK'
- type: 'xpath'
value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:country'
column_header: 'COUNTRY'
is_key: 'N'
- type: 'xpath'
value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:mfiCode'
column_header: 'MFI_ID'
is_key: 'N'
- type: 'xpath'
value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:bankName'
column_header: 'MFI_NAME'
is_key: 'N'
- type: 'xpath'
value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:marginalLending'
column_header: 'MARGINAL_LENDING_FACILITY'
is_key: 'N'
- type: 'xpath'
value: '//ns2:disaggregatedStandingFacilities/ns2:standingFacilities/ns2:disaggregatedStandingFacility/ns2:depositFacility'
column_header: 'DEPOSIT_FACILITY'
is_key: 'N'
- type: 'static'
value: ''
column_header: 'COMMENT_'

102
airflow/ods/lm/standing_facilities/config/sf.xsd Normal file
View File

@@ -0,0 +1,102 @@
<?xml version="1.0" encoding="utf-8"?>
<xsd:schema targetNamespace="http://escb.ecb.int/sf" xmlns="http://escb.ecb.int/sf" xmlns:lm="http://exdi.ecb.int/lm" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:infatype="http://www.informatica.com/types/" elementFormDefault="qualified" attributeFormDefault="unqualified">
<xsd:import namespace="http://exdi.ecb.int/lm" schemaLocation="../../lm_common/lm.xsd"/>
<xsd:complexType name="cbStandingFacilities">
<xs:annotation xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:documentation>Represents standing facilities of a CB</xs:documentation>
</xs:annotation>
<xsd:sequence>
<xsd:element name="header" type="cbHeader"/>
<xsd:element name="standingFacilities" minOccurs="0" maxOccurs="1" type="standingFacilitiesList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="disaggregatedStandingFacilitiesList">
<xsd:sequence>
<xsd:element name="disaggregatedStandingFacility" minOccurs="1" maxOccurs="unbounded" type="disaggregatedStandingFacility"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="standingFacilitiesList">
<xsd:sequence>
<xsd:element name="standingFacility" minOccurs="1" maxOccurs="unbounded" type="standingFacility"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="standingFacility">
<xsd:sequence>
<xsd:element name="mfiCode" type="lm:mfiCode">
</xsd:element>
<xsd:element name="bankName" type="lm:bankName">
</xsd:element>
<xsd:element name="marginalLending" type="lm:decimalEuroValue">
</xsd:element>
<xsd:element name="depositFacility" type="lm:decimalEuroValue">
</xsd:element>
<xsd:element name="comment" minOccurs="0" maxOccurs="1" type="lm:comment">
</xsd:element>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="disaggregatedHeader">
<xsd:sequence>
<xsd:element name="referenceDate" type="xsd:date">
</xsd:element>
<xsd:element name="version" type="lm:positiveInt">
</xsd:element>
<xsd:element name="marginalLendingBSTotal" type="lm:decimalEuroValue">
</xsd:element>
<xsd:element name="depositFacilityBSTotal" type="lm:decimalEuroValue">
</xsd:element>
<xsd:element name="marginalLendingSFTotal" type="lm:decimalEuroValue">
</xsd:element>
<xsd:element name="depositFacilitySFTotal" type="lm:decimalEuroValue">
</xsd:element>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="disaggregatedStandingFacility">
<xsd:complexContent>
<xsd:extension base="standingFacility">
<xsd:sequence>
<xsd:element name="country" type="lm:isoCode">
</xsd:element>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="disaggregatedStandingFacilities">
<xs:annotation xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:documentation>Represents the disaggregated standing facilities</xs:documentation>
</xs:annotation>
<xsd:sequence>
<xsd:element name="header" type="disaggregatedHeader"/>
<xsd:element name="standingFacilities" type="disaggregatedStandingFacilitiesList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="cbHeader">
<xsd:sequence>
<xsd:element name="country" type="lm:isoCode">
</xsd:element>
<xsd:element name="referenceDate" type="xsd:date">
</xsd:element>
<xsd:element name="version" type="lm:positiveInt">
</xsd:element>
<xsd:element name="freeText" minOccurs="0" maxOccurs="1" type="lm:freeText">
</xsd:element>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="standingFacilitiesMessage">
<xsd:complexType>
<xsd:choice>
<xsd:element name="cbStandingFacilities" minOccurs="1" maxOccurs="unbounded" type="cbStandingFacilities"/>
<xsd:element name="disaggregatedStandingFacilities" type="disaggregatedStandingFacilities"/>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>

0
airflow/ods/lm/standing_facilities/dags/.gitkeep Normal file
View File

519
airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES.py Normal file
View File

@@ -0,0 +1,519 @@
# dags/w_ODS_LM_STANDING_FACILITIES.py
# Idempotent, per-object mtime tracking
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta, datetime, timezone
from email.utils import parsedate_to_datetime
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# OCI settings
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML (single config for all files)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
LAST_TS_VAR = f"{dag_id}__last_seen_ts" # legacy watermark (kept for observability)
PROCESSED_SET_VAR = f"{dag_id}__processed_objects" # legacy: list of keys (back-compat only)
PROCESSED_TS_VAR = f"{dag_id}__processed_objects_ts" # NEW: map key -> last processed mtime (epoch float)
# Helpers
def _oci_client():
"""
Create an OCI Object Storage client.
Order: Resource Principals -> Instance Principals.
"""
import oci
region = os.getenv("OCI_REGION") or os.getenv("OCI_RESOURCE_PRINCIPAL_REGION") or "eu-frankfurt-1"
# RP
try:
rp_signer = oci.auth.signers.get_resource_principals_signer()
cfg = {"region": region} if region else {}
logging.info("Using OCI Resource Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=rp_signer)
except Exception as e:
logging.info("RP not available: %s", e)
# IP
try:
ip_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
cfg = {"region": region} if region else {}
logging.info("Using OCI Instance Principals signer (region=%s).", cfg.get("region"))
return oci.object_storage.ObjectStorageClient(cfg, signer=ip_signer)
except Exception as e:
logging.info("IP not available: %s", e)
logging.error("Neither Resource Principals nor Instance Principals authentication found.")
raise RuntimeError("Failed to create OCI client")
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Build config-derived constants directly from YAML
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = CONFIG_DATA.get("inbox_prefix")
if not (isinstance(OBJECT_PREFIX, str) and OBJECT_PREFIX.strip()):
raise AirflowFailException("YAML must define 'inbox_prefix' for OBJECT_PREFIX.")
OBJECT_PREFIX = OBJECT_PREFIX.strip()
logging.info("YAML inbox_prefix -> OBJECT_PREFIX: %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to resolve OBJECT_PREFIX from YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
# New idempotency map (key -> last_processed_ts)
def _load_processed_map() -> dict[str, float]:
"""
Returns {object_key: last_processed_ts}.
Back-compat: if old set variable exists (list), treat those keys as ts=0.
"""
try:
raw = Variable.get(PROCESSED_TS_VAR, default_var="{}")
m = json.loads(raw) or {}
if isinstance(m, dict):
return {k: float(v) for k, v in m.items()}
except Exception:
pass
# Back-compat: migrate old set/list
try:
old = json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))
if isinstance(old, list):
return {k: 0.0 for k in old}
except Exception:
pass
return {}
def _save_processed_map(m: dict[str, float]) -> None:
Variable.set(PROCESSED_TS_VAR, json.dumps(m))
def _mark_processed_ts(objs: list[tuple[str, float]]):
"""
Update processed map with list of (object_key, mtime).
"""
if REPROCESS or not objs:
return
m = _load_processed_map()
for key, ts in objs:
try:
ts = float(ts)
except Exception:
continue
prev = float(m.get(key, 0.0))
if ts > prev:
m[key] = ts
_save_processed_map(m)
logging.info("Processed map updated; size=%d", len(m))
# Object listing (per-key mtime)
def _list_new_xml_objects(prefix: str) -> list[dict]:
"""
List .xml objects and decide inclusion per-object:
include if REPROCESS or object_mtime > processed_map.get(object_key, 0.0)
Returns: [{"name": "<full-key>", "base": "<file.xml>", "mtime": <epoch float>}]
"""
if not OCI_NAMESPACE or not OCI_BUCKET:
raise AirflowFailException("BUCKET_NAMESPACE and INBOX_BUCKET must be set")
client = _oci_client()
processed_map = _load_processed_map()
try:
last_seen = float(Variable.get(LAST_TS_VAR, default_var="0"))
except Exception:
last_seen = 0.0
logging.info("Watermark last_seen=%s; processed_map_count=%d; prefix=%s",
last_seen, len(processed_map), prefix)
# NOTE: add pagination if needed
resp = client.list_objects(OCI_NAMESPACE, OCI_BUCKET, prefix=prefix)
new_items: list[dict] = []
newest_ts = last_seen
for o in (resp.data.objects or []):
name = (o.name or "").strip()
base = name.rsplit("/", 1)[-1] if name else ""
logging.info("Processing object: %s", base)
# Skip folder markers / empty keys
if not name or name.endswith('/') or not base:
logging.debug("Skip: folder marker or empty key: %r", name)
continue
if not base.lower().endswith(".xml"):
logging.debug("Skip: not .xml: %r", name)
continue
# Resolve mtime
ts = None
t = getattr(o, "time_created", None)
if t:
try:
ts = t.timestamp() if hasattr(t, "timestamp") else float(t) / 1000.0
except Exception:
ts = None
if ts is None:
try:
head = client.head_object(OCI_NAMESPACE, OCI_BUCKET, name)
lm = head.headers.get("last-modified") or head.headers.get("Last-Modified")
if lm:
dt = parsedate_to_datetime(lm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
ts = dt.timestamp()
logging.debug("Resolved ts via HEAD Last-Modified for %s: %s", name, ts)
except Exception as e:
logging.warning("head_object failed for %s: %s", name, e)
if ts is None:
ts = datetime.now(timezone.utc).timestamp()
logging.warning("Object %s missing timestamp; falling back to now=%s", name, ts)
last_proc_ts = float(processed_map.get(name, 0.0))
include = REPROCESS or (ts > last_proc_ts)
logging.info(
"Decision for %s: obj_ts=%s, last_proc_ts=%s, REPROCESS=%s -> include=%s",
name, ts, last_proc_ts, REPROCESS, include
)
if not include:
continue
item = {"name": name, "base": base, "mtime": ts}
new_items.append(item)
if ts > newest_ts:
newest_ts = ts
# Watermark advanced for visibility (optional)
if not REPROCESS and new_items and newest_ts > last_seen:
Variable.set(LAST_TS_VAR, str(newest_ts))
logging.info("Advanced watermark from %s to %s", last_seen, newest_ts)
new_items.sort(key=lambda x: x["mtime"]) # ascending
logging.info("Found %d candidate .xml object(s) under prefix %s", len(new_items), prefix)
return new_items
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (polling): single YAML config for all XML files in OCI',
schedule_interval=None, # Run EVERY 10 MIN
catchup=False,
max_active_runs=1,
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "STANDING_FACILITIES"],
) as dag:
@af_task(task_id="poll_oci_for_xml")
def poll_oci_for_xml():
"""
Lists new .xml objects and prepares a workload list.
Returns {"workload": [{"object": "<key>", "base": "<file.xml>", "mtime": <float>} ...]}
"""
if not OBJECT_PREFIX:
raise AirflowFailException("No OCI object prefix configured. Check YAML 'inbox_prefix'.")
new_objs = _list_new_xml_objects(OBJECT_PREFIX)
logging.info("New .xml objects found: %s", json.dumps(new_objs, indent=2))
print("New .xml objects found:", json.dumps(new_objs, indent=2))
# already contains base + mtime
workload = [{"object": it["name"], "base": it["base"], "mtime": it["mtime"]} for it in new_objs]
logging.info("Prepared workload items: %d", len(workload))
print("Prepared workload:", json.dumps(workload, indent=2))
return {"workload": workload}
@af_task(task_id="init_workflow")
def init_workflow(polled: dict):
"""Initialize workflow; start MRDS workflow; build per-file task configs."""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
workload = (polled or {}).get("workload") or []
# Airflow context for run_id
from airflow.operators.python import get_current_context
ctx = get_current_context()
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build TASK_CONFIGS dynamically: one per file, sequential numbering
task_base_name = "m_ODS_LM_STANDING_FACILITIES"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"], # pass basename to MRDS (adjust if you need full key)
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes object + mtime
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Init complete; workload=%d, tasks=%d", len(workload), len(task_configs))
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task with max_active_tis_per_dag=1)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# resolve full object key + mtime by matching base name from workload
full_object_key, object_mtime = None, None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
object_mtime = w.get('mtime')
break
# Print/log the file being processed
logging.info("%s: picking file %s (object=%s, mtime=%s)",
task_name, source_filename, full_object_key or source_filename, object_mtime)
print(f"{task_name}: picking file {source_filename} (object={full_object_key or source_filename}, mtime={object_mtime})")
try:
# NOTE: if MRDS expects full URI, change 'source_filename' to 'full_object_key'
mrds_main(
workflow_context,
source_filename, # or full_object_key if required in your env
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, source_filename)
raise
# Mark processed with the mtime we saw during poll
if full_object_key and object_mtime:
_mark_processed_ts([(full_object_key, object_mtime)])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_STANDING_FACILITIES"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
def check_success_for_mopdb(**context):
"""Check if all processing tasks succeeded before triggering MOPDB."""
from airflow.utils.state import State
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
# Check finalize_workflow task
finalize_task = dag_run.get_task_instance('finalize_workflow')
if finalize_task.state == State.FAILED:
has_failures = True
failure_reasons.append("finalize_workflow failed")
# Check all mapped tasks (per-file processing)
mapped_task_id = "m_ODS_LM_STANDING_FACILITIES"
mapped_tasks = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
for task_instance in mapped_tasks:
if task_instance.state in {State.FAILED, State.UPSTREAM_FAILED}:
has_failures = True
map_idx = getattr(task_instance, 'map_index', 'unknown')
failure_reasons.append(f"Processing task failed at index {map_idx}")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
# Check if all mapped tasks were skipped (no files to process)
all_skipped = all(t.state == State.SKIPPED for t in mapped_tasks) if mapped_tasks else True
if all_skipped or not mapped_tasks:
error_msg = "All processing tasks were skipped (no files to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
# Operators & Dependencies
poll_task = poll_oci_for_xml()
init_out = init_workflow(poll_task)
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_STANDING_FACILITIES", max_active_tis_per_dag=1)
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_STANDING_FACILITY",
trigger_dag_id="w_MOPDB_LM_STANDING_FACILITY",
conf={
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='poll_oci_for_xml')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='poll_oci_for_xml') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=False, # CHANGED: Don't wait for completion
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS, # CHANGED: Only trigger if check succeeds
retries=0,
)
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_DONE, # CHANGED: Always run to mark end
)
# CHANGED: Chain with check task before trigger
poll_task >> init_out >> task_cfgs >> per_file >> finalize_workflow >> check_mopdb >> trigger_mopdb >> all_good
logging.info(
"EXDI DAG ready: inbox_prefix=%s; using per-object processed ts map %s.",
OBJECT_PREFIX, PROCESSED_TS_VAR
)

354
airflow/ods/lm/standing_facilities/dags/w_ODS_LM_STANDING_FACILITIES_event_driven.py Normal file
View File

@@ -0,0 +1,354 @@
# dags/w_ODS_LM_STANDING_FACILITIES_event.py
import sys
import os
import json
import logging
from pathlib import Path
from datetime import timedelta
from airflow import DAG
from airflow.models import Variable
from airflow.decorators import task as af_task
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.operators.empty import EmptyOperator
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.empty import EmptyOperator
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
# Import libs
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/exdi')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from mrds.core import main as mrds_main
# DAG / Defaults
dag_id = Path(__file__).stem
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id,
}
# Optional OCI settings (not used for listing in event mode, but used for processed set keys)
OCI_NAMESPACE = os.getenv("BUCKET_NAMESPACE")
OCI_BUCKET = os.getenv("INBOX_BUCKET")
# Config YAML path (env override)
CONFIG_YAML = os.getenv(
"EXDI_SINGLE_CONFIG_YAML",
"/opt/airflow/src/airflow/dags/ods/lm/standing_facilities/config/m_ODS_LM_Standing_Facilities_PARSE.yaml",
)
logging.info("Using EXDI_SINGLE_CONFIG_YAML=%s", CONFIG_YAML)
# Idempotency controls
REPROCESS = (os.getenv("EXDI_REPROCESS", "false").lower() in ("1", "true", "yes"))
PROCESSED_SET_VAR = f"{dag_id}__processed_objects"
# If your MRDS expects full object keys for pFileUri, keep this True (default).
# Set EXDI_MRDS_USE_FULL_URI=false to send only the basename instead.
USE_FULL_URI_FOR_MRDS = (os.getenv("EXDI_MRDS_USE_FULL_URI", "true").lower() in ("1", "true", "yes"))
# Helpers
def _load_yaml(cfg_path: str) -> dict:
import yaml
p = Path(cfg_path)
if not p.exists():
raise FileNotFoundError(f"Config YAML not found: {cfg_path}")
return yaml.safe_load(p.read_text()) or {}
# Load inbox_prefix for logging / optional validation
try:
CONFIG_DATA = _load_yaml(CONFIG_YAML)
OBJECT_PREFIX = (CONFIG_DATA.get("inbox_prefix") or "").strip() or None
logging.info("YAML inbox_prefix (context only): %s", OBJECT_PREFIX)
except Exception as e:
logging.error("Failed to load CONFIG_YAML %s: %s", CONFIG_YAML, e)
OBJECT_PREFIX = None
def _mark_processed(objs: list[str]):
"""Add object keys to processed set (bounded)."""
if REPROCESS or not objs:
return
try:
processed = list(set(json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]"))))
except Exception:
processed = []
cap = 5000
# append new (unique) keys
for o in objs:
if o not in processed:
processed.append(o)
if len(processed) > cap:
processed = processed[-cap:]
logging.info("Updated processed set size=%d", len(processed))
Variable.set(PROCESSED_SET_VAR, json.dumps(processed))
def _extract_workload_from_conf(conf: dict) -> list[dict]:
"""Build workload from dag_run.conf | Returns: [{"object": "<full/key>", "base": "<file.xml>"} ...]"""
out = []
if not conf:
return out
single = conf.get("object")
many = conf.get("objects")
def _add(obj, mtime=None):
if isinstance(obj, str) and obj.strip():
key = obj.strip()
out.append({
"object": key,
"base": key.rsplit("/", 1)[-1],
"mtime": mtime,
})
if isinstance(single, str):
_add(single, conf.get("mtime")) # optional single mtime
if isinstance(many, list):
for o in many:
if isinstance(o, dict):
_add(o.get("object"), o.get("mtime"))
else:
_add(o)
return out
# DAG
with DAG(
dag_id=dag_id,
default_args=default_args,
description='EXDI workflow (event-driven): process objects from dag_run.conf using a single YAML config',
schedule_interval=None, # Manual/event only
catchup=False,
is_paused_upon_creation=True, # show up paused initially
render_template_as_native_obj=True,
tags=["EXDI", "MRDS", "ODS", "OCI", "event"],
) as dag:
@af_task(task_id="init_workflow")
def init_workflow():
"""
Read dag_run.conf, create MRDS workflow, and build per-file task configs.
"""
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing = []
if not username: missing.append("MRDS_LOADER_DB_USER")
if not password: missing.append("MRDS_LOADER_DB_PASS")
if not tnsalias: missing.append("MRDS_LOADER_DB_TNS")
raise AirflowFailException(f"Missing required env vars: {', '.join(missing)}")
# Access dag_run.conf
from airflow.operators.python import get_current_context
ctx = get_current_context()
dag_run = ctx.get("dag_run")
conf = (dag_run.conf or {}) if dag_run else {}
workload = _extract_workload_from_conf(conf)
if not workload:
raise AirflowSkipException("No objects provided in dag_run.conf (expected 'object' or 'objects').")
# Sort by mtime (first incoming file first)
workload.sort(key=lambda w: w.get("mtime") if w.get("mtime") is not None else float("inf")) # added fall back mtime, if Not needed can remove
# Start MRDS workflow run
run_id = str(ctx['ti'].run_id)
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, run_id)
workflow_context = {
"run_id": run_id,
"a_workflow_history_key": a_workflow_history_key
}
# Build per-file task configs
task_base_name = "m_ODS_LM_STANDING_FACILITIES"
task_configs = []
for idx, w in enumerate(workload, start=1):
task_configs.append({
"task_name": f"{task_base_name}_{idx}",
"source_filename": w["base"],
"config_file": CONFIG_YAML,
})
bundle = {
"workflow_history_key": a_workflow_history_key,
"workflow_context": workflow_context,
"workload": workload, # includes full 'object' keys
"task_configs": task_configs, # list-of-dicts for mapping
"env": env,
}
logging.info("Event init complete; tasks=%d; objects=%s",
len(task_configs), [w['object'] for w in workload])
return bundle
@af_task(task_id="get_task_configs")
def get_task_configs(init_bundle: dict):
return init_bundle["task_configs"]
def run_mrds_task(task_name: str, source_filename: str, config_file: str, **context):
"""Run MRDS for a single file (sequential via mapped task)."""
ti = context['ti']
if not os.path.exists(config_file):
raise FileNotFoundError(f"Config file not found: {config_file}")
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
workflow_context = init_bundle.get('workflow_context')
workload = init_bundle.get('workload') or []
if not workflow_context:
raise AirflowFailException("No workflow_context from init_workflow")
# Re-read processed set
try:
processed = set(json.loads(Variable.get(PROCESSED_SET_VAR, default_var="[]")))
except Exception:
processed = set()
# Resolve full object key by matching basename
full_object_key = None
for w in workload:
if w.get('base') == source_filename:
full_object_key = w.get('object')
break
if (not REPROCESS) and full_object_key and (full_object_key in processed):
logging.info("%s: skipping already-processed %s", task_name, full_object_key)
ti.xcom_push(key='task_status', value='SUCCESS_NOOP')
return "NOOP"
# Decide pFileUri for MRDS
if USE_FULL_URI_FOR_MRDS and full_object_key:
file_uri = full_object_key
else:
# fall back to basename
file_uri = source_filename
logging.info("%s: picking file %s (object=%s) -> MRDS pFileUri=%s",
task_name, source_filename, full_object_key or source_filename, file_uri)
print(f"{task_name}: MRDS pFileUri -> {file_uri}")
try:
mrds_main(
workflow_context,
file_uri, # pass the URI MRDS should match in A_SOURCE_FILE_CONFIG
config_file,
generate_workflow_context=False
)
except Exception:
logging.exception("%s: MRDS failed on %s", task_name, file_uri)
raise
if full_object_key:
_mark_processed([full_object_key])
ti.xcom_push(key='task_status', value='SUCCESS')
logging.info("%s: success", task_name)
return "SUCCESS"
def finalise_workflow_task(**context):
"""Finalize workflow across all per-file tasks (mapped)."""
from airflow.utils.state import State
ti = context['ti']
dag_run = context['dag_run']
init_bundle = ti.xcom_pull(task_ids='init_workflow') or {}
a_workflow_history_key = init_bundle.get('workflow_history_key')
if a_workflow_history_key is None:
raise AirflowFailException("No workflow history key; cannot finalise workflow")
mapped_task_id = "m_ODS_LM_STANDING_FACILITIES"
tis = [t for t in dag_run.get_task_instances() if t.task_id == mapped_task_id]
# If no mapped TIs (shouldn't happen unless no work), succeed
if not tis:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS (no files)", a_workflow_history_key)
return
any_failed = any(ti_i.state in {State.FAILED, State.UPSTREAM_FAILED} for ti_i in tis)
if not any_failed:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info("Finalised workflow %s as SUCCESS", a_workflow_history_key)
return
failed_idxs = [getattr(ti_i, "map_index", None) for ti_i in tis if ti_i.state in {State.FAILED, State.UPSTREAM_FAILED}]
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error("Finalised workflow %s as FAILED (failed map indexes=%s)", a_workflow_history_key, failed_idxs)
raise AirflowFailException(f"Workflow failed for mapped indexes: {failed_idxs}")
# Operators & Dependencies
init_out = init_workflow()
task_cfgs = get_task_configs(init_out)
@af_task(task_id="m_ODS_LM_STANDING_FACILITIES", max_active_tis_per_dag=1) # ensures only one mapped task instance to run at once, so they execute sequentially
def mapped_run(task_name: str, source_filename: str, config_file: str, **context):
return run_mrds_task(task_name=task_name, source_filename=source_filename, config_file=config_file, **context)
per_file = mapped_run.expand_kwargs(task_cfgs)
# Trigger the next DAG and wait for it to finish successfully
trigger_mopdb = TriggerDagRunOperator(
task_id="Trigger_w_MOPDB_LM_STANDING_FACILITY",
trigger_dag_id="w_MOPDB_LM_STANDING_FACILITY",
conf={
# pass along useful context to the next DAG
"source_dag": dag_id,
"upstream_run_id": "{{ run_id }}",
"objects": "{{ (ti.xcom_pull(task_ids='init_workflow')['workload'] | map(attribute='object') | list) if ti.xcom_pull(task_ids='init_workflow') else [] }}",
"workflow_history_key": "{{ (ti.xcom_pull(task_ids='init_workflow')['workflow_history_key']) if ti.xcom_pull(task_ids='init_workflow') else None }}"
},
wait_for_completion=True, # Until the triggered DAG completes
allowed_states=["success"], # treat only SUCCESS as success
failed_states=["failed"], # anything else -> fail
poke_interval=30, # how often to check status (secs)
)
# Final "everything went fine" marker — only runs if the triggered DAG succeeded
all_good = EmptyOperator(
task_id="All_went_well",
trigger_rule=TriggerRule.ALL_SUCCESS,
)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
)
init_out >> task_cfgs >> per_file >> finalize_workflow >> trigger_mopdb >> all_good
logging.info("EXDI EVENT DAG ready... Expect object keys in dag_run.conf (object / objects). YAML inbox_prefix=%s", OBJECT_PREFIX)

6
airflow/ods/rqsd/rqsd_process/config/yaml/flow_config_casper_test.yaml Normal file
View File

@@ -0,0 +1,6 @@
# Flow Configuration
CONNECTOR_TYPE: "casper_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS/RQSD_ANNEX_"
TASK_NAME: "m_ODS_RQSD_CASPER"
OUTPUT_TABLE: "RQSD_ANNEX_"
COLLECTION_ID: "1537"

19
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml Normal file
View File

@@ -0,0 +1,19 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_FX"
OUTPUT_TABLE: "RQSD_FX"
DEVO_QUERY: "select * from
(select tec_ingestion_date,
obs_value,
pow(10, cast(unit_mult as int)) as divisor,
series_key,
time_period,
ROW_NUMBER() OVER (PARTITION BY series_key, time_period ORDER BY tec_ingestion_date DESC) AS rn
from crp_other_pub.fx
where freq = 'Q'
and data_type_fm = 'HSTE'
and provider_fm = 'BL'
and instrument_fm = 'FX'
and currency != 'EUR') t
where t.rn = 1"

34
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml Normal file
View File

@@ -0,0 +1,34 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_FX_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_FX
output_table: RQSD_FX
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'obs_value'
column_header: 'obs_value'
- type: 'csv_header'
value: 'divisor'
column_header: 'divisor'
- type: 'csv_header'
value: 'series_key'
column_header: 'series_key'
- type: 'csv_header'
value: 'time_period'
column_header: 'time_period'
- type: 'csv_header'
value: 'rn'
column_header: 'rn'

31
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml Normal file
View File

@@ -0,0 +1,31 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_OBSERVATIONS"
OUTPUT_TABLE: "RQSD_OBSERVATIONS"
DEVO_QUERY: "with latest_dates as (
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_1_fin_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_1_2_fin_all_mrds
UNION
SELECT max(receivedfilereceiveddate) as max_date FROM crp_rqsd.rqsd_annex_2_all_mrds),
latest_update as (
SELECT max(max_date) as max_date FROM latest_dates),
latest as (
select case when count(max_date) = 0 then '1999-01-01 10:28:22' else max(max_date) end max_date
from latest_update)
SELECT a.*,
CAST(NULL AS STRING) AS tec_source_system,
CAST(NULL AS STRING) AS tec_dataset,
CAST(NULL AS STRING) AS tec_surrogate_key,
CAST(NULL AS STRING) AS tec_crc,
CAST(NULL AS STRING) AS tec_ingestion_date,
CAST(NULL AS STRING) AS tec_version_id,
CAST(NULL AS STRING) AS tec_execution_date,
CAST(NULL AS STRING) AS tec_run_id,
CAST(NULL AS STRING) AS tec_business_date
FROM crp_rqsd.tcrqsd_observations a where receivedfilereceiveddate > (select max(max_date) from latest)"

181
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml Normal file
View File

@@ -0,0 +1,181 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_OBSERVATIONS_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_OBSERVATIONS
output_table: RQSD_OBSERVATIONS
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'datacollectioncode'
column_header: 'datacollectioncode'
- type: 'csv_header'
value: 'datacollectionname'
column_header: 'datacollectionname'
- type: 'csv_header'
value: 'datacollectionowner'
column_header: 'datacollectionowner'
- type: 'csv_header'
value: 'reportingcyclename'
column_header: 'reportingcyclename'
- type: 'csv_header'
value: 'reportingcyclestatus'
column_header: 'reportingcyclestatus'
- type: 'csv_header'
value: 'modulecode'
column_header: 'modulecode'
- type: 'csv_header'
value: 'modulename'
column_header: 'modulename'
- type: 'csv_header'
value: 'moduleversionnumber'
column_header: 'moduleversionnumber'
- type: 'csv_header'
value: 'reportingentitycollectionuniqueid'
column_header: 'reportingentitycollectionuniqueid'
- type: 'csv_header'
value: 'entityattributereportingcode'
column_header: 'entityattributereportingcode'
- type: 'csv_header'
value: 'reportingentityname'
column_header: 'reportingentityname'
- type: 'csv_header'
value: 'reportingentityentitytype'
column_header: 'reportingentityentitytype'
- type: 'csv_header'
value: 'entityattributecountry'
column_header: 'entityattributecountry'
- type: 'csv_header'
value: 'entitygroupentityname'
column_header: 'entitygroupentityname'
- type: 'csv_header'
value: 'obligationmodulereferencedate'
column_header: 'obligationmodulereferencedate'
- type: 'csv_header'
value: 'obligationmoduleremittancedate'
column_header: 'obligationmoduleremittancedate'
- type: 'csv_header'
value: 'receivedfilereceiveddate'
column_header: 'receivedfilereceiveddate'
- type: 'csv_header'
value: 'obligationmoduleexpected'
column_header: 'obligationmoduleexpected'
- type: 'csv_header'
value: 'receivedfileversionnumber'
column_header: 'receivedfileversionnumber'
- type: 'csv_header'
value: 'revalidationversionnumber'
column_header: 'revalidationversionnumber'
- type: 'csv_header'
value: 'revalidationdate'
column_header: 'revalidationdate'
- type: 'csv_header'
value: 'receivedfilesystemfilename'
column_header: 'receivedfilesystemfilename'
- type: 'csv_header'
value: 'obligationstatusstatus'
column_header: 'obligationstatusstatus'
- type: 'csv_header'
value: 'filestatussetsubmissionstatus'
column_header: 'filestatussetsubmissionstatus'
- type: 'csv_header'
value: 'filestatussetvalidationstatus'
column_header: 'filestatussetvalidationstatus'
- type: 'csv_header'
value: 'filestatussetexternalvalidationstatus'
column_header: 'filestatussetexternalvalidationstatus'
- type: 'csv_header'
value: 'numberoferrors'
column_header: 'numberoferrors'
- type: 'csv_header'
value: 'numberofwarnings'
column_header: 'numberofwarnings'
- type: 'csv_header'
value: 'delayindays'
column_header: 'delayindays'
- type: 'csv_header'
value: 'failedattempts'
column_header: 'failedattempts'
- type: 'csv_header'
value: 'observationvalue'
column_header: 'observationvalue'
- type: 'csv_header'
value: 'observationtextvalue'
column_header: 'observationtextvalue'
- type: 'csv_header'
value: 'observationdatevalue'
column_header: 'observationdatevalue'
- type: 'csv_header'
value: 'datapointsetdatapointidentifier'
column_header: 'datapointsetdatapointidentifier'
- type: 'csv_header'
value: 'datapointsetlabel'
column_header: 'datapointsetlabel'
- type: 'csv_header'
value: 'obsrvdescdatatype'
column_header: 'obsrvdescdatatype'
- type: 'csv_header'
value: 'ordinatecode'
column_header: 'ordinatecode'
- type: 'csv_header'
value: 'ordinateposition'
column_header: 'ordinateposition'
- type: 'csv_header'
value: 'tablename'
column_header: 'tablename'
- type: 'csv_header'
value: 'isstock'
column_header: 'isstock'
- type: 'csv_header'
value: 'scale'
column_header: 'scale'
- type: 'csv_header'
value: 'currency'
column_header: 'currency'
- type: 'csv_header'
value: 'numbertype'
column_header: 'numbertype'
- type: 'csv_header'
value: 'ismandatory'
column_header: 'ismandatory'
- type: 'csv_header'
value: 'decimalplaces'
column_header: 'decimalplaces'
- type: 'csv_header'
value: 'serieskey'
column_header: 'serieskey'
- type: 'csv_header'
value: 'tec_source_system'
column_header: 'tec_source_system'
- type: 'csv_header'
value: 'tec_dataset'
column_header: 'tec_dataset'
- type: 'csv_header'
value: 'tec_surrogate_key'
column_header: 'tec_surrogate_key'
- type: 'csv_header'
value: 'tec_crc'
column_header: 'tec_crc'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'tec_version_id'
column_header: 'tec_version_id'
- type: 'csv_header'
value: 'tec_execution_date'
column_header: 'tec_execution_date'
- type: 'csv_header'
value: 'tec_run_id'
column_header: 'tec_run_id'
- type: 'csv_header'
value: 'tec_business_date'
column_header: 'tec_business_date'

736
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml Normal file
View File

@@ -0,0 +1,736 @@
# Flow Configuration
CONNECTOR_TYPE: "devo_connector"
ODS_PREFIX: "INBOX/RQSD/RQSD_PROCESS"
TASK_NAME: "m_ODS_RQSD_SUBA_DEVO"
OUTPUT_TABLE: "RQSD_SUBA_DEVO"
DEVO_QUERY: "with suba_reporting_requirements_temp as (
SELECT sender,
riad_code,
name,
tid,
entity_id,
module_name,
CASE
WHEN cons_level = 'IND' THEN 'SOLO'
WHEN cons_level = 'CON' THEN 'CONS'
ELSE cons_level
END AS cons_level,
accounting_standard,
template_id,
reference_date,
expected,
module_id,
dsd_id,
changed_date,
tec_source_system,
tec_dataset,
tec_surrogate_key,
tec_crc,
tec_ingestion_date,
tec_version_id,
tec_execution_date,
tec_run_id,
tec_business_date
FROM crp_suba06.suba_reporting_requirements),
observations_1_2 as
(
select OBLIGATIONMODULEREFERENCEDATE,
RECEIVEDFILERECEIVEDDATE,
SUBSTR(SERIESKEY, INSTR(SERIESKEY, ':') + 1) AS ETWDR_COMP_KEY,
REPORTINGENTITYCOLLECTIONUNIQUEID,
DATAPOINTSETDATAPOINTIDENTIFIER,
NVL(NVL(CAST(OBSERVATIONVALUE AS STRING), OBSERVATIONTEXTVALUE), OBSERVATIONDATEVALUE) AS OBS_VALUE
FROM crp_rqsd.rqsd_rqsd01_observations
WHERE MODULECODE = 'SCOPF'
AND TABLENAME = 'etwdr'
AND obligationmodulereferencedate not like '%2019-12-06%'
AND obligationmodulereferencedate not like '%2017%'),
max_version_1_2_fin_all as
(
SELECT max(RECEIVEDFILERECEIVEDDATE) as max_version, reportingentitycollectionuniqueid, obligationmodulereferencedate
FROM observations_1_2
group by reportingentitycollectionuniqueid, obligationmodulereferencedate),
annex_1_2_fin_all_pivoted as
(SELECT
OBLIGATIONMODULEREFERENCEDATE,
ETWDR_COMP_KEY,
RECEIVEDFILERECEIVEDDATE,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'ann_1_2_ref_date' THEN OBS_VALUE END) AS ANN_1_2_REF_DATE,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_mfi_id' THEN OBS_VALUE END) AS MFI_ID,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_lei' THEN OBS_VALUE END) AS LEGAL_ENTITY_ID,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_name' THEN OBS_VALUE END) AS INST_NAME,
MAX(CASE WHEN DATAPOINTSETDATAPOINTIDENTIFIER = 'etwdr_submitter' THEN OBS_VALUE END) AS ETWDR_SUBMITTER
FROM
observations_1_2
WHERE RECEIVEDFILERECEIVEDDATE in (select max_version from max_version_1_2_fin_all)
GROUP BY
OBLIGATIONMODULEREFERENCEDATE, ETWDR_COMP_KEY, RECEIVEDFILERECEIVEDDATE),
scopf as
(
select mfi_id, inst_name, legal_entity_id, obligationmodulereferencedate, ann_1_2_ref_date from annex_1_2_fin_all_pivoted where etwdr_submitter = 'SUP'
),
EXP_COREP_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_COREP_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_03.00'
and MODULE_NAME = 'COREP_OF'
and CONS_LEVEL = 'CONS'
),
EXP_COREP_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_COREP_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_03.00'
and MODULE_NAME = 'COREP_OF'
and CONS_LEVEL = 'SOLO'
),
EXP_FINREP_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_FINREP_CONS
from suba_reporting_requirements_temp
where template_id = 'tgF_00.01'
and MODULE_NAME in ('FINREP9', 'FINREP9_DP')
and CONS_LEVEL = 'CONS'
),
EXP_FINREP_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'N'
end as EXP_FINREP_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgF_00.01'
and MODULE_NAME in ('FINREP9', 'FINREP9_DP')
and CONS_LEVEL = 'SOLO'
),
EXP_LEV_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'full&trans'
when EXPECTED = 'O' then 'none'
else 'none'
end as EXP_LEV_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_47.00'
and MODULE_NAME in ('COREP_LR')
and CONS_LEVEL = 'CONS'
),
EXP_LEV_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'full&trans'
when EXPECTED = 'O' then 'none'
else 'none'
end as EXP_LEV_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_47.00'
and MODULE_NAME in ('COREP_LR')
and CONS_LEVEL = 'SOLO'
),
EXP_LCR_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_LCR_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_76.00'
and MODULE_NAME = 'COREP_LCR_DA'
and CONS_LEVEL = 'CONS'
),
EXP_LCR_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_LCR_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_76.00'
and MODULE_NAME = 'COREP_LCR_DA'
and CONS_LEVEL = 'SOLO'
),
EXP_NSFR_CONS as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_NSFR_CONS
from suba_reporting_requirements_temp
where template_id = 'tgC_84.00'
and MODULE_NAME = 'COREP_NSFR'
and CONS_LEVEL = 'CONS'
),
EXP_NSFR_SOLO as (
select riad_code,
reference_date,
module_name,
cons_level,
template_id,
case
when EXPECTED = 'E' then 'Y'
when EXPECTED = 'O' then 'O'
else 'O'
end as EXP_NSFR_SOLO
from suba_reporting_requirements_temp
where template_id = 'tgC_84.00'
and MODULE_NAME = 'COREP_NSFR'
and CONS_LEVEL = 'SOLO'
),
corep_cons_c100 as (
select riad_code,
reported_period,
cons_level,
name,
lei,
c_0100_r0010_c0010,
c_0100_r0020_c0010,
c_0100_r0015_c0010
from crp_suba06.suba_c_0100
where cons_level = 'CONS'
),
corep_solo_c100 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0100_r0010_c0010,
c_0100_r0020_c0010,
c_0100_r0015_c0010
from crp_suba06.suba_c_0100
where cons_level = 'SOLO'
),
corep_cons_c300 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0300_r0010_c0010,
c_0300_r0030_c0010,
c_0300_r0050_c0010
from crp_suba06.suba_c_0300
where cons_level = 'CONS'
),
corep_solo_c300 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0300_r0010_c0010,
c_0300_r0030_c0010,
c_0300_r0050_c0010
from crp_suba06.suba_c_0300
where cons_level = 'SOLO'
),
corep_cons_c200 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0200_r0010_c0010
from crp_suba06.suba_c_0200
where cons_level = 'CONS' and reported_period<'2025-03-31 00:00:00'
union all
select riad_code,
reported_period,
cons_level,
lei,
c_0200a_r0010_c0010
from crp_suba06.suba_c_0200a as c_0200_r0010_c0010
where cons_level = 'CONS' and reported_period>='2025-03-31 00:00:00'
),
corep_solo_c200 as (
select riad_code,
reported_period,
cons_level,
lei,
c_0200_r0010_c0010
from crp_suba06.suba_c_0200
where cons_level = 'SOLO' and reported_period<'2025-03-31 00:00:00'
union all
select riad_code,
reported_period,
cons_level,
lei,
c_0200a_r0010_c0010
from crp_suba06.suba_c_0200a as c_0200_r0010_c0010
where cons_level = 'SOLO' and reported_period>='2025-03-31 00:00:00'
),
finrep_cons as (
select riad_code,
reported_period,
cons_level,
lei,
f_0101_r0380_c0010
from crp_suba06.suba_f_0101
where cons_level = 'CONS'
),
finrep_solo as (
select riad_code,
reported_period,
cons_level,
lei,
f_0101_r0380_c0010
from crp_suba06.suba_f_0101
where cons_level = 'SOLO'
),
lev_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_4700_r0330_c0010,
c_4700_r0340_c0010,
c_4700_r0410_c0010
from crp_suba06.suba_c_4700
where cons_level = 'CONS'
),
lev_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_4700_r0330_c0010,
c_4700_r0340_c0010,
c_4700_r0410_c0010
from crp_suba06.suba_c_4700
where cons_level = 'SOLO'
),
lcr_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_7600a_r0030_c0010
from crp_suba06.suba_c_7600a
where cons_level = 'CONS'
),
lcr_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_7600a_r0030_c0010
from crp_suba06.suba_c_7600a
where cons_level = 'SOLO'
),
nsfr_cons as (
select riad_code,
reported_period,
cons_level,
lei,
c_8400a_r0220_c0040
from crp_suba06.suba_c_8400a
where cons_level = 'CONS'
),
nsfr_solo as (
select riad_code,
reported_period,
cons_level,
lei,
c_8400a_r0220_c0040
from crp_suba06.suba_c_8400a
where cons_level = 'SOLO'
),
liq_subgroups as (
SELECT entity_id,
substr(entity_id, 0, 20) as short_id,
name,
start_date,
end_date
from crp_suba06.suba_entity_master_data_stage2
where substr(entity_id, -12) = 'CRDLIQSUBGRP'
),
liq_subgroup_lcr as (
select liq_subgroups.entity_id,
short_id,
liq_subgroups.name,
start_date,
end_date,
suba_c_7600a.reported_period,
c_7600a_r0030_c0010
from liq_subgroups
left join crp_suba06.suba_c_7600a on liq_subgroups.entity_id = suba_c_7600a.entity_id
where start_date <= reported_period
and end_date >= reported_period
),
liq_subgroup_nsfr as (
select liq_subgroups.entity_id,
short_id,
liq_subgroups.name,
start_date,
end_date,
suba_c_8400a.reported_period,
c_8400a_r0220_c0040
from liq_subgroups
left join crp_suba06.suba_c_8400a on liq_subgroups.entity_id = suba_c_8400a.entity_id
where start_date <= reported_period
and end_date >= reported_period
),
liq_subgroup_data_riad as (
select liq_subgroup_lcr.entity_id,
liq_subgroup_lcr.short_id,
liq_subgroup_lcr.name,
liq_subgroup_lcr.start_date,
liq_subgroup_lcr.end_date,
liq_subgroup_lcr.reported_period,
c_7600a_r0030_c0010 as c_7600a_r0030_c0010_subgr,
c_8400a_r0220_c0040 as c_8400a_r0220_c0040_subgr,
riad_code
from liq_subgroup_lcr
left join crp_suba06.suba_entity_master_data_stage2 on liq_subgroup_lcr.short_id = suba_entity_master_data_stage2.entity_id
and suba_entity_master_data_stage2.start_date <= liq_subgroup_lcr.reported_period
and suba_entity_master_data_stage2.end_date >= liq_subgroup_lcr.reported_period
left join liq_subgroup_nsfr on liq_subgroup_lcr.entity_id = liq_subgroup_nsfr.entity_id
and liq_subgroup_lcr.reported_period = liq_subgroup_nsfr.reported_period
),
all_dates as (
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0100
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0300
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0200
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_0200a
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_f_0101
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_4700
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_7600a
union
select riad_code,
reported_period,
reception_date
from crp_suba06.suba_c_8400a
union
select riad_code,
reference_date,
cast(changed_date as STRING) as reception_date
from suba_reporting_requirements_temp
where template_id in (
'tgC_03.00',
'tgF_00.01',
'tgC_47.00',
'tgC_76.00',
'tgC_84.00'
)
and reference_date in (
select distinct obligationmodulereferencedate
from SCOPF
)
),
max_dates as (
select reported_period,
riad_code,
max(reception_date) as receivedfilereceiveddate
from all_dates
group by reported_period,
riad_code
)
select SCOPF.obligationmodulereferencedate,
cast(null as String) as reportingentitycollectionuniqueid,
cast(NULL as DECIMAL(38, 10)) as receivedfileversionnumber,
max_dates.receivedfilereceiveddate,
cast(NULL as STRING) as revalidationdate,
SCOPF.ann_1_2_ref_date as ref_date,
concat(SCOPF.mfi_id, SCOPF.legal_entity_id) as inst_comp_key,
SCOPF.mfi_id,
SCOPF.legal_entity_id,
SCOPF.inst_name,
'EUR' as currency,
cast(NULL as STRING) as reported_by_supervisor,
cast(NULL as STRING) as confirmed_by_supervisor,
coalesce(EXP_COREP_CONS.EXP_COREP_CONS, 'N') as EXP_COREP_CONS,
coalesce (EXP_COREP_SOLO.EXP_COREP_SOLO, 'N') as EXP_COREP_SOLO,
coalesce (EXP_FINREP_CONS.EXP_FINREP_CONS, 'N') as EXP_FINREP_CONS,
coalesce (EXP_FINREP_SOLO.EXP_FINREP_SOLO, 'N') as EXP_FINREP_SOLO,
coalesce(EXP_LEV_CONS.EXP_LEV_CONS, 'none') as EXP_LEV_CONS,
coalesce (EXP_LEV_SOLO.EXP_LEV_SOLO, 'none') as EXP_LEV_SOLO,
coalesce (EXP_LCR_CONS.EXP_LCR_CONS, 'N') as EXP_LCR_CONS,
coalesce (EXP_LCR_SOLO.EXP_LCR_SOLO, 'N') as EXP_LCR_SOLO,
coalesce (EXP_NSFR_CONS.EXP_NSFR_CONS, 'N') as EXP_NSFR_CONS,
coalesce (EXP_NSFR_SOLO.EXP_NSFR_SOLO, 'N') as EXP_NSFR_SOLO,
cast(
corep_cons_c100.c_0100_r0020_c0010 as DECIMAL(38, 10)
) as cons_cet1_amt,
cast(
corep_cons_c100.c_0100_r0015_c0010 as DECIMAL(38, 10)
) as cons_tier1_amt,
cast(
corep_cons_c100.c_0100_r0010_c0010 as DECIMAL(38, 10)
) as cons_tot_cap_amt,
cast(
corep_cons_c300.c_0300_r0010_c0010 as DECIMAL(38, 10)
) as cons_cet1_ratio,
cast(
corep_cons_c300.c_0300_r0030_c0010 as DECIMAL(38, 10)
) as cons_tier1_ratio,
cast(
corep_cons_c300.c_0300_r0050_c0010 as DECIMAL(38, 10)
) as cons_tot_cap_ratio,
cast(
corep_cons_c200.c_0200_r0010_c0010 as DECIMAL(38, 10)
) as cons_risk_wght_assets,
cast(
corep_solo_c100.c_0100_r0020_c0010 as DECIMAL(38, 10)
) as solo_cet1_amt,
cast(
corep_solo_c100.c_0100_r0015_c0010 as DECIMAL(38, 10)
) as solo_tier1_amt,
cast(
corep_solo_c100.c_0100_r0010_c0010 as DECIMAL(38, 10)
) as solo_tot_cap_amt,
cast(
corep_solo_c300.c_0300_r0010_c0010 as DECIMAL(38, 10)
) as solo_cet1_ratio,
cast(
corep_solo_c300.c_0300_r0030_c0010 as DECIMAL(38, 10)
) as solo_tier1_ratio,
cast(
corep_solo_c300.c_0300_r0050_c0010 as DECIMAL(38, 10)
) as solo_tot_cap_ratio,
cast(
corep_solo_c200.c_0200_r0010_c0010 as DECIMAL(38, 10)
) as solo_risk_wght_assets,
cast(finrep_cons.f_0101_r0380_c0010 as DECIMAL(38, 10)) as cons_tot_assets,
cast(finrep_solo.f_0101_r0380_c0010 as DECIMAL(38, 10)) as solo_tot_assets,
cast(lev_cons.c_4700_r0330_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_full,
cast(lev_cons.c_4700_r0340_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_trans,
cast(lev_cons.c_4700_r0410_c0010 as DECIMAL(38, 10)) as cons_lev_ratio_req,
CASE
WHEN lev_cons.c_4700_r0410_c0010 IS NULL THEN NULL
WHEN lev_cons.c_4700_r0410_c0010 = 0.0300000000 THEN 'N'
ELSE 'Y'
END AS cons_lev_ratio_adj,
cast(lev_solo.c_4700_r0330_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_full,
cast(lev_solo.c_4700_r0340_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_trans,
cast(lev_solo.c_4700_r0410_c0010 as DECIMAL(38, 10)) as solo_lev_ratio_req,
CASE
WHEN lev_solo.c_4700_r0410_c0010 IS NULL THEN NULL
WHEN lev_solo.c_4700_r0410_c0010 = 0.0300000000 THEN 'N'
ELSE 'Y'
END AS solo_lev_ratio_adj,
cast(
case
WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_7600a_r0030_c0010_subgr
ELSE lcr_cons.c_7600a_r0030_c0010
END as DECIMAL(38, 10)
) as cons_lc_ratio,
cast(lcr_solo.c_7600a_r0030_c0010 as DECIMAL(38, 10)) as solo_lc_ratio,
cast(
case
WHEN mfi_id = liq_subgroup_data_riad.riad_code Then c_8400a_r0220_c0040_subgr
ELSE nsfr_cons.c_8400a_r0220_c0040
END as DECIMAL(38, 10)
) as cons_nsfr_ratio,
cast(nsfr_solo.c_8400a_r0220_c0040 as DECIMAL(38, 10)) as solo_nsfr_ratio,
cast(NULL as STRING) as submitter_comment,
cast(NULL as STRING) as datacollectioncode,
cast(NULL as STRING) as reportingcyclename,
cast(NULL as STRING) as reportingcyclestatus,
cast(NULL as STRING) as modulecode,
cast(NULL as DECIMAL(38, 10)) as moduleversionnumber,
cast(NULL as STRING) as reportingentityname,
cast(NULL as STRING) as entityattributecountry,
cast(NULL as STRING) as entitygroupentityname,
cast(NULL as STRING) as obligationmoduleremittancedate,
cast(NULL as STRING) as obligationmoduleexpected,
cast(NULL as DEcimal(38, 10)) as revalidationversionnumber,
cast(NULL as STRING) as receivedfilesystemfilename,
cast(NULL as STRING) as obligationstatusstatus,
cast(NULL as STRING) as filestatussetsubmissionstatus,
cast(NULL as STRING) as filestatussetvalidationstatus,
cast(NULL as DECIMAL(38, 10)) as numberoferrors,
cast(NULL as DECIMAL(38, 10)) as numberofwarnings,
cast(NULL as Decimal(38, 10)) as delayindays,
cast(NULL as DECIMAL(38, 10)) as failedattempts,
cast(NULL as STRING) as tablename,
cast(NULL as STRING) as tec_source_system,
cast(NULL as STRING) as tec_dataset,
cast(NULL as STRING) as tec_surrogate_key,
cast(NULL as STRING) as tec_crc,
cast(NULL as TIMESTAMP) as tec_ingestion_date,
cast(NULL as STRING) as tec_version_id,
cast(NULL as TIMESTAMP) as tec_execution_date,
cast(NULL as STRING) as tec_run_id,
cast(NULL as TIMESTAMP) as tec_business_date
from SCOPF
left join EXP_COREP_CONS on (
scopf.obligationmodulereferencedate = EXP_COREP_CONS.reference_date
and scopf.mfi_id = EXP_COREP_CONS.riad_code
)
left join EXP_COREP_SOLO on (
scopf.obligationmodulereferencedate = EXP_COREP_SOLO.reference_date
and scopf.mfi_id = EXP_COREP_SOLO.riad_code
)
left join EXP_FINREP_CONS on (
scopf.obligationmodulereferencedate = EXP_FINREP_CONS.reference_date
and scopf.mfi_id = EXP_FINREP_CONS.riad_code
)
left join EXP_FINREP_SOLO on (
scopf.obligationmodulereferencedate = EXP_FINREP_SOLO.reference_date
and scopf.mfi_id = EXP_FINREP_SOLO.riad_code
)
left join EXP_LEV_CONS on (
scopf.obligationmodulereferencedate = EXP_LEV_CONS.reference_date
and scopf.mfi_id = EXP_LEV_CONS.riad_code
)
left join EXP_LEV_SOLO on (
scopf.obligationmodulereferencedate = EXP_LEV_SOLO.reference_date
and scopf.mfi_id = EXP_LEV_SOLO.riad_code
)
left join EXP_LCR_CONS on (
scopf.obligationmodulereferencedate = EXP_LCR_CONS.reference_date
and scopf.mfi_id = EXP_LCR_CONS.riad_code
)
left join EXP_LCR_SOLO on (
scopf.obligationmodulereferencedate = EXP_LCR_SOLO.reference_date
and scopf.mfi_id = EXP_LCR_SOLO.riad_code
)
left join EXP_NSFR_CONS on (
scopf.obligationmodulereferencedate = EXP_NSFR_CONS.reference_date
and scopf.mfi_id = EXP_NSFR_CONS.riad_code
)
left join EXP_NSFR_SOLO on (
scopf.obligationmodulereferencedate = EXP_NSFR_SOLO.reference_date
and scopf.mfi_id = EXP_NSFR_SOLO.riad_code
)
left join corep_cons_c100 on (
SCOPF.obligationmodulereferencedate = corep_cons_c100.reported_period
and SCOPF.mfi_id = corep_cons_c100.riad_code
)
left join corep_cons_c300 on (
SCOPF.obligationmodulereferencedate = corep_cons_c300.reported_period
and SCOPF.mfi_id = corep_cons_c300.riad_code
)
left join corep_cons_c200 on (
SCOPF.obligationmodulereferencedate = corep_cons_c200.reported_period
and SCOPF.mfi_id = corep_cons_c200.riad_code
)
left join corep_solo_c100 on (
SCOPF.obligationmodulereferencedate = corep_solo_c100.reported_period
and SCOPF.mfi_id = corep_solo_c100.riad_code
)
left join corep_solo_c200 on (
SCOPF.obligationmodulereferencedate = corep_solo_c200.reported_period
and SCOPF.mfi_id = corep_solo_c200.riad_code
)
left join corep_solo_c300 on (
SCOPF.obligationmodulereferencedate = corep_solo_c300.reported_period
and SCOPF.mfi_id = corep_solo_c300.riad_code
)
left join finrep_cons on (
SCOPF.obligationmodulereferencedate = finrep_cons.reported_period
and SCOPF.mfi_id = finrep_cons.riad_code
)
left join finrep_solo on (
SCOPF.obligationmodulereferencedate = finrep_solo.reported_period
and SCOPF.mfi_id = finrep_solo.riad_code
)
left join lev_cons on (
SCOPF.obligationmodulereferencedate = lev_cons.reported_period
and SCOPF.mfi_id = lev_cons.riad_code
)
left join lev_solo on (
SCOPF.obligationmodulereferencedate = lev_solo.reported_period
and SCOPF.mfi_id = lev_solo.riad_code
)
left join lcr_cons on (
SCOPF.obligationmodulereferencedate = lcr_cons.reported_period
and SCOPF.mfi_id = lcr_cons.riad_code
)
left join lcr_solo on (
SCOPF.obligationmodulereferencedate = lcr_solo.reported_period
and SCOPF.mfi_id = lcr_solo.riad_code
)
left join nsfr_cons on (
SCOPF.obligationmodulereferencedate = nsfr_cons.reported_period
and SCOPF.mfi_id = nsfr_cons.riad_code
)
left join nsfr_solo on (
SCOPF.obligationmodulereferencedate = nsfr_solo.reported_period
and SCOPF.mfi_id = nsfr_solo.riad_code
)
left join liq_subgroup_data_riad on (
SCOPF.obligationmodulereferencedate = liq_subgroup_data_riad.reported_period
and SCOPF.mfi_id = liq_subgroup_data_riad.riad_code
)
left join max_dates on (
SCOPF.obligationmodulereferencedate = max_dates.reported_period
and SCOPF.mfi_id = max_dates.riad_code
)
order by 1,2"

259
airflow/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml Normal file
View File

@@ -0,0 +1,259 @@
# static configs
tmpdir: /tmp
inbox_prefix: INBOX/RQSD/RQSD_PROCESS
archive_prefix: ARCHIVE/RQSD/RQSD_PROCESS
workflow_name: w_ODS_RQSD_PROCESS_DEVO
validation_schema_path: None
file_type: csv
# task configs
tasks:
- task_name: m_ODS_RQSD_SUBA_DEVO_PARSE
ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_SUBA_DEVO
output_table: RQSD_SUBA_DEVO
output_columns:
- type: 'workflow_key'
column_header: 'A_WORKFLOW_HISTORY_KEY'
- type: 'csv_header'
value: 'obligationmodulereferencedate'
column_header: 'obligationmodulereferencedate'
- type: 'csv_header'
value: 'reportingentitycollectionuniqueid'
column_header: 'reportingentitycollectionuniqueid'
- type: 'csv_header'
value: 'receivedfileversionnumber'
column_header: 'receivedfileversionnumber'
- type: 'csv_header'
value: 'receivedfilereceiveddate'
column_header: 'receivedfilereceiveddate'
- type: 'csv_header'
value: 'revalidationdate'
column_header: 'revalidationdate'
- type: 'csv_header'
value: 'ref_date'
column_header: 'ref_date'
- type: 'csv_header'
value: 'inst_comp_key'
column_header: 'inst_comp_key'
- type: 'csv_header'
value: 'mfi_id'
column_header: 'mfi_id'
- type: 'csv_header'
value: 'legal_entity_id'
column_header: 'legal_entity_id'
- type: 'csv_header'
value: 'inst_name'
column_header: 'inst_name'
- type: 'csv_header'
value: 'currency'
column_header: 'currency'
- type: 'csv_header'
value: 'reported_by_supervisor'
column_header: 'reported_by_supervisor'
- type: 'csv_header'
value: 'confirmed_by_supervisor'
column_header: 'confirmed_by_supervisor'
- type: 'csv_header'
value: 'exp_corep_cons'
column_header: 'exp_corep_cons'
- type: 'csv_header'
value: 'exp_corep_solo'
column_header: 'exp_corep_solo'
- type: 'csv_header'
value: 'exp_finrep_cons'
column_header: 'exp_finrep_cons'
- type: 'csv_header'
value: 'exp_finrep_solo'
column_header: 'exp_finrep_solo'
- type: 'csv_header'
value: 'exp_lev_cons'
column_header: 'exp_lev_cons'
- type: 'csv_header'
value: 'exp_lev_solo'
column_header: 'exp_lev_solo'
- type: 'csv_header'
value: 'exp_lcr_cons'
column_header: 'exp_lcr_cons'
- type: 'csv_header'
value: 'exp_lcr_solo'
column_header: 'exp_lcr_solo'
- type: 'csv_header'
value: 'exp_nsfr_cons'
column_header: 'exp_nsfr_cons'
- type: 'csv_header'
value: 'exp_nsfr_solo'
column_header: 'exp_nsfr_solo'
- type: 'csv_header'
value: 'cons_cet1_amt'
column_header: 'cons_cet1_amt'
- type: 'csv_header'
value: 'cons_tier1_amt'
column_header: 'cons_tier1_amt'
- type: 'csv_header'
value: 'cons_tot_cap_amt'
column_header: 'cons_tot_cap_amt'
- type: 'csv_header'
value: 'cons_cet1_ratio'
column_header: 'cons_cet1_ratio'
- type: 'csv_header'
value: 'cons_tier1_ratio'
column_header: 'cons_tier1_ratio'
- type: 'csv_header'
value: 'cons_tot_cap_ratio'
column_header: 'cons_tot_cap_ratio'
- type: 'csv_header'
value: 'cons_risk_wght_assets'
column_header: 'cons_risk_wght_assets'
- type: 'csv_header'
value: 'solo_cet1_amt'
column_header: 'solo_cet1_amt'
- type: 'csv_header'
value: 'solo_tier1_amt'
column_header: 'solo_tier1_amt'
- type: 'csv_header'
value: 'solo_tot_cap_amt'
column_header: 'solo_tot_cap_amt'
- type: 'csv_header'
value: 'solo_cet1_ratio'
column_header: 'solo_cet1_ratio'
- type: 'csv_header'
value: 'solo_tier1_ratio'
column_header: 'solo_tier1_ratio'
- type: 'csv_header'
value: 'solo_tot_cap_ratio'
column_header: 'solo_tot_cap_ratio'
- type: 'csv_header'
value: 'solo_risk_wght_assets'
column_header: 'solo_risk_wght_assets'
- type: 'csv_header'
value: 'cons_tot_assets'
column_header: 'cons_tot_assets'
- type: 'csv_header'
value: 'solo_tot_assets'
column_header: 'solo_tot_assets'
- type: 'csv_header'
value: 'cons_lev_ratio_full'
column_header: 'cons_lev_ratio_full'
- type: 'csv_header'
value: 'cons_lev_ratio_trans'
column_header: 'cons_lev_ratio_trans'
- type: 'csv_header'
value: 'cons_lev_ratio_req'
column_header: 'cons_lev_ratio_req'
- type: 'csv_header'
value: 'cons_lev_ratio_adj'
column_header: 'cons_lev_ratio_adj'
- type: 'csv_header'
value: 'solo_lev_ratio_full'
column_header: 'solo_lev_ratio_full'
- type: 'csv_header'
value: 'solo_lev_ratio_trans'
column_header: 'solo_lev_ratio_trans'
- type: 'csv_header'
value: 'solo_lev_ratio_req'
column_header: 'solo_lev_ratio_req'
- type: 'csv_header'
value: 'solo_lev_ratio_adj'
column_header: 'solo_lev_ratio_adj'
- type: 'csv_header'
value: 'cons_lc_ratio'
column_header: 'cons_lc_ratio'
- type: 'csv_header'
value: 'solo_lc_ratio'
column_header: 'solo_lc_ratio'
- type: 'csv_header'
value: 'cons_nsfr_ratio'
column_header: 'cons_nsfr_ratio'
- type: 'csv_header'
value: 'solo_nsfr_ratio'
column_header: 'solo_nsfr_ratio'
- type: 'csv_header'
value: 'submitter_comment'
column_header: 'submitter_comment'
- type: 'csv_header'
value: 'datacollectioncode'
column_header: 'datacollectioncode'
- type: 'csv_header'
value: 'reportingcyclename'
column_header: 'reportingcyclename'
- type: 'csv_header'
value: 'reportingcyclestatus'
column_header: 'reportingcyclestatus'
- type: 'csv_header'
value: 'modulecode'
column_header: 'modulecode'
- type: 'csv_header'
value: 'moduleversionnumber'
column_header: 'moduleversionnumber'
- type: 'csv_header'
value: 'reportingentityname'
column_header: 'reportingentityname'
- type: 'csv_header'
value: 'entityattributecountry'
column_header: 'entityattributecountry'
- type: 'csv_header'
value: 'entitygroupentityname'
column_header: 'entitygroupentityname'
- type: 'csv_header'
value: 'obligationmoduleremittancedate'
column_header: 'obligationmoduleremittancedate'
- type: 'csv_header'
value: 'obligationmoduleexpected'
column_header: 'obligationmoduleexpected'
- type: 'csv_header'
value: 'revalidationversionnumber'
column_header: 'revalidationversionnumber'
- type: 'csv_header'
value: 'receivedfilesystemfilename'
column_header: 'receivedfilesystemfilename'
- type: 'csv_header'
value: 'obligationstatusstatus'
column_header: 'obligationstatusstatus'
- type: 'csv_header'
value: 'filestatussetsubmissionstatus'
column_header: 'filestatussetsubmissionstatus'
- type: 'csv_header'
value: 'filestatussetvalidationstatus'
column_header: 'filestatussetvalidationstatus'
- type: 'csv_header'
value: 'numberoferrors'
column_header: 'numberoferrors'
- type: 'csv_header'
value: 'numberofwarnings'
column_header: 'numberofwarnings'
- type: 'csv_header'
value: 'delayindays'
column_header: 'delayindays'
- type: 'csv_header'
value: 'failedattempts'
column_header: 'failedattempts'
- type: 'csv_header'
value: 'tablename'
column_header: 'tablename'
- type: 'csv_header'
value: 'tec_source_system'
column_header: 'tec_source_system'
- type: 'csv_header'
value: 'tec_dataset'
column_header: 'tec_dataset'
- type: 'csv_header'
value: 'tec_surrogate_key'
column_header: 'tec_surrogate_key'
- type: 'csv_header'
value: 'tec_crc'
column_header: 'tec_crc'
- type: 'csv_header'
value: 'tec_ingestion_date'
column_header: 'tec_ingestion_date'
- type: 'csv_header'
value: 'tec_version_id'
column_header: 'tec_version_id'
- type: 'csv_header'
value: 'tec_execution_date'
column_header: 'tec_execution_date'
- type: 'csv_header'
value: 'tec_run_id'
column_header: 'tec_run_id'
- type: 'csv_header'
value: 'tec_business_date'
column_header: 'tec_business_date'

524
airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS.py Normal file
View File

@@ -0,0 +1,524 @@
import sys
import os
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.operators.dummy import DummyOperator
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from datetime import datetime, timedelta
import logging
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
sys.path.append('/opt/airflow/python/connectors/devo')
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from devo_connector import DevoConnector
from mrds.core import main as mrds_main
from mrds.utils.security_utils import get_verified_run_id, verify_run_id
TASK_CONFIGS = {
"m_ODS_RQSD_OBSERVATIONS": {
"flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS.yaml",
"env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml",
"source_filename": "RQSD_OBSERVATIONS.csv",
"config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_OBSERVATIONS_PARSE.yaml"
},
"m_ODS_RQSD_FX": {
"flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml",
"env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml",
"source_filename": "RQSD_FX.csv",
"config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml"
},
"m_ODS_RQSD_SUBA_DEVO": {
"flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml",
"env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml",
"source_filename": "RQSD_SUBA_DEVO.csv",
"config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml"
}
}
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=1),
}
dag_id = os.path.splitext(os.path.basename(__file__))[0]
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id
}
with DAG(
dag_id=dag_id,
default_args=default_args,
description='Run devo RQSD data ingestion workflow with conditional MRDS processing - Multi-task',
#schedule_interval='*/10 * * * *', # every 10 minutes
schedule_interval=None,
catchup=False,
max_active_runs=1,
tags=["Devo", "RQSD", "MRDS", "Connector", "Multi-Task"]
) as dag:
def init_workflow_task(**context):
try:
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env_vars = {
'MRDS_ENV': os.getenv("MRDS_ENV"),
'MRDS_LOADER_DB_USER': os.getenv("MRDS_LOADER_DB_USER"),
'MRDS_LOADER_DB_PASS': '***MASKED***' if os.getenv("MRDS_LOADER_DB_PASS") else None,
'MRDS_LOADER_DB_TNS': os.getenv("MRDS_LOADER_DB_TNS"),
'BUCKET_NAMESPACE': os.getenv("BUCKET_NAMESPACE"),
}
for key, value in env_vars.items():
logging.info(f"{key}: {value}")
run_id_value = get_verified_run_id(context)
logging.info(f"Task ID: {context.get('task_instance_key_str', 'N/A')}")
logging.info(f"Run ID: {run_id_value}")
logging.info(f"Execution Date: {context.get('execution_date', 'N/A')}")
dag_obj = context.get('dag')
dag_id_str = dag_obj.dag_id if dag_obj else 'N/A'
logging.info(f"DAG ID: {dag_id_str}")
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing_vars = []
if not username:
missing_vars.append("MRDS_LOADER_DB_USER")
if not password:
missing_vars.append("MRDS_LOADER_DB_PASS")
if not tnsalias:
missing_vars.append("MRDS_LOADER_DB_TNS")
error_msg = f"Missing required environment variables: {', '.join(missing_vars)}"
logging.error(error_msg)
raise ValueError(error_msg)
logging.info(f"Initializing {workflow_name} workflow for env '{env}'")
workflow_run_id = run_id_value
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, workflow_run_id)
logging.info(f"Initialized workflow with history key: {a_workflow_history_key}")
workflow_context = {
"run_id": workflow_run_id,
"a_workflow_history_key": a_workflow_history_key
}
ti = context['ti']
ti.xcom_push(key='workflow_history_key', value=a_workflow_history_key)
ti.xcom_push(key='workflow_context', value=workflow_context)
ti.xcom_push(key='env', value=env)
logging.info("Workflow initialization completed successfully")
except Exception as e:
logging.error(f"Error initializing workflow: {e}", exc_info=True)
raise
def run_devo_connector(**context):
ti = context['ti']
task_id = context['task'].task_id
try:
if task_id.startswith('devo_'):
task_name = task_id.replace('devo_', '')
else:
task_name = task_id
task_config = TASK_CONFIGS.get(task_name)
if not task_config:
error_msg = f"No configuration found for task: {task_name}"
logging.error(error_msg)
ti.xcom_push(key='row_count', value=0)
ti.xcom_push(key='devo_success', value=False)
ti.xcom_push(key='should_run_mrds', value=False)
ti.xcom_push(key='error_message', value=error_msg)
raise ValueError(error_msg)
flow_config_path = task_config["flow_config_path"]
env_config_path = task_config["env_config_path"]
workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow')
env = ti.xcom_pull(key='env', task_ids='init_workflow')
if not workflow_context:
error_msg = "No workflow_context from init task"
logging.error(error_msg)
ti.xcom_push(key='row_count', value=0)
ti.xcom_push(key='devo_success', value=False)
ti.xcom_push(key='should_run_mrds', value=False)
ti.xcom_push(key='error_message', value=error_msg)
raise ValueError(error_msg)
logging.info(f"Starting Devo connector for env '{env}' - {task_name}")
devo_connector = DevoConnector(
flow_config_path=flow_config_path,
env_config_path=env_config_path,
env=env,
logger=logging.getLogger(f"devo_connector_{env}_{task_name}")
)
row_count = devo_connector.run(workflow_context)
logging.info(f"Devo connector completed successfully for {task_name}. Processed {row_count} rows.")
ti.xcom_push(key='row_count', value=row_count)
ti.xcom_push(key='devo_success', value=True)
ti.xcom_push(key='should_run_mrds', value=row_count > 0)
ti.xcom_push(key='error_message', value=None)
return row_count
except Exception as e:
error_msg = f"Error running Devo connector: {str(e)}"
logging.error(error_msg, exc_info=True)
ti.xcom_push(key='row_count', value=0)
ti.xcom_push(key='devo_success', value=False)
ti.xcom_push(key='should_run_mrds', value=False)
ti.xcom_push(key='error_message', value=error_msg)
raise
def check_should_run_mrds(**context):
ti = context['ti']
dag_run = context['dag_run']
task_id = context['task'].task_id
if task_id.startswith('check_'):
task_name = task_id.replace('check_', '')
else:
task_name = task_id
devo_task_id = f'devo_{task_name}'
devo_task_instance = dag_run.get_task_instance(devo_task_id)
logging.info(f"Devo task {devo_task_id} state: {devo_task_instance.state}")
if devo_task_instance.state == 'failed':
error_msg = ti.xcom_pull(key='error_message', task_ids=devo_task_id)
logging.info(f"Devo connector failed for {task_name} - skipping MRDS task. Error: {error_msg}")
raise AirflowSkipException(f"Devo connector failed for {task_name}")
should_run_mrds = ti.xcom_pull(key='should_run_mrds', task_ids=devo_task_id)
row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id)
if task_name == "m_ODS_RQSD_OBSERVATIONS" and (not should_run_mrds or row_count == 0):
logging.info(f"OBSERVATIONS task has no data (row_count: {row_count}) - marking to skip all subsequent tasks")
ti.xcom_push(key='skip_all_tasks', value=True)
raise AirflowSkipException(f"No OBSERVATIONS data found (row_count: {row_count}) - skipping all subsequent processing")
if not should_run_mrds or row_count == 0:
logging.info(f"Skipping MRDS task for {task_name} - row count: {row_count}")
raise AirflowSkipException(f"No data to process for {task_name} (row_count: {row_count})")
logging.info(f"MRDS task should run for {task_name} - row count: {row_count}")
return True
def check_should_run_parallel_tasks(**context):
ti = context['ti']
task_id = context['task'].task_id
if task_id.startswith('check_'):
task_name = task_id.replace('check_', '')
else:
task_name = task_id
skip_all_tasks = ti.xcom_pull(key='skip_all_tasks', task_ids='check_m_ODS_RQSD_OBSERVATIONS')
if skip_all_tasks:
logging.info(f"Skipping {task_name} - OBSERVATIONS task had no data")
raise AirflowSkipException(f"Skipping {task_name} - OBSERVATIONS task had no data")
return check_should_run_mrds(**context)
def run_mrds_task(**context):
ti = context['ti']
task_id = context['task'].task_id
try:
if task_id.endswith('_PARSE'):
task_name = task_id.replace('_PARSE', '')
else:
task_name = task_id
devo_task_id = f'devo_{task_name}'
task_config = TASK_CONFIGS.get(task_name)
if not task_config:
raise ValueError(f"No configuration found for task: {task_name}")
source_filename = task_config["source_filename"]
config_file = task_config["config_file"]
workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow')
row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id)
if not workflow_context:
raise ValueError("No workflow_context from init task")
logging.info(f"Starting MRDS task for {task_name} with workflow context: {workflow_context}")
logging.info(f"Processing {row_count} rows from Devo connector")
mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=False)
logging.info(f"MRDS task completed successfully for {task_name}")
return "SUCCESS"
except Exception as e:
logging.error(f"Error running MRDS task: {e}", exc_info=True)
raise
def check_success_for_mopdb(**context):
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
for task_name in TASK_CONFIGS.keys():
devo_task_id = f'devo_{task_name}'
mrds_task_id = f'{task_name}_PARSE'
devo_task = dag_run.get_task_instance(devo_task_id)
mrds_task = dag_run.get_task_instance(mrds_task_id)
if devo_task.state == 'failed':
has_failures = True
failure_reasons.append(f"{task_name}: Devo connector failed")
if mrds_task.state == 'failed':
has_failures = True
failure_reasons.append(f"{task_name}: MRDS task failed")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
all_skipped = all(
dag_run.get_task_instance(f'{task_name}_PARSE').state == 'skipped'
for task_name in TASK_CONFIGS.keys()
)
if all_skipped:
error_msg = "All MRDS tasks were skipped (no data to process) - skipping MOPDB trigger"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
def end_log_table_task(**context):
try:
logging.info("End log table task - always runs at the end")
return "SUCCESS"
except Exception as e:
logging.error(f"Error in end log table task: {e}", exc_info=True)
raise
def finalise_workflow_task(**context):
a_workflow_history_key = None
try:
ti = context['ti']
dag_run = context['dag_run']
a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='init_workflow')
if a_workflow_history_key is None:
raise ValueError("No workflow history key found in XCom; cannot finalise workflow")
workflow_success = True
failure_reasons = []
for task_name in TASK_CONFIGS.keys():
devo_task_id = f'devo_{task_name}'
mrds_task_id = f'{task_name}_PARSE'
devo_task = dag_run.get_task_instance(devo_task_id)
mrds_task = dag_run.get_task_instance(mrds_task_id)
if devo_task.state == 'failed':
workflow_success = False
failure_reasons.append(f"{task_name}: Devo connector failed")
if mrds_task.state == 'failed':
workflow_success = False
failure_reasons.append(f"{task_name}: MRDS task failed")
elif mrds_task.state == 'skipped':
row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id)
devo_success = ti.xcom_pull(key='devo_success', task_ids=devo_task_id)
if devo_success and row_count == 0:
logging.info(f"{task_name} - MRDS task was skipped due to no data - this is normal")
elif not devo_success:
workflow_success = False
failure_reasons.append(f"{task_name}: Devo connector failed, MRDS skipped")
trigger_mopdb_task = dag_run.get_task_instance('trigger_mopdb_dag')
if trigger_mopdb_task.state == 'failed':
workflow_success = False
failure_reasons.append("MOPDB trigger failed")
end_log_task = dag_run.get_task_instance('end_log_table')
if end_log_task.state == 'failed':
workflow_success = False
failure_reasons.append("End log table failed")
if workflow_success:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS")
else:
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error(f"Finalised workflow with history key {a_workflow_history_key} as FAILED")
logging.error(f"Failure reasons: {', '.join(failure_reasons)}")
raise AirflowFailException(f"Workflow failed: {', '.join(failure_reasons)}")
except AirflowFailException:
raise
except Exception as e:
logging.error(f"Error finalizing workflow: {e}", exc_info=True)
try:
if a_workflow_history_key:
mrds_finalise_workflow(a_workflow_history_key, "N")
except:
pass
raise AirflowFailException(f"Workflow finalization failed: {e}")
init_workflow = PythonOperator(
task_id='init_workflow',
python_callable=init_workflow_task,
provide_context=True,
retries=0,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
trigger_mopdb_dag = TriggerDagRunOperator(
task_id='trigger_mopdb_dag',
trigger_dag_id='w_MOPDB_RQSD_PROCESS',
wait_for_completion=False,
trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS,
retries=0,
)
end_log_table = PythonOperator(
task_id='end_log_table',
python_callable=end_log_table_task,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
task_names = list(TASK_CONFIGS.keys())
first_task_name = task_names[0]
parallel_task_names = task_names[1:]
first_devo_task = PythonOperator(
task_id=f'devo_{first_task_name}',
python_callable=run_devo_connector,
provide_context=True,
retries=0,
)
first_check_task = PythonOperator(
task_id=f'check_{first_task_name}',
python_callable=check_should_run_mrds,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
first_mrds_task = PythonOperator(
task_id=f'{first_task_name}_PARSE',
python_callable=run_mrds_task,
provide_context=True,
retries=0,
)
parallel_tasks = []
for task_name in parallel_task_names:
devo_task = PythonOperator(
task_id=f'devo_{task_name}',
python_callable=run_devo_connector,
provide_context=True,
retries=0,
)
check_task = PythonOperator(
task_id=f'check_{task_name}',
python_callable=check_should_run_parallel_tasks,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
retries=0,
)
mrds_task = PythonOperator(
task_id=f'{task_name}_PARSE',
python_callable=run_mrds_task,
provide_context=True,
retries=0,
)
first_mrds_task >> devo_task >> check_task >> mrds_task
parallel_tasks.extend([devo_task, check_task, mrds_task])
init_workflow >> first_devo_task >> first_check_task >> first_mrds_task
all_final_tasks = [first_mrds_task]
for task_name in parallel_task_names:
mrds_task_id = f'{task_name}_PARSE'
for task in parallel_tasks:
if task.task_id == mrds_task_id:
all_final_tasks.append(task)
break
for task in all_final_tasks:
task >> check_mopdb
check_mopdb >> end_log_table >> trigger_mopdb_dag >> finalize_workflow

424
airflow/ods/rqsd/rqsd_process/dags/w_ODS_RQSD_PROCESS_MANUAL.py Normal file
View File

@@ -0,0 +1,424 @@
"""
DAG ID set to w_ODS_RQSD_PROCESS_MANUAL
Removed OBSERVATIONS from TASK_CONFIGS (only FX and SUBA_DEVO remain)
Updated run_devo_connector to always set should_run_mrds=True (no row count check)
Simplified check_should_run_mrds to always proceed if devo task succeeded (manual mode)
Removed special OBSERVATIONS handling logic
Trigger DAG ID changed to w_MOPDB_RQSD_PROCESS_MANUAL
Added "MANUAL" tag
Both FX and SUBA_DEVO tasks run in parallel from init
Removed skipped task handling for no data scenarios in finalize
"""
import sys
import os
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.operators.dummy import DummyOperator
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule
from datetime import datetime, timedelta
import logging
try:
from airflow.exceptions import AirflowFailException, AirflowSkipException
except Exception:
from airflow.exceptions import AirflowException as AirflowFailException
from airflow.exceptions import AirflowSkipException
sys.path.append('/opt/airflow/python/connectors/devo')
sys.path.append('/opt/airflow/python/mrds_common')
sys.path.append('/opt/airflow/src/airflow/dags/ods/rqsd')
from mrds.utils.manage_runs import init_workflow as mrds_init_workflow, finalise_workflow as mrds_finalise_workflow
from devo_connector import DevoConnector
from mrds.core import main as mrds_main
from mrds.utils.security_utils import get_verified_run_id, verify_run_id
TASK_CONFIGS = {
"m_ODS_RQSD_FX": {
"flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX.yaml",
"env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml",
"source_filename": "RQSD_FX.csv",
"config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_FX_PARSE.yaml"
},
"m_ODS_RQSD_SUBA_DEVO": {
"flow_config_path": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO.yaml",
"env_config_path": "/opt/airflow/python/connectors/devo/config/env_config_rqsd.yaml",
"source_filename": "RQSD_SUBA_DEVO.csv",
"config_file": "/opt/airflow/src/airflow/dags/ods/rqsd/rqsd_process/config/yaml/m_ODS_RQSD_SUBA_DEVO_PARSE.yaml"
}
}
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag_id = "w_ODS_RQSD_PROCESS_MANUAL"
WORKFLOW_CONFIG = {
"database_name": "ODS",
"workflow_name": dag_id
}
with DAG(
dag_id=dag_id,
default_args=default_args,
description='Manual trigger for RQSD data processing',
schedule_interval=None,
catchup=False,
tags=["Devo", "RQSD", "MRDS", "Connector", "Multi-Task", "MANUAL"]
) as dag:
def init_workflow_task(**context):
try:
database_name = WORKFLOW_CONFIG["database_name"]
workflow_name = WORKFLOW_CONFIG["workflow_name"]
env_vars = {
'MRDS_ENV': os.getenv("MRDS_ENV"),
'MRDS_LOADER_DB_USER': os.getenv("MRDS_LOADER_DB_USER"),
'MRDS_LOADER_DB_PASS': '***MASKED***' if os.getenv("MRDS_LOADER_DB_PASS") else None,
'MRDS_LOADER_DB_TNS': os.getenv("MRDS_LOADER_DB_TNS"),
'BUCKET_NAMESPACE': os.getenv("BUCKET_NAMESPACE"),
}
for key, value in env_vars.items():
logging.info(f"{key}: {value}")
run_id_value = get_verified_run_id(context)
logging.info(f"Task ID: {context.get('task_instance_key_str', 'N/A')}")
logging.info(f"Run ID: {run_id_value}")
logging.info(f"Execution Date: {context.get('execution_date', 'N/A')}")
dag_obj = context.get('dag')
dag_id_str = dag_obj.dag_id if dag_obj else 'N/A'
logging.info(f"DAG ID: {dag_id_str}")
env = os.getenv("MRDS_ENV", "dev")
username = os.getenv("MRDS_LOADER_DB_USER")
password = os.getenv("MRDS_LOADER_DB_PASS")
tnsalias = os.getenv("MRDS_LOADER_DB_TNS")
if not all([username, password, tnsalias]):
missing_vars = []
if not username:
missing_vars.append("MRDS_LOADER_DB_USER")
if not password:
missing_vars.append("MRDS_LOADER_DB_PASS")
if not tnsalias:
missing_vars.append("MRDS_LOADER_DB_TNS")
error_msg = f"Missing required environment variables: {', '.join(missing_vars)}"
logging.error(error_msg)
raise ValueError(error_msg)
logging.info(f"Initializing {workflow_name} workflow for env '{env}'")
workflow_run_id = run_id_value
a_workflow_history_key = mrds_init_workflow(database_name, workflow_name, workflow_run_id)
logging.info(f"Initialized workflow with history key: {a_workflow_history_key}")
workflow_context = {
"run_id": workflow_run_id,
"a_workflow_history_key": a_workflow_history_key
}
ti = context['ti']
ti.xcom_push(key='workflow_history_key', value=a_workflow_history_key)
ti.xcom_push(key='workflow_context', value=workflow_context)
ti.xcom_push(key='env', value=env)
logging.info("Workflow initialization completed successfully")
except Exception as e:
logging.error(f"Error initializing workflow: {e}", exc_info=True)
raise
def run_devo_connector(**context):
try:
ti = context['ti']
task_id = context['task'].task_id
if task_id.startswith('devo_'):
task_name = task_id.replace('devo_', '')
else:
task_name = task_id
task_config = TASK_CONFIGS.get(task_name)
if not task_config:
raise ValueError(f"No configuration found for task: {task_name}")
flow_config_path = task_config["flow_config_path"]
env_config_path = task_config["env_config_path"]
workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow')
env = ti.xcom_pull(key='env', task_ids='init_workflow')
if not workflow_context:
raise ValueError("No workflow_context from init task")
logging.info(f"Starting Devo connector for env '{env}' - {task_name}")
devo_connector = DevoConnector(
flow_config_path=flow_config_path,
env_config_path=env_config_path,
env=env,
logger=logging.getLogger(f"devo_connector_{env}_{task_name}")
)
row_count = devo_connector.run(workflow_context)
logging.info(f"Devo connector completed successfully for {task_name}. Processed {row_count} rows.")
ti.xcom_push(key='row_count', value=row_count)
ti.xcom_push(key='devo_success', value=True)
ti.xcom_push(key='should_run_mrds', value=True)
return row_count
except Exception as e:
logging.error(f"Error running Devo connector: {e}", exc_info=True)
ti = context['ti']
ti.xcom_push(key='row_count', value=0)
ti.xcom_push(key='devo_success', value=False)
ti.xcom_push(key='should_run_mrds', value=False)
raise
def check_should_run_mrds(**context):
ti = context['ti']
dag_run = context['dag_run']
task_id = context['task'].task_id
if task_id.startswith('check_'):
task_name = task_id.replace('check_', '')
else:
task_name = task_id
devo_task_id = f'devo_{task_name}'
devo_task_instance = dag_run.get_task_instance(devo_task_id)
logging.info(f"Devo task state: {devo_task_instance.state}")
if devo_task_instance.state == 'failed':
logging.info(f"Devo connector failed for {task_name} - skipping MRDS task")
raise AirflowSkipException(f"Devo connector failed for {task_name}")
logging.info(f"MRDS task will run for {task_name} - manual trigger mode")
return True
def run_mrds_task(**context):
try:
ti = context['ti']
task_id = context['task'].task_id
if task_id.endswith('_PARSE'):
task_name = task_id.replace('_PARSE', '')
else:
task_name = task_id
devo_task_id = f'devo_{task_name}'
task_config = TASK_CONFIGS.get(task_name)
if not task_config:
raise ValueError(f"No configuration found for task: {task_name}")
source_filename = task_config["source_filename"]
config_file = task_config["config_file"]
workflow_context = ti.xcom_pull(key='workflow_context', task_ids='init_workflow')
row_count = ti.xcom_pull(key='row_count', task_ids=devo_task_id)
if not workflow_context:
raise ValueError("No workflow_context from init task")
logging.info(f"Starting MRDS task for {task_name} with workflow context: {workflow_context}")
logging.info(f"Processing {row_count} rows from Devo connector")
mrds_main(workflow_context, source_filename, config_file, generate_workflow_context=False)
logging.info(f"MRDS task completed successfully for {task_name}")
return "SUCCESS"
except Exception as e:
logging.error(f"Error running MRDS task: {e}", exc_info=True)
raise
def check_success_for_mopdb(**context):
try:
ti = context['ti']
dag_run = context['dag_run']
has_failures = False
failure_reasons = []
for task_name in TASK_CONFIGS.keys():
devo_task_id = f'devo_{task_name}'
mrds_task_id = f'{task_name}_PARSE'
devo_task = dag_run.get_task_instance(devo_task_id)
mrds_task = dag_run.get_task_instance(mrds_task_id)
if devo_task.state == 'failed':
has_failures = True
failure_reasons.append(f"{task_name}: Devo connector failed")
if mrds_task.state == 'failed':
has_failures = True
failure_reasons.append(f"{task_name}: MRDS task failed")
if has_failures:
error_msg = f"Tasks failed - skipping MOPDB trigger: {', '.join(failure_reasons)}"
logging.info(error_msg)
raise AirflowSkipException(error_msg)
logging.info("All tasks completed successfully - proceeding to trigger MOPDB")
return "SUCCESS"
except AirflowSkipException:
raise
except Exception as e:
logging.error(f"Error checking success for MOPDB: {e}", exc_info=True)
raise AirflowSkipException(f"Error checking success - skipping MOPDB trigger: {e}")
def end_log_table_task(**context):
try:
logging.info("End log table task - always runs at the end")
return "SUCCESS"
except Exception as e:
logging.error(f"Error in end log table task: {e}", exc_info=True)
raise
def finalise_workflow_task(**context):
try:
ti = context['ti']
dag_run = context['dag_run']
a_workflow_history_key = ti.xcom_pull(key='workflow_history_key', task_ids='init_workflow')
if a_workflow_history_key is None:
raise ValueError("No workflow history key found in XCom; cannot finalise workflow")
workflow_success = True
failure_reasons = []
for task_name in TASK_CONFIGS.keys():
devo_task_id = f'devo_{task_name}'
mrds_task_id = f'{task_name}_PARSE'
devo_task = dag_run.get_task_instance(devo_task_id)
mrds_task = dag_run.get_task_instance(mrds_task_id)
if devo_task.state == 'failed':
workflow_success = False
failure_reasons.append(f"{task_name}: Devo connector failed")
if mrds_task.state == 'failed':
workflow_success = False
failure_reasons.append(f"{task_name}: MRDS task failed")
trigger_mopdb_task = dag_run.get_task_instance('trigger_mopdb_dag')
if trigger_mopdb_task.state == 'failed':
workflow_success = False
failure_reasons.append("MOPDB trigger failed")
end_log_task = dag_run.get_task_instance('end_log_table')
if end_log_task.state == 'failed':
workflow_success = False
failure_reasons.append("End log table failed")
if workflow_success:
mrds_finalise_workflow(a_workflow_history_key, "Y")
logging.info(f"Finalised workflow with history key {a_workflow_history_key} as SUCCESS")
else:
mrds_finalise_workflow(a_workflow_history_key, "N")
logging.error(f"Finalised workflow with history key {a_workflow_history_key} as FAILED")
logging.error(f"Failure reasons: {', '.join(failure_reasons)}")
raise AirflowFailException(f"Workflow failed: {', '.join(failure_reasons)}")
except AirflowFailException:
raise
except Exception as e:
logging.error(f"Error finalizing workflow: {e}", exc_info=True)
try:
if 'a_workflow_history_key' in locals() and a_workflow_history_key:
mrds_finalise_workflow(a_workflow_history_key, "N")
except:
pass
raise AirflowFailException(f"Workflow finalization failed: {e}")
init_workflow = PythonOperator(
task_id='init_workflow',
python_callable=init_workflow_task,
provide_context=True,
)
check_mopdb = PythonOperator(
task_id='check_success_for_mopdb',
python_callable=check_success_for_mopdb,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
)
trigger_mopdb_dag = TriggerDagRunOperator(
task_id='trigger_mopdb_dag',
trigger_dag_id='w_MOPDB_RQSD_PROCESS_MANUAL',
wait_for_completion=False,
trigger_rule=TriggerRule.NONE_FAILED,
)
end_log_table = PythonOperator(
task_id='end_log_table',
python_callable=end_log_table_task,
trigger_rule=TriggerRule.ALL_DONE,
)
finalize_workflow = PythonOperator(
task_id='finalize_workflow',
python_callable=finalise_workflow_task,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
)
all_tasks = []
for task_name in TASK_CONFIGS.keys():
devo_task = PythonOperator(
task_id=f'devo_{task_name}',
python_callable=run_devo_connector,
provide_context=True,
)
check_task = PythonOperator(
task_id=f'check_{task_name}',
python_callable=check_should_run_mrds,
provide_context=True,
trigger_rule=TriggerRule.ALL_DONE,
)
mrds_task = PythonOperator(
task_id=f'{task_name}_PARSE',
python_callable=run_mrds_task,
provide_context=True,
)
devo_task >> check_task >> mrds_task
all_tasks.extend([devo_task, check_task, mrds_task])
devo_tasks = [task for task in all_tasks if task.task_id.startswith('devo_')]
mrds_tasks = [task for task in all_tasks if task.task_id.endswith('_PARSE')]
init_workflow >> devo_tasks
for mrds_task in mrds_tasks:
mrds_task >> check_mopdb
check_mopdb >> trigger_mopdb_dag >> end_log_table >> finalize_workflow

236
airflow/ods/tms/README.md Normal file
View File

@@ -0,0 +1,236 @@
# DAG Factory for TMS Data Ingestion
## Overview
This repository contains a **DAG factory** that generates multiple Apache Airflow DAGs to ingest data from a **Treasury Management System (TMS)** into the data warehouse.
The factory dynamically creates one DAG per TMS dataset, using **YAML-based layouts** to define parameters and metadata. Each DAG:
- Calls the **TMSDB CLI connector** (`TMSDB.py`) to retrieve data in CSV format.
- Loads the data into object storage.
- Creates or refreshes **Oracle external tables** if needed.
- Registers workflow metadata in MRDS tables.
- Processes the landed file for downstream use.
---
## Components
### 1. DAG Factory (`create_dag`)
- **Purpose**: Auto-generates DAGs for each TMS dataset.
- **Inputs**:
- `TMS-layouts/<DAG_NAME>.yml`: defines report parameters, visible/hidden flags, virtual/replacement parameters.
- `config/TMS.yml`: holds system-wide TMS connection info and storage prefixes.
- **Outputs**:
- Airflow DAG objects named like `w_ODS_TMS_<ENTITY>`.
### 2. TMSDB Connector (`TMSDB.py`)
- **Purpose**: CLI tool that interacts with the TMS service.
- **Commands**:
- `retrieve`: fetch rows from TMS into CSV, spool to storage, return exit codes (`0 = data`, `1 = no data`).
- `create-oracle-table`: generate an Oracle DDL file based on dataset metadata.
- `create-model`: generate dbt models for dataset integration.
- **Behavior**:
- Adds synthetic columns (`A_KEY`, `A_WORKFLOW_HISTORY_KEY`).
- Supports additional columns via `-c`.
- Uploads to object storage if `bucket:path/file.csv` is given.
### 3. Manage Files (`mf`)
Utilities for file-level operations:
- `execute_query(sql)`
- `add_source_file_config(...)`
- `process_source_file(prefix, file)`
- `create_external_table(table, source, prefix)`
- `add_column_date_format(...)`
### 4. Manage Runs (`mr`)
Utilities for workflow tracking:
- `init_workflow(db, wf_name, run_id)`
- `set_workflow_property(key, db, name, value)`
- `finalise_workflow(key, status)`
- `select_ods_tab(table, expr, cond)`
---
## How a DAG Works
### DAG Structure
Each DAG has a single task:
- `retrieve_report`: a `PythonOperator` that orchestrates all steps internally.
### Task Flow
1. **Read YAML configs**
- Parameters split into visible (exposed in Airflow UI) and hidden.
- System config (URL, creds, bucket/prefix) loaded from `config/TMS.yml`.
2. **Parameter processing**
- Cartesian product of parameter lists.
- Support for:
- `column(...)` aligned columns.
- `select(...)` SQL evaluation (restricted tables only).
- Virtual parameters (dropped later).
- Replace-parameter logic.
3. **Workflow init**
- `mr.init_workflow` creates a workflow key.
4. **Data retrieval**
- Build a `TMSDB.py retrieve` command.
- Run via subprocess.
- Handle return codes:
- `0`: data returned.
- `1`: no data → workflow finalized as success.
- `!=0`: error → workflow finalized as failure.
5. **First-run bootstrap**
- If no config exists for the dataset:
- Run `TMSDB.py create-oracle-table` to generate SQL.
- Execute SQL via `mf.execute`.
- Add date formats and external table with `mf.create_external_table`.
- Register config with `mf.add_source_file_config`.
6. **File processing**
- `mf.process_source_file(prefix, filename)` ingests the CSV.
7. **Workflow finalization**
- `mr.finalise_workflow(wf_key, 'Y' | 'N')`.
---
## Example DAG
Example: `w_ODS_TMS_TRANSACTION`
```python
with DAG(
dag_id="w_ODS_TMS_TRANSACTION",
default_args=default_args,
schedule_interval=None,
start_date=datetime(2025, 1, 1),
catchup=False,
params={"date_from": "2025-01-01", "date_to": "2025-01-31"},
) as dag:
retrieve_report = PythonOperator(
task_id="retrieve_report",
python_callable=execute_report,
execution_timeout=timedelta(minutes=30),
)
```
---
## Repository Layout
```
tms/
├─ generate_tm_ods_dags.py # DAG generator script (calls create_dag many times)
├─ TMS-layouts/
│ ├─ w_ODS_TMS_TRANSACTION.yml
│ └─ ...
├─ config/
│ └─ TMS.yml
└─ TMS-tables/ # Create table SQL scripts
```
---
## Security Considerations
- **`eval()` is dangerous.**
Only `select(...)` is allowed, and it’s whitelisted to safe tables.
- **No raw shell commands.**
Use `subprocess.run([...], shell=False)` for safety.
- **Secrets in config.**
TMS username/password are stored in `TMS.yml` → best stored in Airflow Connections/Secrets Manager.
- **Exit codes matter.**
Workflow correctness relies on `TMSDB.py` returning the right codes (`0`, `1`, other).
---
## Extending the Factory
### Add a new dataset
1. Create a YAML layout in `TMS-layouts/`, e.g.:
```yaml
parameters:
date_from:
value: "2025-01-01"
date_to:
value: "2025-01-31"
```
2. Add a line in `dag_factory.py`:
```python
create_dag("w_ODS_TMS_NEWENTITY")
```
3. Deploy the DAG file to Airflow.
### Run a DAG manually
In the Airflow UI:
1. Find DAG `w_ODS_TMS_<ENTITY>`.
2. Trigger DAG → optionally override visible parameters.
3. Monitor logs for `retrieve_report`.
---
## Diagram
**DAG Factory Flow**
```mermaid
flowchart LR
subgraph DAGFactory["Dag Factory"]
direction TB
B["Load TMS config (TMS.yml)"] --> C["Load dataset layout (YAML)"]
C --> D["Extract visible & hidden parameters"]
D --> E["Define Airflow DAG with retrieve_report task"]
E --> F["Register DAG globally"]
F --> G["Repeat for each dataset name"]
G --> H["All DAGs available in Airflow"]
end
A["Airflow parses dag_factory.py"] --> DAGFactory
```
**Sample DAG Execution Flow**
```mermaid
flowchart LR
subgraph ExampleDAG
direction TB
B[Read YAML configs] --> C[Build parameter combinations]
C --> D["Evaluate select(...) and replace virtual params"]
D --> E["Init workflow (mr.init_workflow)"]
E --> F["Run TMSDB.py retrieve (subprocess)"]
%% Branches on return codes
F --> |rc=1: No data| G[Finalise workflow success]
F --> |rc=0: Data returned| H[Check if source file config exists]
F --> |rc!=0: Error| M[Finalise workflow failure]
%% Config missing branch
H --> |"Config missing (first run)"| I[Run TMSDB.py create-oracle-table → Generate DDL]
I --> J[Execute DDL via mf.execute → Create Oracle external table]
J --> K["Register file source config (mf.add_source_file_config)"]
K --> L["Process landed file (mf.process_source_file)"]
L --> N[Finalise workflow success]
%% Config exists branch
H --> |Config exists| P["Process landed file (mf.process_source_file)"]
P --> N[Finalise workflow success]
end
```
---
## Dependencies
- **Airflow 2.x**
- **Python 3.9+**
- **mrds** package (providing `utils.manage_files` and `utils.manage_runs`)
- **Oracle client / Impala client** (for table creation & querying)
- **Object storage client** (for uploading CSVs)
---
## Summary
The DAG factory is a scalable way to create **dozens of ingestion DAGs** for TMS datasets with minimal boilerplate. It leverages:
- **YAML configs** for parameters,
- **TMSDB CLI** for data retrieval and DDL generation,
- **MRDS utilities** for workflow tracking and file handling.
It standardizes ingestion while keeping each dataset’s DAG lightweight and uniform.

1641
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.fkr Normal file
View File

File diff suppressed because it is too large Load Diff

15
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMCURRENCYFLOW.yml Normal file
View File

@@ -0,0 +1,15 @@
parameters:
StartDate:
value: "t-10d"
hidden: false
EndDate:
value: "t"
hidden: false
ACMLedger_id:
value: 1
hidden: true

3143
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.fkr Normal file
View File

File diff suppressed because it is too large Load Diff

20
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACMENTRYSTATELEDGERGROUP.yml Normal file
View File

@@ -0,0 +1,20 @@
parameters:
StartEntryBookingDate:
value: "t-10d"
hidden: false
EndEntryBookingDate:
value: "t"
hidden: false
StartEventDate:
value: ""
hidden: true
EndEventDate:
value: ""
hidden: true

338
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.fkr Normal file
View File

@@ -0,0 +1,338 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=activity
Name=Activity Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Activity Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=group_id,id,type_id,domain_id,flags,name,service_name,service_group,start_date,time_zone_id,end_date,prerequisite_id,owner_id,type_name,start_date_local,end_date_local
Selected=group_id,id,type_id,domain_id,flags,name,service_name,service_group,start_date,time_zone_id,end_date,prerequisite_id,owner_id,type_name,start_date_local,end_date_local
Grouping=
Sorting=
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
Fields=
[Format group_id]
Name=group_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format id]
Name=id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format type_id]
Name=type_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format domain_id]
Name=domain_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format flags]
Name=flags
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format name]
Name=name
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format service_name]
Name=service_name
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format service_group]
Name=service_group
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format start_date]
Name=start_date
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format time_zone_id]
Name=time_zone_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format end_date]
Name=end_date
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format prerequisite_id]
Name=prerequisite_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format owner_id]
Name=owner_id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format type_name]
Name=type_name
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format start_date_local]
Name=start_date_local
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format end_date_local]
Name=end_date_local
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

5
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY.yml Normal file
View File

@@ -0,0 +1,5 @@
parameters:

248
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.fkr Normal file
View File

@@ -0,0 +1,248 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=activity-log-due
Name=Activity Log Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Activity Log Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=start_date = ''
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id
Selected=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id
Grouping=
Sorting=due_date
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
Fields=from,to,due_date_from,due_date_to,state,batch_id
from=
to=
due_date_from=
due_date_to=
state=
batch_id=
[Format id]
Name=ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=152
width_in_characters=30
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format number]
Name=NUMBER
Expression=
report_total=
date_format=
time_format=
Justify=right
width=82
width_in_characters=16
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format batch_id]
Name=BATCH_ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=125
width_in_characters=25
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format prerequisite_id]
Name=PREREQUISITE_ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=96
width_in_characters=19
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format due_date]
Name=DUE_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=146
width_in_characters=29
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format start_date]
Name=START_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=163
width_in_characters=32
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format finish_date]
Name=FINISH_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=126
width_in_characters=25
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format description]
Name=DESCRIPTION
Expression=
report_total=
date_format=
time_format=
Justify=left
width=353
width_in_characters=70
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format domain_id]
Name=DOMAIN
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format user_id]
Name=USER_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

5
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITYLOGDUE.yml Normal file
View File

@@ -0,0 +1,5 @@
parameters:

248
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.fkr Normal file
View File

@@ -0,0 +1,248 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=activity-log
Name=Activity Log Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Activity Log Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id
Selected=id,number,batch_id,prerequisite_id,due_date,start_date,finish_date,description,domain_id,user_id
Grouping=
Sorting=
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
Fields=from,to,due_date_from,due_date_to,state,batch_id
from=
to=
due_date_from=
due_date_to=
state=
batch_id=
[Format id]
Name=ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format number]
Name=NUMBER_
Expression=
report_total=
date_format=
time_format=
Justify=right
width=85
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format batch_id]
Name=BATCH_ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=78
width_in_characters=15
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format prerequisite_id]
Name=PREREQUISITE_ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=125
width_in_characters=25
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format due_date]
Name=DUE_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=127
width_in_characters=25
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format start_date]
Name=START_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=92
width_in_characters=18
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format finish_date]
Name=FINISH_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=218
width_in_characters=43
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format description]
Name=DESCRIPTION
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format domain_id]
Name=DOMAIN_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format user_id]
Name=USER_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=88
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

13
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.json Normal file
View File

@@ -0,0 +1,13 @@
{
"from":
{
"value": "t-1d"
},
"to":
{
"value": "t+1d"
}
}

12
airflow/ods/tms/TMS-layouts/w_ODS_TMS_ACTIVITY_LOG.yml Normal file
View File

@@ -0,0 +1,12 @@
parameters:
from:
value: "t-1d"
hidden: false
to:
value:
hidden: false

896
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.fkr Normal file
View File

@@ -0,0 +1,896 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=balance
Name=Balance Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Balance Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=number,id,date,amount,accrued_interest_percent,client_id,client_unit_id,bank_id,bank_unit_id,account_id,currency_id,cp_client_id,cp_client_unit_id,type,order_number,flags,name,portfolio_id,instrument_id,instrument_group,state_id,payment_id,payment_date,payment_currency_id,payment_amount,fx_spot_rate,affect_number,payment_client_id,payment_client_unit_id,credit_client_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19
Selected=number,id,date,amount,accrued_interest_percent,client_id,client_unit_id,bank_id,bank_unit_id,account_id,currency_id,cp_client_id,cp_client_unit_id,type,order_number,flags,name,portfolio_id,instrument_id,instrument_group,state_id,payment_id,payment_date,payment_currency_id,payment_amount,fx_spot_rate,affect_number,payment_client_id,payment_client_unit_id,credit_client_id,param_0,param_1,param_2,param_3,param_4,param_5,param_6,param_7,param_8,param_9,param_10,param_11,param_12,param_13,param_14,param_15,param_16,param_17,param_18,param_19
Grouping=
Sorting=
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
Fields=from_date,to_date,portfolio_id,currency_id,currency_class_id,client_id,client_unit_id,bank_id,bank_unit_id,account_id,minimum_state_id,rule_id,display_all_balances_p,display_ai_p
from_date=
to_date=
portfolio_id=
currency_id=
currency_class_id=
client_id=
client_unit_id=
bank_id=
bank_unit_id=
account_id=
minimum_state_id=
rule_id=
display_all_balances_p=
display_ai_p=
[Format number]
Name=NUMBER_
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format id]
Name=ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format date]
Name=DATE_
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format amount]
Name=AMOUNT
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=4
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format accrued_interest_percent]
Name=ACCRUED_INTEREST_PERCENT
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=10
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format client_id]
Name=CLIENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format client_unit_id]
Name=CLIENT_UNIT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format bank_id]
Name=BANK_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format bank_unit_id]
Name=BANK_UNIT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format account_id]
Name=ACCOUNT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format currency_id]
Name=CURRENCY_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format cp_client_id]
Name=CP_CLIENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format cp_client_unit_id]
Name=CP_CLIENT_UNIT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format type]
Name=TYPE
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format order_number]
Name=ORDER_NUMBER
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format flags]
Name=FLAGS
Expression=
report_total=
date_format=
time_format=
Justify=left
width=119
width_in_characters=23
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format name]
Name=NAME
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format portfolio_id]
Name=PORTFOLIO_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format instrument_id]
Name=INSTRUMENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format instrument_group]
Name=INSTRUMENT_GROUP
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format state_id]
Name=STATE_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_id]
Name=PAYMENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_date]
Name=PAYMENT_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_currency_id]
Name=PAYMENT_CURRENCY_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_amount]
Name=PAYMENT_AMOUNT
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=4
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format fx_spot_rate]
Name=FX_SPOT_RATE
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=10
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format affect_number]
Name=AFFECT_NUMBER
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_client_id]
Name=PAYMENT_CLIENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format payment_client_unit_id]
Name=PAYMENT_CLIENT_UNIT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format credit_client_id]
Name=CREDIT_CLIENT_ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_0]
Name=PARAM_0
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_1]
Name=PARAM_1
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_2]
Name=PARAM_2
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_3]
Name=PARAM_3
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_4]
Name=PARAM_4
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_5]
Name=PARAM_5
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_6]
Name=PARAM_6
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_7]
Name=PARAM_7
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_8]
Name=PARAM_8
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_9]
Name=PARAM_9
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_10]
Name=PARAM_10
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_11]
Name=PARAM_11
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_12]
Name=PARAM_12
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_13]
Name=PARAM_13
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_14]
Name=PARAM_14
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_15]
Name=PARAM_15
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_16]
Name=PARAM_16
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_17]
Name=PARAM_17
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_18]
Name=PARAM_18
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format param_19]
Name=PARAM_19
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

20
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BALANCE.yml Normal file
View File

@@ -0,0 +1,20 @@
parameters:
from_date:
value: "t-10d"
hidden: false
to_date:
value: "t"
hidden: false
display_all_balances_p:
value: 1
hidden: true
display_ai_p:
value: 1
hidden: true

418
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.fkr Normal file
View File

@@ -0,0 +1,418 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=blackoutlog
Name=Blackout History Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Blackout History Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=log_date,log_user,log_action,log_event,log_type,id,isin,programme,active_from,active_until,event_date,user1,user2,flags,description,comment,log_caller,rule_id,name,tms_instrument_id,log_id
Selected=log_id,log_date,log_user,log_action,id,isin,programme,active_from,active_until,event_date,tms_instrument_id,rule_id,description,comment,user1,user2,name
Grouping=
Sorting=
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
DigitGrouping=03
[Parameters]
[Format log_date]
Name=Log Date
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=130
width_in_characters=26
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format log_user]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=88
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format log_action]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=86
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format log_event]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=60
width_in_characters=10
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format log_type]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=60
width_in_characters=10
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format id]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=81
width_in_characters=16
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format isin]
Name=ISIN
Expression=
report_total=
date_format=
time_format=
Justify=left
width=92
width_in_characters=18
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format programme]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=87
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format active_from]
Name=Active From
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=
Justify=left
width=85
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format active_until]
Name=Active Until
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=
Justify=left
width=97
width_in_characters=19
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format event_date]
Name=Event Date
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=
Justify=left
width=101
width_in_characters=20
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format user1]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=86
width_in_characters=17
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format user2]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=75
width_in_characters=15
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format flags]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=60
width_in_characters=10
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format description]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=147
width_in_characters=29
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format comment]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=111
width_in_characters=22
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format log_caller]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=125
width_in_characters=10
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format rule_id]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=202
width_in_characters=40
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format name]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=130
width_in_characters=26
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0
[Format tms_instrument_id]
Name=TMS Instrument ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=279
width_in_characters=55
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=1
ExportHeaderRows=1
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1
[Format log_id]
Name=
Expression=
report_total=
date_format=
time_format=
Justify=
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=1
NoNumberFormatting=0

9
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BLACKOUT_LOG.yml Normal file
View File

@@ -0,0 +1,9 @@
parameters:
min_log_id:
value: select("w_ODS_TMS_BLACKOUT_LOG", "max(log_id) + 1")
hidden: false

145
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.fkr Normal file
View File

@@ -0,0 +1,145 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=branch
Name=Branch Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Branch Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=id,number,name,order_number
Selected=id,name,number,order_number
Grouping=
Sorting=number,order_number
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
[Format id]
Name=id
Expression=
report_total=
date_format=
time_format=
Justify=left
width=149
width_in_characters=29
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format number]
Name=number_
Expression=
report_total=
date_format=
time_format=
Justify=right
width=90
width_in_characters=18
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format name]
Name=name
Expression=
report_total=
date_format=
time_format=
Justify=left
width=221
width_in_characters=44
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format order_number]
Name=order_number
Expression=
report_total=
date_format=
time_format=
Justify=right
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

5
airflow/ods/tms/TMS-layouts/w_ODS_TMS_BRANCH.yml Normal file
View File

@@ -0,0 +1,5 @@
parameters:

161
airflow/ods/tms/TMS-layouts/w_ODS_TMS_CALENDAR.fkr Normal file
View File

@@ -0,0 +1,161 @@
# encoding: UTF-8
[fk report]
Version=2.3
VersionWarning=
[Main]
Type=calendar
Name=Calendar Report
ExcelWorkbook=
HeaderLeft=[\n]%pagebreak
HeaderCenter=Calendar Report
HeaderRight=%datetime[\n]Page %pagenum/%pagecount
FooterLeft=
FooterCenter=
FooterRight=
Criteria=
TotalsOnly=0
MergeCellsVertically=
MergeCellsHorizontally=
Fields=id,name,holiday_date,holiday_reason,default_holidays
Selected=id,name,holiday_date,holiday_reason,default_holidays
Grouping=
Sorting=
SortDescending=
GroupSortLevel=0
GroupSortColumns=
GroupSortDescending=
Pagebreak=0
ZoomPercent=100
FontGridHeading=Tahoma,-11,700,
Font0=Tahoma,-11,400,i
Font1=Tahoma,-11,400,
Font2=Tahoma,-11,700,
FontGridHeadingDrillDown=Tahoma,-11,700,
Font1DrillDown=Tahoma,-11,400,u
Font2DrillDown=Tahoma,-11,700,
GridHeadingColors=14005927, 6441018, 6441018
HeaderRowColors=13812152, 6441018
DataRowColors=16182761, 0, 0
TotalRowColors=14005927, 6441018, 6441018
DecimalSeparator=.
ThousandsSeparator=,
[Parameters]
[Format id]
Name=ID
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format name]
Name=NAME
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format holiday_date]
Name=HOLIDAY_DATE
Expression=
report_total=
date_format=dd/MM/yyyy
time_format=HH:mm:ss
Justify=left
width=156
width_in_characters=31
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format holiday_reason]
Name=HOLIDAY_REASON
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Format default_holidays]
Name=DEFAULT_HOLIDAYS
Expression=
report_total=
date_format=
time_format=
Justify=left
width=60
width_in_characters=12
divider=1
precision=0
max_precision=0
show_zero=0
NoNumberFormatting=0
[Paper]
Type=9
Width=29700
Height=21000
Left=2000
Right=2000
Top=2000
Bottom=2000
[Export]
ExportCoverPage=1
ExportReportName=0
ExportHeaderRows=0
ExportPlainRows=1
ExportFooterRows=1
ExportEmptyReport=1
[Print]
PrintVertGridLines=1
PrintHorzGridLines=1
PrintFrame=1
PrintRowNumbers=1
PrintInColor=0
PrintCenterHorizontally=0
PrintCenterVertically=0
PrintTransparentBackground=0
PrintCoverPage=1
PrintHeaderRows=1
PrintPlainRows=1
PrintFooterRows=1

Some files were not shown because too many files have changed in this diff Show More