init

python/mrds_common/mrds/docs/rqsd_sample.yaml

@@ -0,0 +1,186 @@
+# static configs
+tmpdir: /tmp
+inbox_prefix: INBOX/RQSD/RQSD_PROCESS
+workflow_name: w_ODS_RQSD_PROCESS_DEVO
+validation_schema_path: None
+file_type: csv
+
+# task configs
+tasks:
+  - task_name: m_ODS_RQSD_OBSERVATIONS_PARSE 
+    ods_prefix: INBOX/RQSD/RQSD_PROCESS/RQSD_OBSERVATIONS
+    output_table: RQSD_OBSERVATIONS
+    output_columns:
+      - type: 'workflow_key'
+        column_header: 'A_WORKFLOW_HISTORY_KEY'
+      - type: 'csv_header'
+        value: 'datacollectioncode'
+        column_header: 'datacollectioncode'
+      - type: 'csv_header'
+        value: 'datacollectionname'
+        column_header: 'datacollectionname'
+      - type: 'csv_header'
+        value: 'datacollectionowner'
+        column_header: 'datacollectionowner'
+      - type: 'csv_header'
+        value: 'reportingcyclename'
+        column_header: 'reportingcyclename'
+      - type: 'csv_header'
+        value: 'reportingcyclestatus'
+        column_header: 'reportingcyclestatus'
+      - type: 'csv_header'
+        value: 'modulecode'
+        column_header: 'modulecode'
+      - type: 'csv_header'
+        value: 'modulename'
+        column_header: 'modulename'
+      - type: 'csv_header'
+        value: 'moduleversionnumber'
+        column_header: 'moduleversionnumber'
+      - type: 'csv_header'
+        value: 'reportingentitycollectionuniqueid'
+        column_header: 'reportingentitycollectionuniqueid'
+      - type: 'csv_header'
+        value: 'entityattributereportingcode'
+        column_header: 'entityattributereportingcode'
+      - type: 'csv_header'
+        value: 'reportingentityname'
+        column_header: 'reportingentityname'
+      - type: 'csv_header'
+        value: 'reportingentityentitytype'
+        column_header: 'reportingentityentitytype'
+      - type: 'csv_header'
+        value: 'entityattributecountry'
+        column_header: 'entityattributecountry'
+      - type: 'csv_header'
+        value: 'entitygroupentityname'
+        column_header: 'entitygroupentityname'
+      - type: 'csv_header'
+        value: 'obligationmodulereferencedate'
+        column_header: 'obligationmodulereferencedate'
+      - type: 'csv_header'
+        value: 'obligationmoduleremittancedate'
+        column_header: 'obligationmoduleremittancedate'
+      - type: 'csv_header'
+        value: 'receivedfilereceiveddate'
+        column_header: 'receivedfilereceiveddate'
+      - type: 'csv_header'
+        value: 'obligationmoduleexpected'
+        column_header: 'obligationmoduleexpected'
+      - type: 'csv_header'
+        value: 'receivedfileversionnumber'
+        column_header: 'receivedfileversionnumber'
+      - type: 'csv_header'
+        value: 'revalidationversionnumber'
+        column_header: 'revalidationversionnumber'
+      - type: 'csv_header'
+        value: 'revalidationdate'
+        column_header: 'revalidationdate'
+      - type: 'csv_header'
+        value: 'receivedfilesystemfilename'
+        column_header: 'receivedfilesystemfilename'
+      - type: 'csv_header'
+        value: 'obligationstatusstatus'
+        column_header: 'obligationstatusstatus'
+      - type: 'csv_header'
+        value: 'filestatussetsubmissionstatus'
+        column_header: 'filestatussetsubmissionstatus'
+      - type: 'csv_header'
+        value: 'filestatussetvalidationstatus'
+        column_header: 'filestatussetvalidationstatus'
+      - type: 'csv_header'
+        value: 'filestatussetexternalvalidationstatus'
+        column_header: 'filestatussetexternalvalidationstatus'
+      - type: 'csv_header'
+        value: 'numberoferrors'
+        column_header: 'numberoferrors'
+      - type: 'csv_header'
+        value: 'numberofwarnings'
+        column_header: 'numberofwarnings'
+      - type: 'csv_header'
+        value: 'delayindays'
+        column_header: 'delayindays'
+      - type: 'csv_header'
+        value: 'failedattempts'
+        column_header: 'failedattempts'
+      - type: 'csv_header'
+        value: 'observationvalue'
+        column_header: 'observationvalue'
+      - type: 'csv_header'
+        value: 'observationtextvalue'
+        column_header: 'observationtextvalue'
+      - type: 'csv_header'
+        value: 'observationdatevalue'
+        column_header: 'observationdatevalue'
+      - type: 'csv_header'
+        value: 'datapointsetdatapointidentifier'
+        column_header: 'datapointsetdatapointidentifier'
+      - type: 'csv_header'
+        value: 'datapointsetlabel'
+        column_header: 'datapointsetlabel'
+      - type: 'csv_header'
+        value: 'obsrvdescdatatype'
+        column_header: 'obsrvdescdatatype'
+      - type: 'csv_header'
+        value: 'ordinatecode'
+        column_header: 'ordinatecode'
+      - type: 'csv_header'
+        value: 'ordinateposition'
+        column_header: 'ordinateposition'
+      - type: 'csv_header'
+        value: 'tablename'
+        column_header: 'tablename'
+      - type: 'csv_header'
+        value: 'isstock'
+        column_header: 'isstock'
+      - type: 'csv_header'
+        value: 'scale'
+        column_header: 'scale'
+      - type: 'csv_header'
+        value: 'currency'
+        column_header: 'currency'
+      - type: 'csv_header'
+        value: 'numbertype'
+        column_header: 'numbertype'
+      - type: 'csv_header'
+        value: 'ismandatory'
+        column_header: 'ismandatory'
+      - type: 'csv_header'
+        value: 'decimalplaces'
+        column_header: 'decimalplaces'
+      - type: 'csv_header'
+        value: 'serieskey'
+        column_header: 'serieskey'
+      - type: 'csv_header'
+        value: 'tec_source_system'
+        column_header: 'tec_source_system'
+      - type: 'csv_header'
+        value: 'tec_dataset'
+        column_header: 'tec_dataset'
+      - type: 'csv_header'
+        value: 'tec_surrogate_key'
+        column_header: 'tec_surrogate_key'
+      - type: 'csv_header'
+        value: 'tec_crc'
+        column_header: 'tec_crc'
+      - type: 'csv_header'
+        value: 'tec_ingestion_date'
+        column_header: 'tec_ingestion_date'
+      - type: 'csv_header'
+        value: 'tec_version_id'
+        column_header: 'tec_version_id'
+      - type: 'csv_header'
+        value: 'tec_execution_date'
+        column_header: 'tec_execution_date'
+      - type: 'csv_header'
+        value: 'tec_run_id'
+        column_header: 'tec_run_id'
+      - type: 'static'
+        value: 'test test'
+        column_header: 'BLABLA'
+      - type: 'a_key'
+        column_header: 'A_KEY'
+      - type: 'csv_header'
+        value: 'tec_business_date'
+        column_header: 'tec_business_dateTest!'
+

python/mrds_common/mrds/docs/upload.py

@@ -0,0 +1,50 @@
+# file uploader
+import os
+import sys
+import logging
+from mrds.utils import objectstore
+
+BUCKET = os.getenv("INBOX_BUCKET", "mrds_inbox_poc")
+BUCKET_NAMESPACE = os.getenv("BUCKET_NAMESPACE", "frcnomajoc7v")
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(name)s - %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+    ],
+)
+
+
+source_filepath = '/home/dbt/tmp/mrds_4twsw_ib/20250630_Pre-Production_DV_P2_DBT_I4.zip'
+source_filename = '20250630_Pre-Production_DV_P2_DBT_I4.zip'
+target_prefix = 'INBOX/CSDB/STC_CentralizedSecuritiesDissemination_ECB'
+
+def upload_file():
+
+    client = objectstore.get_client()
+
+    logging.info(
+        f"uploading source file to '{BUCKET}/{target_prefix}/{source_filename}'"
+    )
+    objectstore.upload_file(
+        client,
+        source_filepath,
+        BUCKET_NAMESPACE,
+        BUCKET,
+        target_prefix,
+        source_filename,
+    )
+    logging.info(
+        f"Source file uploaded to '{BUCKET}/{target_prefix}/{source_filename}'"
+    )
+
+
+if __name__ == "__main__":
+    try:
+        upload_file()
+        sys.exit(0)
+    except Exception as e:
+        logging.error(f"Unexpected error: {e}")
+        sys.exit(1)