diff --git a/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql b/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql index 9133189..aecfce7 100644 --- a/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql +++ b/MARS_Packages/REL02_POST/MARS-956/01_MARS_956_export_c2d_mpec_data.sql @@ -3,8 +3,8 @@ -- =================================================================== -- Purpose: One-time export of historical C2D MPEC delta data from -- OU_C2D operational database to DATA bucket as CSV files --- Method: Using DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE procedure --- Target: DATA bucket with folder structure ODS/C2D/{TABLE_NAME} +-- Method: Using DATA_EXPORTER.EXPORT_TABLE_DATA procedure +-- Target: DATA bucket with folder structure DATA/C2D/{TABLE_NAME} -- Format: CSV files for complete historical data access -- =================================================================== @@ -14,8 +14,9 @@ PROMPT ========================================================================= PROMPT Export Strategy: PROMPT - Source: OU_C2D schema tables (operational database) PROMPT - Target: DATA bucket as CSV files -PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE +PROMPT - Method: DATA_EXPORTER.EXPORT_TABLE_DATA PROMPT - Structure: Must match ODS template tables +PROMPT - Registration: Files registered in A_SOURCE_FILE_RECEIVED PROMPT ========================================================================= -- Log export start @@ -25,24 +26,21 @@ VALUES ('MARS-956', 'EXPORT_C2D_MPEC_DATA', 'INFO', 'Starting historical C2D MPE COMMIT; -- =================================================================== --- TABLE 1: OU_C2D.MPEC_ADMIN -> ODS/C2D/C2D_MPEC_ADMIN +-- TABLE 1: OU_C2D.MPEC_ADMIN -> DATA/C2D/C2D_MPEC_ADMIN -- =================================================================== PROMPT Exporting Table 1/3: OU_C2D.MPEC_ADMIN -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN +PROMPT Target: mrds_data_dev/DATA/C2D/C2D_MPEC_ADMIN BEGIN - CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE( + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_C2D', pTableName => 'MPEC_ADMIN', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', -- Adjust if different + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'DATA', - pFolderName => 'ODS/C2D/C2D_MPEC_ADMIN', - pColumnList => NULL, -- Use all columns - pMinDate => DATE '2020-01-01', -- Full historical range - pMaxDate => SYSDATE, -- Up to current date - pParallelDegree => 1, -- Sequential for safety - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_ADMIN', -- Template for mapping + pFolderName => 'DATA/C2D/C2D_MPEC_ADMIN', + pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_ADMIN', -- Template for column order + pRegisterExport => TRUE, -- Register files in A_SOURCE_FILE_RECEIVED pCredentialName => 'DEF_CRED_ARN' ); @@ -59,24 +57,21 @@ END; / -- =================================================================== --- TABLE 2: OU_C2D.MPEC_CONTENT -> ODS/C2D/C2D_MPEC_CONTENT +-- TABLE 2: OU_C2D.MPEC_CONTENT -> DATA/C2D/C2D_MPEC_CONTENT -- =================================================================== PROMPT Exporting Table 2/3: OU_C2D.MPEC_CONTENT -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT +PROMPT Target: mrds_data_dev/DATA/C2D/C2D_MPEC_CONTENT BEGIN - CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE( + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_C2D', pTableName => 'MPEC_CONTENT', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', -- Adjust if different + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'DATA', - pFolderName => 'ODS/C2D/C2D_MPEC_CONTENT', - pColumnList => NULL, -- Use all columns - pMinDate => DATE '2020-01-01', -- Full historical range - pMaxDate => SYSDATE, -- Up to current date - pParallelDegree => 1, -- Sequential for safety - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT', -- Template for mapping + pFolderName => 'DATA/C2D/C2D_MPEC_CONTENT', + pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT', -- Template for column order + pRegisterExport => TRUE, -- Register files in A_SOURCE_FILE_RECEIVED pCredentialName => 'DEF_CRED_ARN' ); @@ -93,24 +88,21 @@ END; / -- =================================================================== --- TABLE 3: OU_C2D.MPEC_CONTENT_CRITERION -> ODS/C2D/C2D_MPEC_CONTENT_CRITERION +-- TABLE 3: OU_C2D.MPEC_CONTENT_CRITERION -> DATA/C2D/C2D_MPEC_CONTENT_CRITERION -- =================================================================== PROMPT Exporting Table 3/3: OU_C2D.MPEC_CONTENT_CRITERION -PROMPT Target: mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION +PROMPT Target: mrds_data_dev/DATA/C2D/C2D_MPEC_CONTENT_CRITERION BEGIN - CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE( + CT_MRDS.DATA_EXPORTER.EXPORT_TABLE_DATA( pSchemaName => 'OU_C2D', pTableName => 'MPEC_CONTENT_CRITERION', - pKeyColumnName => 'A_WORKFLOW_HISTORY_KEY', -- Adjust if different + pKeyColumnName => 'A_ETL_LOAD_SET_FK', -- ETL key for data lookup pBucketArea => 'DATA', - pFolderName => 'ODS/C2D/C2D_MPEC_CONTENT_CRITERION', - pColumnList => NULL, -- Use all columns - pMinDate => DATE '2020-01-01', -- Full historical range - pMaxDate => SYSDATE, -- Up to current date - pParallelDegree => 1, -- Sequential for safety - pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT_CRITERION', -- Template for mapping + pFolderName => 'DATA/C2D/C2D_MPEC_CONTENT_CRITERION', + pTemplateTableName => 'CT_ET_TEMPLATES.C2D_MPEC_CONTENT_CRITERION', -- Template for column order + pRegisterExport => TRUE, -- Register files in A_SOURCE_FILE_RECEIVED pCredentialName => 'DEF_CRED_ARN' ); diff --git a/MARS_Packages/REL02_POST/MARS-956/README.md b/MARS_Packages/REL02_POST/MARS-956/README.md index 71451c9..e6935f3 100644 --- a/MARS_Packages/REL02_POST/MARS-956/README.md +++ b/MARS_Packages/REL02_POST/MARS-956/README.md @@ -4,7 +4,7 @@ **Purpose**: One-time export of historical C2D MPEC delta data from operational database (OU_C2D) to DATA bucket as CSV files. -**Approach**: Use FILE_MANAGER export functionality EXPORT_TABLE_DATA_BY_DATE for bulk data movement. +**Approach**: Use DATA_EXPORTER export functionality EXPORT_TABLE_DATA for bulk data movement with file registration. **Input**: Old tables in OU_C2D operational database **Output**: CSV files in DATA bucket @@ -13,18 +13,19 @@ ## Tables to Export | Source Table (OU_C2D) | Target Location (DATA) | Export Type | Time Dependency | -|------------------------|-------------------------|-------------|-----------------| -| `MPEC_ADMIN` | `mrds_data_dev/ODS/C2D/C2D_MPEC_ADMIN` | CSV to DATA | Sync with REL_02 | -| `MPEC_CONTENT` | `mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT` | CSV to DATA | Sync with REL_02 | -| `MPEC_CONTENT_CRITERION` | `mrds_data_dev/ODS/C2D/C2D_MPEC_CONTENT_CRITERION` | CSV to DATA | Sync with REL_02 | +|------------------------|-------------------------|-------------|------------------| +| `MPEC_ADMIN` | `mrds_data_dev/DATA/C2D/C2D_MPEC_ADMIN` | CSV to DATA | Sync with REL_02 | +| `MPEC_CONTENT` | `mrds_data_dev/DATA/C2D/C2D_MPEC_CONTENT` | CSV to DATA | Sync with REL_02 | +| `MPEC_CONTENT_CRITERION` | `mrds_data_dev/DATA/C2D/C2D_MPEC_CONTENT_CRITERION` | CSV to DATA | Sync with REL_02 | ## Export Strategy - **Format**: CSV files in DATA bucket - **Reason**: Complete history of delta records needed for all queries -- **Method**: `DATA_EXPORTER.EXPORT_TABLE_DATA_BY_DATE` procedure +- **Method**: `DATA_EXPORTER.EXPORT_TABLE_DATA` procedure - **Bucket Area**: `'DATA'` -- **Folder Structure**: `'ODS/C2D/{TABLE_NAME}'` +- **Folder Structure**: `'DATA/C2D/{TABLE_NAME}'` +- **File Registration**: Files registered in A_SOURCE_FILE_RECEIVED table ## Installation Steps @@ -47,7 +48,7 @@ MARS-956/ ## Prerequisites - OU_C2D schema access for source tables -- DATA_EXPORTER package v2.7.4+ deployed +- DATA_EXPORTER package v2.7.5+ deployed (with pRegisterExport support) - DEF_CRED_ARN credentials configured - DATA bucket accessible @@ -57,6 +58,7 @@ MARS-956/ 2. Verify CSV files created in DATA bucket 3. Validate file structure matches template tables 4. Confirm row counts match source tables +5. Check file registration in A_SOURCE_FILE_RECEIVED table ## Notes diff --git a/MARS_Packages/REL02_POST/MARS-956/validate_export.sql b/MARS_Packages/REL02_POST/MARS-956/validate_export.sql index 865225b..e7cbc38 100644 --- a/MARS_Packages/REL02_POST/MARS-956/validate_export.sql +++ b/MARS_Packages/REL02_POST/MARS-956/validate_export.sql @@ -85,24 +85,62 @@ GROUP BY owner, table_name ORDER BY table_name, owner; PROMPT -PROMPT 4. Export File Validation Commands +PROMPT 4. File Registration Validation +PROMPT ========================================================================= + +-- Check if exported files were registered in A_SOURCE_FILE_RECEIVED +PROMPT Registered export files (last 24 hours): +SELECT A_SOURCE_FILE_RECEIVED_KEY, + A_SOURCE_FILE_CONFIG_KEY, + SOURCE_FILE_NAME, + ROUND(BYTES/1024, 2) AS SIZE_KB, + PROCESSING_STATUS, + TO_CHAR(RECEPTION_DATE, 'YYYY-MM-DD HH24:MI:SS') AS REGISTERED_TIME +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +WHERE RECEPTION_DATE >= SYSDATE - 1 -- Last 24 hours + AND (SOURCE_FILE_NAME LIKE '%MPEC_%' OR A_SOURCE_FILE_CONFIG_KEY IN ( + SELECT A_SOURCE_FILE_CONFIG_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_KEY = 'C2D' AND TABLE_ID LIKE '%MPEC%' + )) +ORDER BY RECEPTION_DATE DESC; + +-- Count registered files per config key +PROMPT File registration summary: +SELECT + CASE WHEN A_SOURCE_FILE_CONFIG_KEY = -1 THEN 'Default (no config)' + ELSE 'Config Key: ' || A_SOURCE_FILE_CONFIG_KEY + END AS CONFIG_INFO, + COUNT(*) AS REGISTERED_FILES +FROM CT_MRDS.A_SOURCE_FILE_RECEIVED +WHERE RECEPTION_DATE >= SYSDATE - 1 -- Last 24 hours + AND (SOURCE_FILE_NAME LIKE '%MPEC_%' OR A_SOURCE_FILE_CONFIG_KEY IN ( + SELECT A_SOURCE_FILE_CONFIG_KEY + FROM CT_MRDS.A_SOURCE_FILE_CONFIG + WHERE A_SOURCE_KEY = 'C2D' AND TABLE_ID LIKE '%MPEC%' + )) +GROUP BY A_SOURCE_FILE_CONFIG_KEY +ORDER BY A_SOURCE_FILE_CONFIG_KEY; + +PROMPT +PROMPT 5. Export File Validation Commands PROMPT ========================================================================= PROMPT To validate exported CSV files, use these OCI CLI commands: PROMPT PROMPT # List exported files -PROMPT oci os object list --bucket-name mrds_data_dev --prefix "ODS/C2D/C2D_MPEC" +PROMPT oci os object list --bucket-name mrds_data_dev --prefix "DATA/C2D/C2D_MPEC" PROMPT PROMPT # Check file sizes -PROMPT oci os object list --bucket-name mrds_data_dev --prefix "ODS/C2D/C2D_MPEC_ADMIN" -PROMPT oci os object list --bucket-name mrds_data_dev --prefix "ODS/C2D/C2D_MPEC_CONTENT" -PROMPT oci os object list --bucket-name mrds_data_dev --prefix "ODS/C2D/C2D_MPEC_CONTENT_CRITERION" +PROMPT oci os object list --bucket-name mrds_data_dev --prefix "DATA/C2D/C2D_MPEC_ADMIN" +PROMPT oci os object list --bucket-name mrds_data_dev --prefix "DATA/C2D/C2D_MPEC_CONTENT" +PROMPT oci os object list --bucket-name mrds_data_dev --prefix "DATA/C2D/C2D_MPEC_CONTENT_CRITERION" PROMPT PROMPT # Download sample file for validation -PROMPT oci os object get --bucket-name mrds_data_dev --name "ODS/C2D/C2D_MPEC_ADMIN/filename.csv" --file sample.csv +PROMPT oci os object get --bucket-name mrds_data_dev --name "DATA/C2D/C2D_MPEC_ADMIN/filename.csv" --file sample.csv PROMPT -PROMPT 5. Data Quality Checks (Manual) +PROMPT 6. Data Quality Checks (Manual) PROMPT ========================================================================= PROMPT Manual verification steps: @@ -112,9 +150,10 @@ PROMPT 3. Check data formats (especially dates) match expectations PROMPT 4. Confirm row counts approximately match source tables PROMPT 5. Validate no empty files were created PROMPT 6. Test loading sample data into external tables +PROMPT 7. Verify file registration entries in A_SOURCE_FILE_RECEIVED PROMPT -PROMPT 6. Next Steps for ODS Integration +PROMPT 7. Next Steps for ODS Integration PROMPT ========================================================================= PROMPT After validation success: @@ -136,6 +175,7 @@ PROMPT Review the output above to confirm: PROMPT ✓ Export processes completed without errors PROMPT ✓ Source table row counts are reasonable PROMPT ✓ Template tables exist and have matching structure +PROMPT ✓ Exported files registered in A_SOURCE_FILE_RECEIVED table PROMPT ✓ Manual file validation steps are understood PROMPT PROMPT If any issues found, check export logs and re-run specific exports if needed.