Merge 3.3.1 into 3.4.0 (MARS-1046 into MARS-1057)

MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkb

@@ -1,6 +1,66 @@
 create or replace PACKAGE BODY CT_MRDS.FILE_MANAGER
 AS
   
+   ----------------------------------------------------------------------------------------------------
+   -- PRIVATE FUNCTION: NORMALIZE_DATE_FORMAT
+   ----------------------------------------------------------------------------------------------------
+   /**
+    * Purpose: Normalize Oracle date format strings for use in external tables
+    * 
+    * Problem: ISO 8601 formats like 'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM' fail because
+    *          literal character 'T' must be enclosed in double quotes for Oracle
+    *          external table DATE column definitions.
+    * 
+    * Solution: Detect unquoted 'T' separator and wrap it in double quotes
+    * 
+    * Parameters:
+    *    pDateFormat - Original date format from A_COLUMN_DATE_FORMAT table
+    * 
+    * Returns: Normalized format with quoted 'T' if applicable
+    * 
+    * Examples:
+    *    Input:  'YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM'
+    *    Output: 'YYYY-MM-DD"T"HH24:MI:SS.FF3TZH:TZM'
+    *    
+    *    Input:  'DD/MM/YYYY HH24:MI:SS' (no T)
+    *    Output: 'DD/MM/YYYY HH24:MI:SS' (unchanged)
+    *    
+    *    Input:  'YYYY-MM-DD"T"HH24:MI:SS' (already quoted)
+    *    Output: 'YYYY-MM-DD"T"HH24:MI:SS' (unchanged)
+    * 
+    * Author: Grzegorz Michalski
+    * Date: 2025-11-27
+    * Version: 1.0.0 (MARS-1046)
+    */
+   FUNCTION NORMALIZE_DATE_FORMAT(pDateFormat VARCHAR2) RETURN VARCHAR2 IS
+      vNormalizedFormat VARCHAR2(500);
+   BEGIN
+      -- Return NULL if input is NULL
+      IF pDateFormat IS NULL THEN
+         RETURN NULL;
+      END IF;
+      
+      vNormalizedFormat := pDateFormat;
+      
+      -- Check if 'T' separator exists and is NOT already quoted
+      -- Pattern: [YMD]T[HM] (date component + T + time component)
+      IF INSTR(vNormalizedFormat, '"T"') = 0 AND 
+         REGEXP_LIKE(vNormalizedFormat, '[YMD]T[HM]') THEN
+         
+         -- Wrap 'T' in double quotes using regex replace
+         -- Pattern matches: (date format char) + T + (time format char)
+         -- Replacement: \1 + "T" + \2
+         vNormalizedFormat := REGEXP_REPLACE(vNormalizedFormat, '([YMD])T([HM])', '\1"T"\2');
+      END IF;
+      
+      RETURN vNormalizedFormat;
+      
+   EXCEPTION
+      WHEN OTHERS THEN
+         -- If normalization fails, return original format (safety fallback)
+         RETURN pDateFormat;
+   END NORMALIZE_DATE_FORMAT;
+
    ----------------------------------------------------------------------------------------------------
 
    FUNCTION GET_SOURCE_FILE_CONFIG(pFileUri               IN VARCHAR2 DEFAULT NULL
@@ -1254,8 +1314,17 @@ AS
            -- Note: field_list uses CHAR() for CSV field definitions - this is correct behavior
            pFieldList := pFieldList || 
             CASE
-             WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') IN ('DATE', 'TIMESTAMP') THEN
+             WHEN rec.data_type = 'DATE' THEN
-                rec.quoted_column_name || ' DATE ' || CHR(39) || GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name) || CHR(39)
+                -- MARS-1046: DATE format - wrap with NORMALIZE_DATE_FORMAT to fix ISO 8601 'T' separator
+                rec.quoted_column_name || ' DATE ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
+             WHEN rec.data_type LIKE 'TIMESTAMP%WITH TIME ZONE' THEN
+                -- MARS-1046: TIMESTAMP WITH TIME ZONE format for ISO 8601 with fractional seconds and timezone
+                -- Syntax: column_name CHAR(length) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK "format"
+                -- Use fixed length of 50 for ISO 8601 format (e.g., "2012-03-02T14:16:23.798+01:00" = 29 chars)
+                rec.quoted_column_name || ' CHAR(50) DATE_FORMAT TIMESTAMP WITH TIME ZONE MASK ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
+             WHEN REGEXP_SUBSTR(rec.data_type, '^[A-Z]+') = 'TIMESTAMP' THEN
+                -- Other TIMESTAMP types (without timezone)
+                rec.quoted_column_name || ' TIMESTAMP ' || CHR(39) || NORMALIZE_DATE_FORMAT(GET_DATE_FORMAT(pTemplateTableName => pTemplateTableName, pColumnName => rec.column_name)) || CHR(39)
              WHEN rec.data_type IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2') THEN
                 -- For CSV field definitions, use data_length for CHAR() specification
                 rec.quoted_column_name || ' CHAR(' || rec.data_length || ')'

MARS_Packages/REL03/MARS-1057/new_version/FILE_MANAGER.pkg

@@ -24,6 +24,7 @@ AS
    -- Version History (Latest changes first)
    VERSION_HISTORY     CONSTANT VARCHAR2(4000) := 
       '3.4.0 (2025-11-27): MARS-1057 - Added CREATE_EXTERNAL_TABLES_SET and CREATE_EXTERNAL_TABLES_BATCH procedures for batch external table creation' || CHR(13)||CHR(10) ||
+      '3.3.1 (2025-11-27): MARS-1046 - Fixed ISO 8601 datetime format parsing with milliseconds and timezone (e.g., 2012-03-02T14:16:23.798+01:00)' || CHR(13)||CHR(10) ||
       '3.3.0 (2025-11-26): MARS-1056 - Fixed VARCHAR2 definitions in GENERATE_EXTERNAL_TABLE_PARAMS to preserve CHAR/BYTE semantics from template tables' || CHR(13)||CHR(10) ||
       '3.2.1 (2025-11-24): MARS-1049 - Added pEncoding parameter support for CSV character set specification' || CHR(13)||CHR(10) ||
       '3.2.0 (2025-10-22): Added package versioning system using centralized ENV_MANAGER functions' || CHR(13)||CHR(10) ||