Merge pull request #11305 from dcamper/hpcc-19868-matchdate2

HPCC-19868 Fix Std.Date parsing of 2-digit years with %Y

Reviewed-By: Gavin Halliday <gavin.halliday@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecllibrary/std/Date.ecl

@@ -637,7 +637,7 @@ EXPORT Seconds_t SecondsFromDateTimeRec(DateTime_rec datetime, BOOLEAN is_local_
     %e          Day of month (two digits, or a space followed by a single digit)
     %m          Month (two digits)
     %t          Whitespace
-    %y          year within century (00-99)
+    %y          Year within century (00-99)
     %Y          Full year (yyyy)
     %j          Julian day (1-366)
 
@@ -696,6 +696,24 @@ EXPORT Time_t FromStringToTime(STRING time_text, VARSTRING format) :=
  *                      (See documentation for strftime)
  * @return              The date that was matched in the string.
  *                      Returns 0 if failed to match.
+ *
+ * Supported characters:
+    %B          Full month name
+    %b or %h    Abbreviated month name
+    %d          Day of month (two digits)
+    %e          Day of month (two digits, or a space followed by a single digit)
+    %m          Month (two digits)
+    %t          Whitespace
+    %y          Year within century (00-99)
+    %Y          Full year (yyyy)
+    %j          Julian day (1-366)
+
+Common date formats
+    American    '%m/%d/%Y'  mm/dd/yyyy
+    Euro        '%d/%m/%Y'  dd/mm/yyyy
+    Iso format  '%Y-%m-%d'  yyyy-mm-dd
+    Iso basic   '%Y%m%d'    yyyymmdd
+                '%d-%b-%Y'  dd-mon-yyyy    e.g., '21-Mar-1954'
  */
 
 EXPORT Date_t MatchDateString(STRING date_text, SET OF VARSTRING formats) :=
@@ -810,10 +828,16 @@ EXPORT STRING ToString(Date_t date, VARSTRING format) := DateToString(date, form
  * @param from_format   The format the date is to be converted from.
  * @param to_format     The format the date is to be converted to.
  * @return              The converted string, or blank if it failed to match the format.
+ * @see                 FromStringToDate
  */
 
-EXPORT STRING ConvertDateFormat(STRING date_text, VARSTRING from_format='%m/%d/%Y', VARSTRING to_format='%Y%m%d') :=
-    DateToString(FromStringToDate(date_text, from_format), to_format);
+EXPORT STRING ConvertDateFormat(STRING date_text, VARSTRING from_format='%m/%d/%Y', VARSTRING to_format='%Y%m%d') := FUNCTION
+    parsedDate := FromStringToDate(date_text, from_format);
+
+    reformatResult := IF(parsedDate = (Date_t)0, '', DateToString(parsedDate, to_format));
+
+    RETURN reformatResult;
+END;
 
 
 /**
@@ -849,6 +873,7 @@ EXPORT STRING ConvertTimeFormat(STRING time_text, VARSTRING from_format='%H%M%S'
  * @param from_formats  The list of formats the date is to be converted from.
  * @param to_format     The format the date is to be converted to.
  * @return              The converted string, or blank if it failed to match the format.
+ * @see                 MatchDateString
  */
 
 EXPORT STRING ConvertDateFormatMultiple(STRING date_text, SET OF VARSTRING from_formats, VARSTRING to_format='%Y%m%d') := FUNCTION

ecllibrary/teststd/Date/TestFormat.ecl

@@ -7,6 +7,7 @@ IMPORT Std.Date;
 EXPORT TestFormat := MODULE
 
   SHARED DateFormats := ['%d %b %Y', '%Y %b %d', '%Y%m%d', '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'];
+  SHARED DateFormats2Y := ['%d %b %y', '%y %b %d', '%y%m%d', '%y-%m-%d', '%d/%m/%y', '%m/%d/%y'];
   SHARED TimeFormats := ['%H%M%S', '%H:%M:%S', '%H:%M'];
 
   EXPORT TestConstant := [
@@ -21,6 +22,7 @@ EXPORT TestFormat := MODULE
     ASSERT(Date.FromStringToDate('1 \t De   2056', '%d %b %Y') = 0, CONST);
     ASSERT(Date.FromStringToDate('1December1', '%d%b%Y') = 00011201, CONST);
     ASSERT(Date.FromStringToDate('1970-02-01', '%Y-%m-%d') = 19700201, CONST);
+    ASSERT(Date.FromStringToDate('', '%Y-%m-%d') = 0, CONST); // HPCC-16780; Invalid input date
 
     ASSERT(Date.FromStringToTime('12:34:56', '%H:%M:%S') = 123456, CONST);
 
@@ -49,12 +51,21 @@ EXPORT TestFormat := MODULE
 
   EXPORT TestDynamic := [
     ASSERT(Date.MatchDateString('1dec2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1dec11',DateFormats2Y) = 20111201);
     ASSERT(Date.MatchDateString('2011dec1',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('11dec1',DateFormats2Y) = 20011211);
     ASSERT(Date.MatchDateString('1 december 2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1 december 11',DateFormats2Y) = 20111201);
     ASSERT(Date.MatchDateString('2011\tdecem\t01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('11\tdecem\t01',DateFormats2Y) = 20011211);
     ASSERT(Date.MatchDateString('20111201',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('111201',DateFormats2Y) = 20111201);
     ASSERT(Date.MatchDateString('2011-12-01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('11-12-01',DateFormats2Y) = 20111201);
     ASSERT(Date.MatchDateString('1/12/2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1/12/11',DateFormats2Y) = 20111201);
+    ASSERT(Date.MatchDateString('1/12/11',DateFormats) = 111201);
+    ASSERT(Date.MatchDateString('1/12/11',DateFormats2Y) = 20111201);
 
     ASSERT(Date.DateToString(19700101, '%Y-%m-%d') = '1970-01-01');
     ASSERT(Date.DateToString(19700101, '%d/%m/%y') = '01/01/70');
@@ -72,12 +83,19 @@ EXPORT TestFormat := MODULE
     ASSERT(Date.SecondsToString(917872496,'%Y-%m-%dT%H:%M:%S') = '1999-02-01T12:34:56');
 
     ASSERT(Date.ConvertDateFormat('1/12/2011','%m/%d/%Y','%Y-%m-%d') = '2011-01-12');
+    ASSERT(Date.ConvertDateFormat('','%m/%d/%Y','%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
+    ASSERT(Date.ConvertDateFormat('1234','%m/%d/%Y','%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
 
     ASSERT(Date.ConvertTimeFormat('123456','%H%M%S','%H:%M:%S') = '12:34:56');
 
     ASSERT(Date.ConvertDateFormatMultiple('1/31/2011',DateFormats,'%Y-%m-%d') = '2011-01-31');
-
-    ASSERT(Date.ConvertDateFormatMultiple('',DateFormats,'%Y-%m-%d') = '');
+    ASSERT(Date.ConvertDateFormatMultiple('1/31/11',DateFormats2Y,'%Y-%m-%d') = '2011-01-31');
+    ASSERT(Date.ConvertDateFormatMultiple('1/31/11',DateFormats,'%Y-%m-%d') = '11-01-31');
+    ASSERT(Date.ConvertDateFormatMultiple('1/31/11',DateFormats2Y,'%Y-%m-%d') = '2011-01-31');
+    ASSERT(Date.ConvertDateFormatMultiple('',DateFormats,'%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
+    ASSERT(Date.ConvertDateFormatMultiple('',DateFormats2Y,'%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
+    ASSERT(Date.ConvertDateFormatMultiple('1234',DateFormats,'%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
+    ASSERT(Date.ConvertDateFormatMultiple('1234',DateFormats2Y,'%Y-%m-%d') = ''); // HPCC-16780; Invalid input date
 
     ASSERT(Date.ConvertTimeFormatMultiple('123456',TimeFormats,'%H:%M:%S') = '12:34:56');
 

plugins/timelib/timelib.cpp

@@ -568,33 +568,30 @@ TIMELIB_API void TIMELIB_CALL tlDateToString(size32_t &__lenResult, char* &__res
     __result = NULL;  // Return blank string on error
     __lenResult = 0;
 
-    // date is expected to be in form year*10000 + month * 100 + day, and must be later than 1900
-    // or we can't store it in a struct tm
+    // date is expected to be in form year*10000 + month * 100 + day
 
-    if (date >= 1900 * 10000)
-    {
-        struct tm       timeInfo;
-        const size_t    kBufferSize = 256;
-        char            buffer[kBufferSize];
+    struct tm       timeInfo;
+    const size_t    kBufferSize = 256;
+    char            buffer[kBufferSize];
 
-        memset(&timeInfo, 0, sizeof(timeInfo));
-        tlInsertDateIntoTimeStruct(&timeInfo, date);
-        timeInfo.tm_isdst = -1;
-        tlMKTime(&timeInfo);
+    memset(&timeInfo, 0, sizeof(timeInfo));
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+    timeInfo.tm_isdst = -1;
+    tlMKTime(&timeInfo);
 
 #if defined(__clang__) || defined(__GNUC__)
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
 #endif
-        __lenResult = strftime(buffer, kBufferSize, format, &timeInfo);
+    __lenResult = strftime(buffer, kBufferSize, format, &timeInfo);
 #if defined(__clang__) || defined(__GNUC__)
 #pragma GCC diagnostic pop
 #endif
-        if (__lenResult > 0)
-        {
-            __result = reinterpret_cast<char*>(CTXMALLOC(parentCtx, __lenResult));
-            memcpy(__result, buffer, __lenResult);
-        }
+
+    if (__lenResult > 0)
+    {
+        __result = reinterpret_cast<char*>(CTXMALLOC(parentCtx, __lenResult));
+        memcpy(__result, buffer, __lenResult);
     }
 }