HPCC-18034 Various issues with typecasts

Constant folding casts from integer to varstringNN/varunicodeNN were
incorrectly adding trailing spaces.

When casting to varstringNN or varunicodeNN, the length of the target string
should not be taken into account when creating the intermediate string value
(otherwise you are liable to get unwanted trailing spaces).

Casting utf8 to varunicodeNN should not add trailing spaces.

Casting varunicode to data could give internal compiler error.

Add test case.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

HPCC-18034 Various issues with typecasts

Casting utf8 to dataNN should fill with 0 rather than spaces, for consistency
with (most) other string types casting to data.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/defvalue.cpp

@@ -1617,9 +1617,10 @@ IValue *IntValue::castTo(ITypeInfo *t)
     case type_qstring:
     case type_utf8:
     case type_unicode:
+    case type_varstring:
+    case type_varunicode:
         return castViaString(t);
     case type_string:
-    case type_varstring:
     {
         if (nLen == UNKNOWN_LENGTH)
             return castViaString(t);

ecl/hqlcpp/hqlcpp.cpp

@@ -2659,8 +2659,19 @@ void HqlCppTranslator::buildExprAssignViaType(BuildCtx & ctx, const CHqlBoundTar
     buildExprAssign(ctx, target, temp);
 }
 
-void HqlCppTranslator::buildExprAssignViaString(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, unsigned len)
+void HqlCppTranslator::buildExprAssignViaString(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, ITypeInfo *dest)
 {
+    unsigned len;
+    switch (dest->getTypeCode())
+    {
+    case type_varstring:
+    case type_varunicode:
+        len = UNKNOWN_LENGTH;
+        break;
+    default:
+        len = dest->getStringLen();
+        break;
+    }
     OwnedITypeInfo type = makeStringType(len, NULL, NULL);
     buildExprAssignViaType(ctx, target, expr, type);
 }
@@ -10624,7 +10635,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
         default:
             //Need to go via a temporary string.
             OwnedHqlExpr temp = pure.getTranslatedExpr();
-            buildExprAssignViaString(ctx, target, temp, to->getStringLen());
+            buildExprAssignViaString(ctx, target, temp, to);
             return;
         }
         break;
@@ -10741,7 +10752,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
                     if (queryDefaultTranslation(tgtset, from->queryCharset()))
                     {
                         OwnedHqlExpr temp = pure.getTranslatedExpr();
-                        buildExprAssignViaString(ctx, target, temp, to->getStringLen());
+                        buildExprAssignViaString(ctx, target, temp, to);
                     }
                     else
                     {
@@ -10796,7 +10807,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
                             break;
                         case type_varstring:
                             OwnedHqlExpr temp = pure.getTranslatedExpr();
-                            buildExprAssignViaString(ctx, target, temp, to->getStringLen());
+                            buildExprAssignViaString(ctx, target, temp, to);
                             return;
                         }
                         args.append(*getSizetConstant(toSize));
@@ -10940,7 +10951,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
                     if (toType == type_varunicode)
                     {
                         OwnedHqlExpr temp = pure.getTranslatedExpr();
-                        OwnedITypeInfo type = makeUnicodeType(to->getStringLen(), NULL);
+                        OwnedITypeInfo type = makeUnicodeType(UNKNOWN_LENGTH, NULL);
                         buildExprAssignViaType(ctx, target, temp, type);
                         return;
                     }
@@ -10961,7 +10972,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
             }
         default:
             OwnedHqlExpr temp = pure.getTranslatedExpr();
-            buildExprAssignViaString(ctx, target, temp, to->getStringLen());
+            buildExprAssignViaString(ctx, target, temp, to);
             return;
         }
         break;
@@ -11005,7 +11016,7 @@ void HqlCppTranslator::assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & tar
             }
         default:
             OwnedHqlExpr temp = pure.getTranslatedExpr();
-            buildExprAssignViaString(ctx, target, temp, to->getStringLen());
+            buildExprAssignViaString(ctx, target, temp, to);
             return;
         }
         break;

ecl/hqlcpp/hqlcpp.ipp

@@ -1214,7 +1214,7 @@ public:
     void doBuildChoose(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * expr);
 
     void buildExprAssignViaType(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, ITypeInfo * type);
-    void buildExprAssignViaString(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, unsigned len);
+    void buildExprAssignViaString(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, ITypeInfo * dest);
     void doBuildDivideByZero(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * zero, CHqlBoundExpr * bound);
 
     void doBuildAssignAddSets(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * value);

ecl/hqlcpp/hqlcppsys.ecl

@@ -148,6 +148,7 @@ const char * cppSystemText[]  = {
     "   varstring unicode2VCodepageX(const unicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVCodepageX';",
     "   string vunicode2CodepageX(const varunicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToCodepageX';",
     "   varstring vunicode2VCodepageX(const varunicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVCodepageX';",
+    "   data vunicode2DataX(const varunicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToDataX';",
     "   unicode unicode2UnicodeX(const unicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUnicodeX';",
     "   varunicode unicode2VUnicodeX(const unicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVUnicodeX';",
     "   unicode vunicode2UnicodeX(const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToUnicodeX';",

rtl/eclrtl/eclrtl.cpp

@@ -3265,6 +3265,11 @@ void rtlVUnicodeToData(unsigned outlen, void * out, UChar const * in)
     rtlUnicodeToData(outlen, out, rtlUnicodeStrlen(in), in);
 }
 
+void rtlVUnicodeToDataX(unsigned& outlen, void * &out, UChar const * in)
+{
+    rtlUnicodeToDataX(outlen, out, rtlUnicodeStrlen(in), in);
+}
+
 void rtlVUnicodeToVCodepage(unsigned outlen, char * out, UChar const * in, char const * codepage)
 {
     rtlUnicodeToVCodepage(outlen, out, rtlUnicodeStrlen(in), in, codepage);
@@ -4663,7 +4668,13 @@ unsigned rtlUnicodeStrlen(UChar const * str)
 void rtlUtf8ToData(size32_t outlen, void * out, size32_t inlen, const char *in)
 {
     unsigned insize = rtlUtf8Size(inlen, in);
-    rtlCodepageToCodepage(outlen, (char *)out, insize, in, ASCII_LIKE_CODEPAGE, UTF8_CODEPAGE);
+    if (insize >= outlen)
+        rtlCodepageToCodepage(outlen, (char *)out, insize, in, ASCII_LIKE_CODEPAGE, UTF8_CODEPAGE);
+    else
+    {
+        rtlCodepageToCodepage(insize, (char *)out, insize, in, ASCII_LIKE_CODEPAGE, UTF8_CODEPAGE);
+        memset((char*)out + insize, 0, outlen-insize);
+    }
 }
 
 void rtlUtf8ToDataX(size32_t & outlen, void * & out, size32_t inlen, const char *in)

rtl/eclrtl/eclrtl.hpp

@@ -335,6 +335,7 @@ ECLRTL_API void rtlUnicodeToData(unsigned outlen, void * out, unsigned inlen, UC
 ECLRTL_API void rtlUnicodeToVCodepage(unsigned outlen, char * out, unsigned inlen, UChar const * in, char const * codepage);
 ECLRTL_API void rtlVUnicodeToCodepage(unsigned outlen, char * out, UChar const * in, char const * codepage);
 ECLRTL_API void rtlVUnicodeToData(unsigned outlen, void * out, UChar const * in);
+ECLRTL_API void rtlVUnicodeToDataX(unsigned& outlen, void * &out, UChar const * in);
 ECLRTL_API void rtlVUnicodeToVCodepage(unsigned outlen, char * out, UChar const * in, char const * codepage);
 ECLRTL_API void rtlCodepageToUnicodeX(unsigned & outlen, UChar * & out, unsigned inlen, char const * in, char const * codepage);
 ECLRTL_API UChar * rtlCodepageToVUnicodeX(unsigned inlen, char const * in, char const * codepage);

testing/regress/ecl/casts2.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// Test some casts to varstring etc
+
+r := record
+  integer i;
+  utf8 s;
+  varunicode vu;
+  varunicode vu5;
+end;
+
+r1 := record
+  varstring5 s1;
+  varstring5 s2;
+  varunicode5 u1;
+  varunicode5 u2;
+  data d;
+  data5 d5;
+  data d1;
+  data5 d15;
+end;
+
+d := dataset([{1,'d', u'€', u'€'}], r);
+
+r1 t(d l) := transform
+  self.s1 := (varstring5) l.i;
+  self.s2 := (varstring5) l.s;
+  self.u1 := (varunicode5) l.i;
+  self.u2 := (varunicode5) l.s;
+  self.d := (data) l.vu;
+  self.d5 := (data5) l.vu;
+  self.d1 := (data) l.vu5;
+  self.d15 := (data5) l.vu5;
+end;
+
+project(d, t(left));
+project(nofold(d), t(left));

testing/regress/ecl/key/casts2.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><s1>1</s1><s2>d</s2><u1>1</u1><u2>d</u2><d>1A</d><d5>1A00000000</d5><d1>1A</d1><d15>1A00000000</d15></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><s1>1</s1><s2>d</s2><u1>1</u1><u2>d</u2><d>1A</d><d5>1A00000000</d5><d1>1A</d1><d15>1A00000000</d15></Row>
+</Dataset>