Merge pull request #5873 from ghalliday/issue11401

HPCC-11401 Ensure strings are correctly converted to utf8 text

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/defvalue.cpp

@@ -544,6 +544,24 @@ const char *StringValue::getStringValue(StringBuffer &out)
     return out.toCharArray();
 }
 
+const char *StringValue::getUTF8Value(StringBuffer &out)
+{
+    ICharsetInfo * srcCharset = type->queryCharset();
+    Owned<ICharsetInfo> asciiCharset = getCharset(asciiAtom);
+    if (queryDefaultTranslation(asciiCharset, srcCharset))
+    {
+        Owned<ITypeInfo> asciiType = getAsciiType(type);
+        Owned<IValue> asciiValue = castTo(asciiType);
+        return asciiValue->getUTF8Value(out);
+    }
+
+    rtlDataAttr temp;
+    unsigned bufflen;
+    rtlStrToUtf8X(bufflen, temp.refstr(), type->getSize(), (const char *)val.get());
+    out.append(rtlUtf8Size(bufflen, temp.getstr()), temp.getstr());
+    return out.toCharArray();
+}
+
 void StringValue::pushDecimalValue()
 {
     DecPushString(type->getSize(),(char *)val.get());

common/deftype/defvalue.ipp

@@ -121,7 +121,7 @@ public:
     virtual __int64 getIntValue();
     virtual void pushDecimalValue();
     virtual const char *getStringValue(StringBuffer &); 
-
+    virtual const char *getUTF8Value(StringBuffer & out);
 }; 
 
 class UnicodeValue : public MemoryValue

ecl/regress/issue11401.ecl

@@ -0,0 +1,4 @@
+ds := dataset('x', { unsigned id; }, thor);
+output(ds,XMLNS('bob1','OhlàlàStraße'));
+output(ds,XMLNS('bob2',U8'OhlàlàStraße'));
+output(ds,XMLNS('bob3',(ebcdic string)'OhlàlàStraße'));     // You really have to be daft to use an ebcdic string...