Merge pull request #8802 from richardkchapman/subst-string-utf8

HPCC-14988 Projecting fields into embeds

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hqlcpp/hqlcpp.cpp

@@ -11847,7 +11847,7 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
             substValue->getUTF8Value(search);
             rtlDataAttr result;
             unsigned resultLen;
-            rtlSubstituteEmbeddedScript(resultLen, result.refstr(), origBody.length(), origBody.str(), fieldlist.length()-1, fieldlist.str()+1, search.length(), search.str());
+            rtlSubstituteEmbeddedScript(resultLen, result.refstr(), origBody.lengthUtf8(), origBody.str(), fieldlist.lengthUtf8()-1, fieldlist.str()+1, search.lengthUtf8(), search.str());
             scriptArgs.append(*createConstant(createUtf8Value(resultLen, result.getstr(), makeUtf8Type(resultLen, NULL))));
         }
     }

rtl/eclrtl/eclrtl.cpp

@@ -5306,7 +5306,7 @@ void rtlSubstituteEmbeddedScript(size32_t &__lenResult, char * &__result, size32
 {
     StringBuffer result;
     ::replaceString(result, rtlUtf8Size(scriptChars, script), script, rtlUtf8Size(searchChars, search), search, rtlUtf8Size(outFieldsChars, outFields), outFields);
-    __lenResult = result.length();
+    __lenResult = result.lengthUtf8();
     __result = result.detach();
 }
 

system/jlib/jstring.cpp

@@ -33,6 +33,7 @@
 #include "jfile.hpp"
 #include "jdebug.hpp"
 #include "jutil.hpp"
+#include "junicode.hpp"
 
 #define DOUBLE_FORMAT   "%.16g"
 #define FLOAT_FORMAT    "%.7g"
@@ -500,6 +501,14 @@ void StringBuffer::setLength(unsigned len)
     curLen = len;
 }
 
+size32_t StringBuffer::lengthUtf8() const
+{
+    size32_t chars = 0;
+    for (unsigned offset=0; offset < curLen; offset += readUtf8Size(buffer+offset))
+        chars++;
+    return chars;
+}
+
 char * StringBuffer::reserve(size32_t size)
 {
     ensureCapacity(size);

system/jlib/jstring.hpp

@@ -49,6 +49,7 @@ public:
     inline bool     isEmpty() const                     { return (curLen == 0); }
     void            setLength(unsigned len);
     inline void     ensureCapacity(unsigned max)        { if (maxLen <= curLen + max) _realloc(curLen + max); }
+    size32_t        lengthUtf8() const;
 
     StringBuffer &  append(char value);
     StringBuffer &  append(unsigned char value);