HPCC-20587 esdl2ecl default generated ECL strings to UTF8

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

esp/xslt/esdl2ecl.xslt

@@ -20,6 +20,7 @@
     <xsl:output method="text" version="1.0" encoding="UTF-8" indent="yes"/>
     <xsl:param name="sourceFileName" select="'UNKNOWN'"/>
     <xsl:param name="importsList" select="''"/>
+    <xsl:param name="utf8strings" select="'yes'"/>
     <xsl:variable name="docname" select="/expesdl/esxdl/@name"/>
     <xsl:template match="/">
         <xsl:apply-templates select="expesdl"/>
@@ -109,7 +110,7 @@ END;
 	</xsl:template>
 	<xsl:template match="EsdlArray[@type='string']|EsdlList[@type='string']">
 		<xsl:if test="not(@ecl_hide) and (@ecl_keep or not(@get_data_from))">
-        <xsl:text disable-output-escaping="yes">	SET OF STRING </xsl:text><xsl:call-template name="output_ecl_name"/>
+        <xsl:text disable-output-escaping="yes">	SET OF </xsl:text><xsl:call-template name="output_ecl_stringtype"/><xsl:text> </xsl:text><xsl:call-template name="output_ecl_name"/>
         <xsl:text disable-output-escaping="yes"> {XPATH('</xsl:text>
 		<xsl:choose>
 			<xsl:when test="name(.) ='EsdlArray'">
@@ -176,7 +177,7 @@ END;
 			<xsl:text disable-output-escaping="yes">
 </xsl:text>
 			<xsl:apply-templates select="*"/>
-            <xsl:if test="@element and not(*[@name='Content_'])">	STRING Content_ {XPATH('')};
+            <xsl:if test="@element and not(*[@name='Content_'])"><xsl:text>	</xsl:text><xsl:call-template name="output_ecl_stringtype"/><xsl:text> </xsl:text> Content_ {XPATH('')};
 </xsl:if>
             <xsl:text disable-output-escaping="yes">END;
 
@@ -239,6 +240,13 @@ END;
 
 </xsl:template>
 
+<xsl:template name="output_ecl_stringtype">
+  <xsl:choose>
+    <xsl:when test="$utf8strings='yes'">UTF8</xsl:when>
+    <xsl:otherwise>STRING</xsl:otherwise>
+  </xsl:choose>
+</xsl:template>
+
 <xsl:template name="output_basic_type">
 	<xsl:param name="basic_type" select="@type"/>
 	<xsl:param name="size" select="@max_len"/>
@@ -249,12 +257,12 @@ END;
         <xsl:when test="$basic_type='short'"><xsl:text disable-output-escaping="yes">INTEGER2</xsl:text></xsl:when>
         <xsl:when test="$basic_type='int64'"><xsl:text disable-output-escaping="yes">INTEGER8</xsl:text></xsl:when>
         <xsl:when test="$basic_type='bool'"><xsl:text disable-output-escaping="yes">BOOLEAN</xsl:text></xsl:when>
-        <xsl:when test="$basic_type='string'"><xsl:text disable-output-escaping="yes">STRING</xsl:text><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
+        <xsl:when test="$basic_type='string'"><xsl:call-template name="output_ecl_stringtype"/><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
         <xsl:when test="$basic_type='double'"><xsl:text disable-output-escaping="yes">REAL8</xsl:text></xsl:when>
         <xsl:when test="$basic_type='float'"><xsl:text disable-output-escaping="yes">REAL4</xsl:text></xsl:when>
         <xsl:when test="$basic_type='base64Binary'"><xsl:text disable-output-escaping="yes">STRING</xsl:text></xsl:when>
 		<xsl:when test="$basic_type"><xsl:value-of select="$basic_type"/><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
-        <xsl:otherwise><xsl:text disable-output-escaping="yes">STRING</xsl:text><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:otherwise>
+        <xsl:otherwise><xsl:call-template name="output_ecl_stringtype"/><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:otherwise>
 	</xsl:choose>
 </xsl:template>
 

tools/esdlcmd/esdl2ecl.cpp

@@ -269,6 +269,8 @@ public:
             return EsdlCmdOptionMatch;
         if (iter.matchFlag(optHPCCCompFilesDir, HPCC_COMPONENT_FILES_DIR_CDE))
             return EsdlCmdOptionMatch;
+        if (iter.matchFlag(optUseUtf8Strings, ESDLOPT_USE_UTF8_STRINGS))
+            return EsdlCmdOptionMatch;
         if (iter.matchOption(optHPCCCompFilesDir, ESDL_OPTION_ECL_INCLUDE_LIST))
             return EsdlCmdOptionMatch;
         if (iter.matchOption(optECLHeaderBlock, ESDL_OPTION_ECL_HEADER_BLOCK))
@@ -389,6 +391,7 @@ public:
                 "      --ecl-imports         Comma-delimited import list to be attached to output ECL\n"
                 "                            each entry generates a corresponding import *.<entry>\n"
                 "      --ecl-header          Text included in target header (must be valid ECL) \n"
+                "      --utf8-               Don't use UTF8 strings"
                 "   " ESDLOPT_INCLUDE_PATH_USAGE
                 ,stdout);
     }
@@ -463,6 +466,7 @@ public:
         params->setProp("sourceFileName", finger);
         params->setProp("importsList", eclimports);
         params->setProp("eclHeader", eclheader);
+        params->setProp("utf8strings", optUseUtf8Strings ? "yes" : "no");
         StringBuffer esdl2eclxslt (optHPCCCompFilesDir.get());
         esdl2eclxslt.append("/xslt/esdl2ecl.xslt");
         esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str());
@@ -507,6 +511,7 @@ public:
     bool optRollUpEclToSingleFile;
     bool optProcessIncludes;
     bool optOutputExpandedXML;
+    bool optUseUtf8Strings = true;
     StringAttr optHPCCCompFilesDir;
     StringAttr optECLIncludesList;
     StringAttr optECLHeaderBlock;

tools/esdlcmd/esdlcmd_common.hpp

@@ -77,6 +77,7 @@ typedef IEsdlCommand *(*EsdlCommandFactory)(const char *cmdname);
 #define ESDLOPT_NO_COLLAPSE             "--show-inheritance"
 #define ESDLOPT_NO_ARRAYOF              "--no-arrayof"
 #define ESDLOPT_OUTPUT_CATEGORIES       "--output-categories"
+#define ESDLOPT_USE_UTF8_STRINGS        "--utf8"
 
 #define ESDLOPT_WSDL_ADDRESS            "--wsdl-address"