HPCC-9779 Add an internal option to generate the ecl for a subgraph

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/deftype/defvalue.cpp

@@ -566,7 +566,7 @@ IValue *createStringValue(const char *val, ITypeInfo *type)
     return new StringValue(val, type);
 }
 
-IValue *createStringValue(const char *val, ITypeInfo *type, int srcLength, ICharsetInfo *srcCharset)
+IValue *createStringValue(const char *val, ITypeInfo *type, size32_t srcLength, ICharsetInfo *srcCharset)
 {
     ITranslationInfo * translation = queryDefaultTranslation(type->queryCharset(), srcCharset);
     size32_t tgtLength = type->getSize();
@@ -624,8 +624,10 @@ const char *UnicodeValue::generateECL(StringBuffer &out)
 {
     char * buff;
     unsigned bufflen;
-    rtlUnicodeToQuotedUTF8X(bufflen, buff, type->getStringLen(), (UChar const *)val.get());
-    out.append("U'").append(bufflen, buff).append('\'');
+    rtlUnicodeToUtf8X(bufflen, buff, type->getStringLen(), (UChar const *)val.get());
+    out.append("U'");
+    appendUtf8AsECL(out, rtlUtf8Size(bufflen, buff), buff);
+    out.append('\'');
     rtlFree(buff);
     return out.str();
 }
@@ -831,8 +833,10 @@ const char * VarUnicodeValue::generateECL(StringBuffer & out)
 {
     char * buff;
     unsigned bufflen;
-    rtlUnicodeToQuotedUTF8X(bufflen, buff, val.length(), val.get());
-    out.append("U'").append(bufflen, buff).append('\'');
+    rtlUnicodeToUtf8X(bufflen, buff, val.length(), val.get());
+    out.append("U'");
+    appendUtf8AsECL(out, rtlUtf8Size(bufflen, buff), buff);
+    out.append('\'');
     rtlFree(buff);
     return out.str();
 }

common/deftype/defvalue.hpp

@@ -73,7 +73,7 @@ extern DEFTYPE_API IValue * createValueFromMem(ITypeInfo * type, const void * pt
 
 extern DEFTYPE_API IValue * createStringValue(const char * value, unsigned size);
 extern DEFTYPE_API IValue * createStringValue(const char * value, ITypeInfo *type);
-extern DEFTYPE_API IValue * createStringValue(const char *val, ITypeInfo *type, int srcLength, ICharsetInfo *srcCharset);
+extern DEFTYPE_API IValue * createStringValue(const char *val, ITypeInfo *type, size32_t srcLength, ICharsetInfo *srcCharset);
 extern DEFTYPE_API IValue * createUnicodeValue(char const * value, unsigned size, char const * locale, bool utf8, bool unescape = false); // size is length of ascii or utf8 string; locale is like fr_BE_EURO
 extern DEFTYPE_API IValue * createUnicodeValue(char const * value, ITypeInfo *type); // only for ascii string, use above for utf8
 extern DEFTYPE_API IValue * createUnicodeValue(char const * value, ITypeInfo *type, unsigned srclen); // only for ascii string

ecl/hql/hqlgram.y

@@ -1760,7 +1760,7 @@ transform
                         {
                             $$.setExpr(parser->bindParameters($1, $4.getExpr()), $1);
                         }
-    | startInlineTransform transformOptions ',' transformations ')'
+    | startInlineTransform transformOptions semiComma transformations ')'
                         {
                             $$.setExpr(parser->closeTransform($4), $1);
                             parser->leaveCompoundObject();
@@ -1818,7 +1818,7 @@ startTransform
 
 transformOptions
     :
-    | transformOptions ',' transformOption
+    | transformOptions semiComma transformOption
                         {
                             parser->appendTransformOption($3.getExpr());
                             $$.setPosition($1);

ecl/hql/hqlgram2.cpp

@@ -2476,6 +2476,9 @@ IHqlExpression * HqlGram::endIfBlock()
 void HqlGram::checkFieldnameValid(const attribute &errpos, IIdAtom * name)
 {
     OwnedHqlExpr self = getSelfScope();
+    if (!self)
+        throwUnexpected();
+
     IHqlSimpleScope *recordScope = self->querySimpleScope();
     OwnedHqlExpr t(recordScope->lookupSymbol(name));
     if (t.get())

ecl/hql/hqlthql.cpp

@@ -50,19 +50,6 @@
 
 #define MAX_GRAPHTEXT_LEN   80      // Truncate anything more than 80 characters
 
-inline bool isInternalAttribute(IHqlExpression * e)
-{
-    if (e->isAttribute())
-    {
-        IAtom * name= e->queryName();
-        if ((name == sequenceAtom) || isInternalAttributeName(name))
-            return true;
-        if ((name == updateAtom) && e->hasAttribute(alwaysAtom))
-            return true;
-    }
-    return false;
-}
-
 bool endsWithDotDotDot(const StringBuffer & s)
 {
     if (s.length() < 3)
@@ -89,6 +76,7 @@ public:
     void setLowerCaseIds(bool value)        { lowerCaseIds = value; }
     void setMinimalSelectors(bool value)    { minimalSelectors = value; }
     void setMaxRecurseDepth(int depth)      { maxDatasetDepth = depth; }
+    void setTryToRegenerate(bool value)         { tryToRegenerate = value; }
 
 private:
     void childrenToECL(IHqlExpression *expr, StringBuffer &s, bool inType, bool needComma, unsigned first);
@@ -133,7 +121,41 @@ private:
 
     StringBuffer & appendId(StringBuffer & s, IIdAtom * name);
     StringBuffer & queryNewline(StringBuffer &s);
-    
+
+    bool isExported(IHqlExpression * expr)
+    {
+        return ::isExported(expr) && !tryToRegenerate;
+    }
+
+    bool isShared(IHqlExpression * expr)
+    {
+        return ::isShared(expr) && !tryToRegenerate;
+    }
+
+    bool isPublicSymbol(IHqlExpression * expr)
+    {
+        return ::isPublicSymbol(expr) && !tryToRegenerate;
+    }
+
+    bool isInternalAttribute(IHqlExpression * e)
+    {
+        if (e->isAttribute())
+        {
+            IAtom * name= e->queryName();
+            if ((name == sequenceAtom) || isInternalAttributeName(name))
+                return true;
+            if ((name == updateAtom) && e->hasAttribute(alwaysAtom))
+                return true;
+            if (tryToRegenerate)
+            {
+                if (name == jobTempAtom)
+                    return true;
+            }
+        }
+        return false;
+    }
+
+private:
     bool          m_recurse;
     bool          ignoreModuleNames;
     bool          insideNewTransform;
@@ -151,6 +173,7 @@ private:
     bool          ignoreVirtualAttrs;
     bool          lowerCaseIds;
     bool          minimalSelectors;
+    bool          tryToRegenerate;
     StringBufferArray m_exports;
     StringBufferArray m_service_names;
     StringBufferArray m_export_names;
@@ -178,6 +201,7 @@ HqltHql::HqltHql(bool recurse, bool _xgmmlGraphText) :
     ignoreVirtualAttrs = false;
     minimalSelectors = false;
     lowerCaseIds = false;
+    tryToRegenerate = false;
     clashCounter = 0;
 }
 
@@ -1984,14 +2008,35 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             break;
         case no_typetransfer:
         {
-            s.append(getEclOpString(no));
-            s.append('(');
-            toECL(child0, s, child0->getPrecedence() < 0, inType);
-            s.append(", ");
-            getTypeString(expr->queryType(), s);
-            s.append(')');
+            if (expr->isDatarow())
+            {
+                s.append(getEclOpString(no));
+                s.append('(');
+                toECL(child1, s, child0->getPrecedence() < 0, inType);
+                s.append(", ");
+                toECL(child0, s, child0->getPrecedence() < 0, inType);
+                s.append(')');
+            }
+            else
+            {
+                s.append(getEclOpString(no));
+                s.append('(');
+                toECL(child0, s, child0->getPrecedence() < 0, inType);
+                s.append(", ");
+                getTypeString(expr->queryType(), s);
+                s.append(')');
+            }
             break;
         }
+        case no_embedbody:
+            {
+                s.append("BEGINC++\n");
+                IValue * value = child0->queryValue();
+                if (value)
+                    value->getUTF8Value(s);
+                s.append("ENDC++");
+                break;
+            }
         case no_nofold:
         case no_nohoist:
         case no_selectfields:
@@ -2065,14 +2110,20 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                 s.append("TRANSFORM(");
                 getTypeString(expr->queryType(), s);
                 s.append(",");
+                if (tryToRegenerate)
+                    s.newline();
                 unsigned kids = expr->numChildren();
                 for (unsigned idx = 0; idx < kids; idx++)
                 {
                     if (queryAddDotDotDot(s, startLength))
                         break;
                     IHqlExpression *child = expr->queryChild(idx);
+                    if (tryToRegenerate)
+                        s.append("\t");
                     toECL(child, s, child->getPrecedence() < 0, inType);
                     s.append(';');
+                    if (tryToRegenerate)
+                        s.newline();
                 }
                 s.append(")");
             }
@@ -2189,6 +2240,8 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                 toECL(child0, s, false, inType);
                 curDatasetDepth++;
             }
+            else if (tryToRegenerate)
+                defaultToECL(child0, s, inType);
             else
                 defaultToECL(expr, s, inType);
             break;
@@ -2499,14 +2552,21 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             break;
         case no_subgraph:
             {
-                s.append(getEclOpString(expr->getOperator())).append("(");
-                ForEachChild(i, expr)
+                if (tryToRegenerate)
                 {
-                    s.newline().append("\t");
-                    toECL(expr->queryChild(i), s, false, inType);
+                    childrenToECL(expr, s, false, false, 0);
+                }
+                else
+                {
+                    s.append(getEclOpString(expr->getOperator())).append("(");
+                    ForEachChild(i, expr)
+                    {
+                        s.newline().append("\t");
+                        toECL(expr->queryChild(i), s, false, inType);
+                    }
+                    s.append(")");
+                    break;
                 }
-                s.append(")");
-                break;
             }
         case no_sequence:
             if (expr->queryName())
@@ -2594,6 +2654,8 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                 toECL(child0, s, false, inType);
                 curDatasetDepth++;
             }
+            else if (tryToRegenerate)
+                toECL(child0, s, false, inType);
             else
                 defaultToECL(expr, s, inType);
             break;
@@ -2607,6 +2669,26 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             else
                 defaultToECL(expr, s, inType);
             break;
+        case no_compound_indexread:
+        case no_compound_disknormalize:
+        case no_compound_diskaggregate:
+        case no_compound_diskcount:
+        case no_compound_diskgroupaggregate:
+        case no_compound_indexnormalize:
+        case no_compound_indexaggregate:
+        case no_compound_indexcount:
+        case no_compound_indexgroupaggregate:
+        case no_compound_childread:
+        case no_compound_childnormalize:
+        case no_compound_childaggregate:
+        case no_compound_childcount:
+        case no_compound_childgroupaggregate:
+        case no_compound_inline:
+            if (tryToRegenerate)
+                toECL(child0, s, false, inType);
+            else
+                defaultToECL(expr, s, inType);
+            break;
         //case no_table:
         //case no_count:
         //case no_if:
@@ -2846,7 +2928,7 @@ const char * HqltHql::getEclOpString(node_operator op)
     case no_ne:
         return "!=";
     case no_not:
-        return "~";
+        return "NOT ";
     case no_or:
         return "OR";
     case no_and:
@@ -3127,7 +3209,7 @@ void HqltHql::defineCallTarget(IHqlExpression * call, StringBuffer & name)
                 newdef.append("SHARED ");
             }
         }
-        else
+        else if (!tryToRegenerate)
             newdef.append("EXPORT ");           //not really right - the information is lost about whether the service definition is exported or not.
 
         newdef.append(name).append(" := ").newline();
@@ -3223,6 +3305,18 @@ StringBuffer &toECL(IHqlExpression * expr, StringBuffer &s, bool recurse, bool x
 }
 
 
+StringBuffer &regenerateECL(IHqlExpression * expr, StringBuffer &s)
+{
+    HqltHql hqlthql(true, false);
+    hqlthql.setIgnoreModuleNames(true);
+    hqlthql.setTryToRegenerate(true);
+
+    HqlExprArray queries;
+    unwindCommaCompound(queries, expr);
+    return toECL(s, hqlthql, queries, true);
+}
+
+
 StringBuffer &toUserECL(StringBuffer &s, IHqlExpression * expr, bool recurse)
 {
     HqltHql hqlthql(recurse, false);

ecl/hql/hqlthql.hpp

@@ -21,6 +21,7 @@
 #include "hql.hpp"
 #include "hqlexpr.hpp"
 
+extern HQL_API StringBuffer &regenerateECL(IHqlExpression * expr, StringBuffer &s);
 extern HQL_API StringBuffer &toECL(IHqlExpression * expr, StringBuffer &s, bool recurse=false, bool xgmmlGraphText = false);
 extern HQL_API StringBuffer &toECLSimple(IHqlExpression * expr, StringBuffer &s);
 extern HQL_API void splitECL(IHqlExpression * expr, StringBuffer &s, StringBuffer &d);

ecl/hqlcpp/hqlcpp.cpp

@@ -1512,6 +1512,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.optimizeBoolReturn,"optimizeBoolReturn", true),
         DebugOption(options.freezePersists,"freezePersists", false),
         DebugOption(options.maxRecordSize, "defaultMaxLengthRecord", MAX_RECORD_SIZE),
+        DebugOption(options.subgraphToRegeneate, "subgraphToRegeneate", 0),
 
         DebugOption(options.checkRoxieRestrictions,"checkRoxieRestrictions", true),     // a debug aid for running regression suite
         DebugOption(options.checkThorRestrictions,"checkThorRestrictions", true),       // a debug aid for running regression suite

ecl/hqlcpp/hqlcpp.ipp

@@ -556,6 +556,7 @@ struct HqlCppOptions
     unsigned            applyInstantEclTransformationsLimit;
     unsigned            complexClassesThreshold;
     unsigned            complexClassesActivityFilter;
+    unsigned            subgraphToRegeneate;
     CompilerType        targetCompiler;
     DBZaction           divideByZeroAction;
     bool                peephole;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -8954,6 +8954,15 @@ unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpressi
 
     OwnedHqlExpr idExpr = createConstant((__int64)thisId);
     ctx.associateExpr(expr, idExpr);
+    if (thisId == options.subgraphToRegeneate)
+    {
+        StringBuffer ecl;
+        regenerateECL(expr, ecl);
+        ecl.replaceString("\r","");
+        fputs(ecl.str(), stdout);
+
+        fflush(stdout);
+    }
 
     BuildCtx subctx(ctx);
     ForEachChild(idx, expr)

rtl/eclrtl/eclrtl.cpp

@@ -3186,16 +3186,6 @@ void rtlUnicodeToEscapedStrX(unsigned & outlen, char * & out, unsigned inlen, UC
     memcpy(out, outbuff.str(), outlen);
 }
 
-void rtlUnicodeToQuotedUTF8X(unsigned & outlen, char * & out, unsigned inlen, UChar const * in)
-{
-    UnicodeString unicode(in, inlen);
-    unicode.findAndReplace("'", "\\'");
-    //pre-flight length - may be more efficient to guess length and only re-extract if guess no good, but what to guess?
-    outlen = unicode.extract(0, unicode.length(), 0, 0, UTF8_CODEPAGE);
-    out = (char *)rtlMalloc(outlen);
-    unicode.extract(0, unicode.length(), out, outlen, UTF8_CODEPAGE);
-}
-
 bool rtlCodepageToCodepage(unsigned outlen, char * out, unsigned inlen, char const * in, char const * outcodepage, char const * incodepage)
 {
     UConverter * inconv = queryRTLUnicodeConverter(incodepage)->query();

rtl/eclrtl/eclrtl.hpp

@@ -336,7 +336,6 @@ ECLRTL_API void rtlUnicodeToStr(unsigned outlen, char * out, unsigned inlen, UCh
 ECLRTL_API void rtlStrToUnicodeX(unsigned & outlen, UChar * & out, unsigned inlen, char const * in);
 ECLRTL_API void rtlUnicodeToStrX(unsigned & outlen, char * & out, unsigned inlen, UChar const * in);
 ECLRTL_API void rtlUnicodeToEscapedStrX(unsigned & outlen, char * & out, unsigned inlen, UChar const * in);
-ECLRTL_API void rtlUnicodeToQuotedUTF8X(unsigned & outlen, char * & out, unsigned inlen, UChar const * in);
 ECLRTL_API bool rtlCodepageToCodepage(unsigned outlen, char * out, unsigned inlen, char const * in, char const * outcodepage, char const * incodepage); //returns success, false probably means illegal input or overflow
 ECLRTL_API int rtlSingleUtf8ToCodepage(char * out, unsigned inlen, char const * in, char const * outcodepage); //returns number of trailbytes on character in UTF8 if valid, -1 error (illegal input or overflow (this routine assumes output is single character, which can break if outcodepage uses multibyte sequences, so don't do that))
 

system/jlib/jstring.cpp

@@ -1256,7 +1256,7 @@ void appendURL(StringBuffer *dest, const char *src, size32_t len, char lower)
 }
 
 
-static StringBuffer & appendStringExpandControl(StringBuffer &out, unsigned len, const char * src, bool addBreak, bool isCpp)
+static StringBuffer & appendStringExpandControl(StringBuffer &out, unsigned len, const char * src, bool addBreak, bool isCpp, bool isUtf8)
 {
     const int minBreakPos = 0;
     const int commaBreakPos = 70;
@@ -1313,7 +1313,7 @@ static StringBuffer & appendStringExpandControl(StringBuffer &out, unsigned len,
                     out.append(c);
                 break;
             default:
-                if ((c >= ' ') && (c <= 126))
+                if (isUtf8 || (c >= ' ') && (c <= 126))
                     out.append(c);
                 else
                     out.appendf("\\%03o", c); 
@@ -1331,13 +1331,19 @@ static StringBuffer & appendStringExpandControl(StringBuffer &out, unsigned len,
 
 StringBuffer & appendStringAsCPP(StringBuffer &out, unsigned len, const char * src, bool addBreak)
 {
-    return appendStringExpandControl(out, len, src, addBreak, true);
+    return appendStringExpandControl(out, len, src, addBreak, true, false);
 }
 
 
 StringBuffer & appendStringAsECL(StringBuffer &out, unsigned len, const char * src)
 {
-    return appendStringExpandControl(out, len, src, false, false);
+    return appendStringExpandControl(out, len, src, false, false, false);
+}
+
+
+StringBuffer & appendUtf8AsECL(StringBuffer &out, unsigned len, const char * src)
+{
+    return appendStringExpandControl(out, len, src, false, false, true);
 }
 
 

system/jlib/jstring.hpp

@@ -359,6 +359,7 @@ extern jlib_decl StringBuffer & appendDataAsHex(StringBuffer &out, unsigned len,
 extern jlib_decl StringBuffer & appendStringAsSQL(StringBuffer & out, unsigned len, const char * src);
 extern jlib_decl StringBuffer & appendStringAsECL(StringBuffer & out, unsigned len, const char * src);
 extern jlib_decl StringBuffer & appendStringAsQuotedECL(StringBuffer &out, unsigned len, const char * src);
+extern jlib_decl StringBuffer & appendUtf8AsECL(StringBuffer &out, unsigned len, const char * src);
 
 extern jlib_decl const char *decodeJSON(const char *x, StringBuffer &ret, unsigned len=(unsigned)-1, const char **errMark=NULL);
 extern jlib_decl void extractItem(StringBuffer & res, const char * src, const char * sep, int whichItem, bool caps);