Merge remote-tracking branch 'origin/closedown-4.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/remote/rmtspawn.cpp

@@ -467,9 +467,12 @@ void CRemoteSlave::run(int argc, char * argv[])
                     msg.clear();
                     if (catchReadBuffer(masterSocket, msg, RMTTIME_RESPONSE_MASTER))
                     {
+                        LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag);
                         msg.read(ok);
                         assertex(ok);
                     }
+                    else
+                        LOG(MCdebugProgress, unknownJob, "No terminate acknowledgement received from master for slave %d", info.replyTag);
 
                     if (error)
                         break;

common/remote/rmtssh.cpp

@@ -531,6 +531,13 @@ public:
             while (res.length()&&(res.charAt(res.length()-1)<=' '))
                 res.setLength(res.length()-1);
             PROGLOG("%s result(%d):\n%s",useplink?"plink":"ssh",reply.item(0),res.str());
+            if (res.length())
+            {
+                int code = reply.item(0);
+                if (code == 0)
+                    code = -1;
+                throw MakeStringExceptionDirect(code, res.str());
+            }
         }
     }
     void exec(

dali/dfu/dfurun.cpp

@@ -166,7 +166,7 @@ class CDFUengine: public CInterface, implements IDFUengine
                 PROGLOG("ABORT notified");
             abort = true;
         }
-    } abortnotify;
+    };
 
 
     class cDFUlistener: public Thread

dali/ft/filecopy.cpp

@@ -2529,8 +2529,14 @@ void FileSprayer::waitForTransferSem(Semaphore & sem)
             StringBuffer list;
             ForEachItemIn(i, transferSlaves)
                 transferSlaves.item(i).logIfRunning(list);
+
             if (timeSinceProgress>RESPONSE_TIME_TIMEOUT)
-                throwError1(RFSERR_TimeoutWaitSlave, list.str());
+            {
+                //Set an error - the transfer threads will check it after a couple of minutes, and then terminate gracefully
+                CriticalBlock lock(errorCS);
+                if (!error)
+                    error.setown(MakeStringException(RFSERR_TimeoutWaitSlave, RFSERR_TimeoutWaitSlave_Text, list.str()));
+            }
         }
     }
 }
@@ -2545,7 +2551,10 @@ void FileSprayer::addTarget(unsigned idx, INode * node)
 }
 
 bool FileSprayer::isAborting()
-{ 
+{
+    if (aborting || error)
+        return true;
+
     unsigned nowTick = msTick();
     if (abortChecker && (nowTick - lastAbortCheckTick >= abortCheckFrequency))
     {

ecl/hql/hqlexpr.cpp

@@ -3470,6 +3470,15 @@ void CHqlExpression::updateFlagsAfterOperands()
     case no_failure:
         infoFlags &= ~HEFonFailDependent;
         break;
+    case no_externalcall:
+        if (constant())
+        {
+            IHqlExpression * body = queryExternalDefinition()->queryChild(0);
+            assertex(body);
+            if (!body->hasProperty(pureAtom))
+                infoFlags2 &= ~HEF2constant;
+        }
+        break;
     case no_call:
         {
             IHqlExpression * funcdef = queryBody()->queryFunctionDefinition();
@@ -10302,18 +10311,88 @@ protected:
         case no_if:
             {
                 IHqlExpression * cond  = expr->queryChild(0);
-                //Only fold things that become constant because of a parameter substitution, otherwise we get crc mismatches
-                if (!cond->isFullyBound())
+                OwnedHqlExpr newcond = transform(cond);
+                if (newcond->isConstant())
+                    newcond.setown(foldHqlExpression(newcond));
+                IValue * value = newcond->queryValue();
+                if (value && !expr->isAction())
+                {
+                    unsigned branch = value->getBoolValue() ? 1 : 2;
+                    IHqlExpression * arg = expr->queryChild(branch);
+                    if (arg)
+                        return transform(arg);
+                }
+                break;
+            }
+        case no_map:
+            {
+                //This could strip leading failing matches, but for the moment only do that if we know
+                //which branch matches.
+                ForEachChild(i, expr)
+                {
+                    IHqlExpression * cur = expr->queryChild(i);
+                    if (cur->getOperator() == no_mapto)
+                    {
+                        OwnedHqlExpr newcond = transform(cur->queryChild(0));
+
+                        if (newcond->isConstant())
+                            newcond.setown(foldHqlExpression(newcond));
+                        IValue * value = newcond->queryValue();
+                        if (!value)
+                            break;
+
+                        if (value->getBoolValue())
+                            return transform(cur->queryChild(1));
+                    }
+                    else if (!cur->isAttribute())
+                        return transform(cur);
+                }
+                break;
+            }
+        case no_case:
+            {
+                IHqlExpression * search = expr->queryChild(0);
+                if (!search->isConstant())
+                    break;
+                OwnedHqlExpr folded = foldHqlExpression(search);
+                IValue * searchValue = folded->queryValue();
+                if (!searchValue)
+                    break;
+                ITypeInfo * searchType = searchValue->queryType();
+                ForEachChildFrom(i, expr, 1)
                 {
-                    OwnedHqlExpr newcond = transform(cond);
-                    IValue * value = newcond->queryValue();
-                    if (value && !expr->isAction())
+                    IHqlExpression * cur = expr->queryChild(i);
+                    if (cur->getOperator() == no_mapto)
                     {
-                        unsigned branch = value->getBoolValue() ? 1 : 2;
-                        IHqlExpression * arg = expr->queryChild(branch);
-                        if (arg)
-                            return transform(arg);
+                        OwnedHqlExpr newcond = transform(cur->queryChild(0));
+
+                        if (newcond->isConstant())
+                            newcond.setown(foldHqlExpression(newcond));
+                        IValue * compareValue = newcond->queryValue();
+                        if (!compareValue)
+                            break;
+
+                        if (searchType != newcond->queryType())
+                        {
+                            Owned<ITypeInfo> type = ::getPromotedECLCompareType(searchType, newcond->queryType());
+                            OwnedHqlExpr castSearch = ensureExprType(folded, type);
+                            OwnedHqlExpr castCompare = ensureExprType(newcond, type);
+                            IValue * castSearchValue = castSearch->queryValue();
+                            IValue * castCompareValue = castCompare->queryValue();
+                            if (!castSearchValue || !castCompareValue)
+                                break;
+
+                            if (castSearchValue->compare(castCompareValue) == 0)
+                                return transform(cur->queryChild(1));
+                        }
+                        else
+                        {
+                            if (searchValue->compare(compareValue) == 0)
+                                return transform(cur->queryChild(1));
+                        }
                     }
+                    else if (!cur->isAttribute())
+                        return transform(cur);
                 }
                 break;
             }

ecl/hql/hqlfold.cpp

@@ -3321,6 +3321,17 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
                 return createActionList(args);
             break;
         }
+    case no_exists:
+        if (isNull(expr->queryChild(0)))
+            return createConstant(false);
+        break;
+    case no_alias:
+        {
+            IHqlExpression * arg = expr->queryChild(0);
+            if (arg->getOperator() == no_constant)
+                return LINK(arg);
+            break;
+        }
     }
 
     return LINK(expr);
@@ -4834,6 +4845,22 @@ IHqlExpression * createListMatchStructure(node_operator op, IHqlExpression * exp
     OwnedHqlExpr newRhs = createListMatchStructure(op, rhs, args, idx);
     if ((lhs == newLhs) && (rhs == newRhs))
         return LINK(expr);
+
+    if (op == no_and)
+    {
+        if (matchesBoolean(newLhs, true))
+            return newRhs.getClear();
+        if (matchesBoolean(newRhs, true))
+            return newLhs.getClear();
+    }
+    else
+    {
+        if (matchesBoolean(newLhs, false))
+            return newRhs.getClear();
+        if (matchesBoolean(newRhs, false))
+            return newLhs.getClear();
+    }
+
     OwnedHqlExpr value = createValue(op, expr->getType(), newLhs.getClear(), newRhs.getClear());
     return expr->cloneAllAnnotations(value);
 }

ecl/hqlcpp/hqlttcpp.cpp

@@ -9717,7 +9717,7 @@ void AnnotationNormalizerTransformer::analyseExpr(IHqlExpression * expr)
     IHqlExpression * body = expr->queryBody();
     if (expr != body)
     {
-        queryCommonExtra(body)->noteAnnotation(expr);
+        queryLocationIndependentExtra(body)->noteAnnotation(expr);
         //Note: expr already tested if expr == body...
         if (alreadyVisited(body))
             return;
@@ -9751,7 +9751,7 @@ IHqlExpression * AnnotationNormalizerTransformer::createTransformed(IHqlExpressi
     case no_constant:
 //  case no_null:
         {
-            //AnnotationTransformInfo * extra = queryCommonExtra(body);
+            //AnnotationTransformInfo * extra = queryLocationIndependentExtra(body);
 
             //Don't common up the location information for this, otherwise it gets silly!  Possibly worth removing altogether if ambiguous?
             //MORE: This should probably depend on whether there is more than one annotation on the constant.
@@ -9763,10 +9763,10 @@ IHqlExpression * AnnotationNormalizerTransformer::createTransformed(IHqlExpressi
         return transform(body);
 
     OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
-    return queryCommonExtra(body)->cloneAnnotations(transformed);
+    return queryLocationIndependentExtra(body)->cloneAnnotations(transformed);
 }
 
-AnnotationTransformInfo * AnnotationNormalizerTransformer::queryCommonExtra(IHqlExpression * expr)
+AnnotationTransformInfo * AnnotationNormalizerTransformer::queryLocationIndependentExtra(IHqlExpression * expr)
 {
     return static_cast<AnnotationTransformInfo *>(queryTransformExtra(queryLocationIndependent(expr)));
 }
@@ -10160,7 +10160,7 @@ ANewTransformInfo * HqlTreeNormalizer::createTransformInfo(IHqlExpression * expr
     return CREATE_NEWTRANSFORMINFO(HqlTreeNormalizerInfo, expr);
 }
 
-HqlTreeNormalizerInfo * HqlTreeNormalizer::queryCommonExtra(IHqlExpression * expr)
+HqlTreeNormalizerInfo * HqlTreeNormalizer::queryLocationIndependentExtra(IHqlExpression * expr)
 {
     return static_cast<HqlTreeNormalizerInfo *>(queryTransformExtra(queryLocationIndependent(expr)));
 }
@@ -10366,7 +10366,7 @@ void HqlTreeNormalizer::analyseExpr(IHqlExpression * expr)
     {
         IHqlExpression * symbol = queryNamedSymbol(expr);
         if (symbol)
-            queryCommonExtra(body)->noteSymbol(expr);
+            queryLocationIndependentExtra(body)->noteSymbol(expr);
     }
 
     if (alreadyVisited(body))
@@ -10487,8 +10487,12 @@ IHqlExpression * HqlTreeNormalizer::transformCaseToIfs(IHqlExpression * expr)
         OwnedHqlExpr castCurValue = ensureExprType(curValue, type);
 
         OwnedHqlExpr test = createBoolExpr(no_eq, ensureExprType(testVar, type), transform(castCurValue));
+        if (options.constantFoldNormalize)
+            test.setown(foldConstantOperator(test, 0, NULL));
         OwnedHqlExpr trueExpr = transform(cur->queryChild(1));
         elseExpr.setown(createIf(test.getClear(), trueExpr.getClear(), elseExpr.getClear()));
+        if (options.constantFoldNormalize)
+            elseExpr.setown(foldConstantOperator(elseExpr, 0, NULL));
     }
     return elseExpr.getClear();
 }
@@ -10614,6 +10618,8 @@ IHqlExpression * HqlTreeNormalizer::transformMap(IHqlExpression * expr)
     {
         IHqlExpression * cur = expr->queryChild(idx);
         elseExpr.setown(createIf(transform(cur->queryChild(0)), transform(cur->queryChild(1)), elseExpr.getClear()));
+        if (options.constantFoldNormalize)
+            elseExpr.setown(foldConstantOperator(elseExpr, 0, NULL));
     }
     return elseExpr.getClear();
 }
@@ -11169,9 +11175,6 @@ IHqlExpression * HqlTreeNormalizer::createTransformed(IHqlExpression * expr)
     node_operator op = expr->getOperator();
     if (expr != body)
     {
-#if 0
-        OwnedHqlExpr transformedBody = transform(body);
-#else
         OwnedHqlExpr transformedBody;
         try
         {
@@ -11190,7 +11193,11 @@ IHqlExpression * HqlTreeNormalizer::createTransformed(IHqlExpression * expr)
             }
             throw;
         }
-#endif
+
+        //Don't retain any annotations on records - except for a symbol which maybe added in the createTransform()
+        //code.  Otherwise expressions that would otherwise be commoned up are treated as different.
+        if (op == no_record)
+            return transformedBody.getClear();
 
         switch (expr->getAnnotationKind())
         {
@@ -11426,7 +11433,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             if (options.ensureRecordsHaveSymbols)
             {
                 //Ensure all records only have a single unique name, and transform it here so that record types also map to that unique name
-                IHqlExpression * recordSymbol = queryCommonExtra(expr)->symbol;
+                IHqlExpression * recordSymbol = queryLocationIndependentExtra(expr)->symbol;
                 if (recordSymbol)
                 {
                     _ATOM name = recordSymbol->queryName();

ecl/hqlcpp/hqlttcpp.ipp

@@ -1076,7 +1076,7 @@ public:
 //  virtual IHqlExpression * transformSelector(IHqlExpression * expr);
 
 protected:
-    AnnotationTransformInfo * queryCommonExtra(IHqlExpression * expr);
+    AnnotationTransformInfo * queryLocationIndependentExtra(IHqlExpression * expr);
 };
 
 void normalizeAnnotations(HqlCppTranslator & translator, HqlExprArray & exprs);
@@ -1169,7 +1169,7 @@ protected:
 protected:
     IHqlExpression * transformChildrenNoAnnotations(IHqlExpression * expr);
     IHqlExpression * makeRecursiveName(_ATOM searchModule, _ATOM searchName);
-    HqlTreeNormalizerInfo * queryCommonExtra(IHqlExpression * expr);
+    HqlTreeNormalizerInfo * queryLocationIndependentExtra(IHqlExpression * expr);
     IHqlExpression * transformSimpleConst(IHqlExpression * expr)
     {
         if (expr->isConstant())

ecl/regress/in2.ecl

@@ -16,11 +16,11 @@
 ############################################################################## */
 
 #option ('globalFold', false);
-string1 getcity1(string1 city_char) := if(city_char in
+string1 getcity1(string1 city_char) := if(NOFOLD(city_char) in
 ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R'
 ,'S','T','U','V','W','X','Y','Z'], city_char, ' ');
 
-string1 getcity2(string1 city_char) := if((string) city_char in
+string1 getcity2(string1 city_char) := if(NOFOLD((string) city_char) in
 ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R'
 ,'S','T','U','V','W','X','Y','Z'], city_char, ' ');
 

ecl/regress/setsql.ecl

@@ -19,7 +19,7 @@
 
 person := dataset('person', { unsigned8 person_id, string1 per_sex, string2 per_st, string40 per_first_name, string40 per_last_name}, thor);
 
-inlist(string x, set of string y) := if ( x in y, 'yes','no ');
+inlist(string x, set of string y) := if ( NOFOLD(x) in y, 'yes','no ');
 
 'one' in ['Gavin','Jason','Emma','Vicky'];
 inlist('one', ['Gavin','Jason','Emma','Vicky']);

ecl/regress/setsql2.ecl

@@ -19,7 +19,7 @@
 
 person := dataset('person', { unsigned8 person_id, string1 per_sex, string2 per_st, string40 per_first_name, string40 per_last_name}, thor);
 
-inlist(string x, set of string y) := if ( x in y, 'yes','no');
+inlist(string x, set of string y) := if ( NOFOLD(x) in y, 'yes','no');
 inlist2(string x, set of string20 y) := inlist(x, y);
 
 'one' in ['Gavin','Jason','Emma','Vicky'];

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -535,26 +535,21 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
             return; // recursive
 
         int startlen = out.length();
-        out.appendf("<%s", tag);
+        appendXMLOpenTag(out, tag, NULL, false);
         if (ns)
             out.append(' ').append(ns);
         int taglen=out.length()+1;
         for (size_t i=0; i<type->getAttrCount(); i++)
         {
             IXmlAttribute* attr = type->queryAttr(i);
+            StringBuffer s;
+            const char *attrval;
             if (parmtree)
-            {
-                StringBuffer attrpath("@");
-                const char *attrval = parmtree->queryProp(attrpath.append(attr->queryName()).str());
-                if (attrval)
-                    out.appendf(" %s='", attr->queryName()).append(attrval);
-            }
+                attrval = parmtree->queryProp(s.append('@').append(attr->queryName()));
             else
-            {
-                out.appendf(" %s='", attr->queryName());
-                attr->getSampleValue(out);
-            }
-            out.append('\'');
+                attrval = attr->getSampleValue(s);
+            if (attrval)
+                appendXMLAttr(out, attr->queryName(), attrval);
         }
         out.append('>');
         if (typeName)
@@ -567,9 +562,9 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
             assertex(flds==0);
             if (parmtree)
             {
-                const char *attrval = parmtree->queryProp(NULL);
-                if (attrval)
-                    out.append(attrval);
+                const char *val = parmtree->queryProp(NULL);
+                if (val)
+                    encodeXML(val, out);
             }
             else if (flags & REQXML_SAMPLE_DATA)
                 type->queryFieldType(0)->getSampleValue(out,tag);
@@ -592,7 +587,7 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
         if ((flags & REQXML_TRIM) && !(flags & REQXML_ROOT) && out.length()==taglen)
             out.setLength(startlen);
         else
-            out.appendf("</%s>",tag);
+            appendXMLCloseTag(out, tag);
     }
     else if (type->isArray())
     {
@@ -608,7 +603,7 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
             parent.push_back(typeName);
 
         int startlen = out.length();
-        out.appendf("<%s", tag);
+        appendXMLOpenTag(out, tag);
         if (ns)
             out.append(' ').append(ns);
         out.append(">");
@@ -656,7 +651,7 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
         if ((flags & REQXML_TRIM) && !(flags & REQXML_ROOT) && out.length()==taglen)
             out.setLength(startlen);
         else
-            out.appendf("</%s>",tag);
+            appendXMLCloseTag(out, tag);
     }
     else // simple type
     {
@@ -672,18 +667,12 @@ static void buildReqXml(StringStack& parent, IXmlType* type, StringBuffer& out,
             {
                 if (!strieq(parmval, "default"))
                 {
-                    out.appendf("<%s>", tag);
-                    if (parmval.length())
-                        out.append((strieq(parmval.str(),"1")||strieq(parmval.str(),"true")||strieq(parmval.str(), "on")) ? '1' : '0');
-                    out.appendf("</%s>", tag);
+                    bool val = (strieq(parmval.str(),"1")||strieq(parmval.str(),"true")||strieq(parmval.str(), "on"));
+                    appendXMLTag(out, tag, val ? "1" : "0");
                 }
             }
             else
-            {
-                out.appendf("<%s>", tag);
-                out.append(parmval);
-                out.appendf("</%s>", tag);
-            }
+                appendXMLTag(out, tag, parmval);
         }
     }
 }