Merge branch 'candidate-7.4.x' into candidate-7.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -1229,7 +1229,7 @@ public:
                         translateBuf.setLength(0);
                         MemoryBufferBuilder rowBuilder(translateBuf, 0);
                         translator->translate(rowBuilder, *fieldCallback, row);
-                        row = reinterpret_cast<const byte *>(translateBuf.toByteArray());
+                        row = rowBuilder.getSelf();
                     }
                     return row;
                 }

ecl/hql/hqlexpr.cpp

@@ -4640,8 +4640,11 @@ switch (op)
         break;
     case no_funcdef:
         {
-            assertex(queryType()->getTypeCode() == type_function && queryChild(1) && queryChild(1)->getOperator() == no_sortlist);
+            IHqlExpression * formals = queryChild(1);
+            assertex(queryType()->getTypeCode() == type_function && queryChild(1) && formals->getOperator() == no_sortlist);
             assertex(queryType()->queryChildType() == queryChild(0)->queryType());
+            ForEachChild(i, formals)
+                assertex(formals->queryChild(i)->getOperator() == no_param);
             break;
         }
     case no_setresult:
@@ -9282,6 +9285,23 @@ void CHqlLibraryInstance::sethash()
 
 //==============================================================================================================
 
+//Replace expressions, but do not replace parameters
+static HqlTransformerInfo quickNoParamExpressionReplacerInfo("QuickNoParamExpressionReplacer");
+class HQL_API QuickNoParamExpressionReplacer : public QuickExpressionReplacer
+{
+public:
+    QuickNoParamExpressionReplacer() : QuickExpressionReplacer(quickNoParamExpressionReplacerInfo)
+    {
+    }
+    virtual IHqlExpression * createTransformedBody(IHqlExpression * expr)
+    {
+        if (expr->getOperator() == no_param)
+            return LINK(expr);
+        return QuickExpressionReplacer::createTransformedBody(expr);
+    }
+};
+
+
 //Each function instance needs to have a unique set of parameters
 static IHqlExpression * cloneFunction(IHqlExpression * expr)
 {
@@ -9292,7 +9312,7 @@ static IHqlExpression * cloneFunction(IHqlExpression * expr)
     if (formals->numChildren() == 0)
         return LINK(expr);
 
-    QuickExpressionReplacer replacer;
+    QuickNoParamExpressionReplacer replacer;
     ForEachChild(i, formals)
     {
         IHqlExpression * formal = formals->queryChild(i);
@@ -11925,7 +11945,8 @@ protected:
                 if (oldModule != newModule)
                 {
                     IIdAtom * selectedName = expr->queryChild(3)->queryId();
-                    newModule.setown(checkCreateConcreteModule(ctx.errors, newModule, newModule->queryAttribute(_location_Atom)));
+                    IHqlExpression * location = newModule->queryAttribute(_location_Atom);
+                    newModule.setown(checkCreateConcreteModule(ctx.errors, newModule, location));
 
                     HqlDummyLookupContext dummyctx(ctx.errors);
                     IHqlScope * newScope = newModule->queryScope();

ecl/hql/hqltrans.ipp

@@ -264,6 +264,7 @@ class HQL_API QuickExpressionReplacer : public QuickHqlTransformer
 {
 public:
     QuickExpressionReplacer();
+    QuickExpressionReplacer(HqlTransformerInfo & _info) : QuickHqlTransformer(_info, NULL) {}
 
     void setMapping(IHqlExpression * oldValue, IHqlExpression * newValue);
 };

ecl/hql/hqlvalid.cpp

@@ -153,12 +153,16 @@ void reportAbstractModule(IErrorReceiver & errors, IHqlExpression * expr, const
         }
     }
 
+    StringBuffer name;
+    if (expr->queryId())
+        name.append(" (").append(str(expr->queryId())).append(")");
+
     if (fieldText.length())
-        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE in this context (%s undefined)", fieldText.str());
+        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE%s in this context (%s undefined)", name.str(), fieldText.str());
     else if (expr->hasAttribute(interfaceAtom))
-        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE in this context (INTERFACE must be instantiated)");
+        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE%s in this context (INTERFACE must be instantiated)", name.str());
     else
-        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE in this context");
+        reportError(&errors, ERR_ABSTRACT_MODULE, errpos, "Cannot use an abstract MODULE%s in this context", name.str());
 }
 
 IHqlExpression * checkCreateConcreteModule(IErrorReceiver * errors, IHqlExpression * expr, const IHqlExpression * locationExpr)

ecl/hthor/hthor.cpp

@@ -8816,7 +8816,7 @@ const void *CHThorDiskReadActivity::nextRow()
                         {
                             MemoryBufferBuilder aBuilder(translated, 0);
                             translator->translate(aBuilder, *this, next);
-                            next = reinterpret_cast<const byte *>(translated.toByteArray());
+                            next = aBuilder.getSelf();
                         }
                         if (likely(helper.canMatch(next)))
                             thisSize = helper.transform(outBuilder.ensureRow(), next);

ecl/hthor/hthorkey.cpp

@@ -2492,7 +2492,7 @@ public:
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(fetch->pos);
             translator->translate(aBuilder, fieldCallback, rawBuffer);
-            rawBuffer = reinterpret_cast<const byte *>(buf.toByteArray());
+            rawBuffer = aBuilder.getSelf();
         }
 
         CriticalBlock procedure(transformCrit);
@@ -3685,7 +3685,7 @@ public:
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(pos);
             translator->translate(aBuilder, fieldCallback, row);
-            row = reinterpret_cast<const byte *>(buf.toByteArray());
+            row = aBuilder.getSelf();
         }
         if(match(fetch->ms, row))
         {

ecl/regress/issue23028.ecl

@@ -0,0 +1,19 @@
+#onwarning (1021, ignore); // Ignore warning that all output values are constant
+
+r1 := RECORD
+   string35 code;
+  END;
+
+r2 := RECORD
+  integer2 id;
+  embedded DATASET(r1) child;
+ END;
+
+mk1(unsigned c) := TRANSFORM(r1, SELF.code := (string)c; SELF := []);
+mk2(unsigned c) := TRANSFORM(r2, SELF.id := c; SELF.child := DATASET((c-1) % 5, mk1(COUNTER+c)), SELF := []);
+
+d := DATASET(20, mk2(counter));
+sequential(
+    output(d,,'~myspill', overwrite);
+    output(DATASET('~myspill', r2, THOR), { one := 1 })
+);

roxie/ccd/ccdactivities.cpp

@@ -3769,7 +3769,7 @@ public:
             translator->translate(aBuilder, fieldCallback, diskRow);
             //  note the swapped parameters - left and right map to input and raw differently for JOIN vs FETCH
             IHThorFetchArg *h = (IHThorFetchArg *) helper;
-            return h->transform(rowBuilder, buf.toByteArray(), inputData, rawpos);
+            return h->transform(rowBuilder, aBuilder.getSelf(), inputData, rawpos);
         }
         else
         {
@@ -4464,7 +4464,7 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(headerPtr->fpos);
             translator->translate(aBuilder, fieldCallback, rawRHS);
-            rawRHS = (const byte *) buf.toByteArray();
+            rawRHS = aBuilder.getSelf();
         }
 
         inputData = &headerPtr->rhsdata[0];

roxie/ccd/ccdkey.cpp

@@ -471,7 +471,7 @@ protected:
             buf.setLength(0);
             MemoryBufferBuilder aBuilder(buf, 0);
             translator->translate(aBuilder, *this, row);
-            return reinterpret_cast<const byte *>(buf.toByteArray());
+            return aBuilder.getSelf();
         }
         else
             return row.queryRow();
@@ -1153,7 +1153,7 @@ public:
                     buf.setLength(0);
                     MemoryBufferBuilder aBuilder(buf, 0);
                     translator->translate(aBuilder, unexpectedVirtualFieldCallback, keySearcher->queryRow().queryRow()); // MORE - could pass in partially-resolved RtlRow
-                    return (const byte *) buf.toByteArray();
+                    return aBuilder.getSelf();
                 }
                 else
                     return keySearcher->queryRow().queryRow();

rtl/eclrtl/rtlds_imp.hpp

@@ -678,11 +678,11 @@ class ECLRTL_API MemoryBufferBuilder : public RtlRowBuilderBase
 {
 public:
     MemoryBufferBuilder(MemoryBuffer & _buffer, unsigned _minSize)
-        : buffer(&_buffer), minSize(_minSize)
+        : buffer(&_buffer), minSize(_minSize ? _minSize : 1U)
     {
     }
 
-    MemoryBufferBuilder(unsigned _minSize) : minSize(_minSize)
+    MemoryBufferBuilder(unsigned _minSize) : minSize(_minSize ? _minSize : 1U)
     {
     }
 

system/jhtree/jhtree.cpp

@@ -460,7 +460,7 @@ public:
             buf.setLength(0);
             MemoryBufferBuilder aBuilder(buf, 0);
             layoutSize = layoutTrans->translate(aBuilder, unexpectedFieldCallback, reinterpret_cast<byte const *>(keyCursor->queryKeyBuffer()));
-            return reinterpret_cast<byte const *>(buf.toByteArray());
+            return aBuilder.getSelf();
         }
         else
             return reinterpret_cast<byte const *>(keyCursor->queryKeyBuffer());

testing/regress/ecl/key/projectnone.xml

@@ -0,0 +1,27 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>20</Result_3></Row>
+</Dataset>

testing/regress/ecl/projectnone.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.setup;
+prefix := setup.Files(false, false).QueryFilePrefix;
+
+#onwarning (1021, ignore); // Ignore warning that all output values are constant
+
+r1 := RECORD
+   string35 code;
+  END;
+
+r2 := RECORD
+  integer2 id;
+  embedded DATASET(r1) child;
+ END;
+
+mk1(unsigned c) := TRANSFORM(r1, SELF.code := (string)c; SELF := []);
+mk2(unsigned c) := TRANSFORM(r2, SELF.id := c; SELF.child := DATASET((c-1) % 5, mk1(COUNTER+c)), SELF := []);
+
+d := DATASET(20, mk2(counter));
+sequential(
+    output(d,,prefix+'myspill', overwrite);
+    output(DATASET(prefix+'myspill', r2, THOR), { one := 1 });
+    output(count(nocombine(DATASET(prefix+'myspill', r2, THOR))));
+);

thorlcr/activities/xmlwrite/thxmlwrite.cpp

@@ -30,11 +30,9 @@ class CXmlWriteActivityMaster : public CWriteMasterBase
 {
     IHThorXmlWriteArg *helper;
     ThorActivityKind kind;
-    unsigned headerLength;
-    unsigned footerLength;
 
 public:
-    CXmlWriteActivityMaster(CMasterGraphElement *info, ThorActivityKind _kind) : CWriteMasterBase(info), kind(_kind), headerLength(0), footerLength(0)
+    CXmlWriteActivityMaster(CMasterGraphElement *info, ThorActivityKind _kind) : CWriteMasterBase(info), kind(_kind)
     {
         helper = (IHThorXmlWriteArg *)queryHelper();
     }
@@ -70,19 +68,12 @@ public:
         }
         else
         {
-            StringBuffer supplied(helper->getHeader());
-            size32_t headerLength = supplied.length();
-            if (headerLength == 0 )
-                headerLength = supplied.set(DEFAULTXMLHEADER).newline().length();
-
-            props.setPropInt(FPheaderLength, headerLength);
-
-            supplied.set(helper->getFooter());
-            size32_t footerLength = supplied.length();
-            if (footerLength == 0 )
-                footerLength = supplied.set(DEFAULTXMLFOOTER).newline().length();
-
-            props.setPropInt(FPfooterLength, footerLength);
+            const char *header = helper->getHeader();
+            size32_t headerLen = header ? strlen(header) : strlen(DEFAULTXMLHEADER)+1; // DEFAULTXMLHEADER+'\n' is default output in slave
+            props.setPropInt(FPheaderLength, headerLen);
+            const char *footer = helper->getFooter();
+            size32_t footerLen = footer ? strlen(footer) : strlen(DEFAULTXMLFOOTER)+1; // DEFAULTXMLFOOTER+'\n' is default output in slave
+            props.setPropInt(FPfooterLength, footerLen);
         }
     }
 };

thorlcr/slave/slavmain.cpp

@@ -849,7 +849,7 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                 MemoryBufferBuilder aBuilder(diskFetchRowMb, 0);
                 LocalVirtualFieldCallback fieldCallback("<MORE>", requestHeader.fpos, 0);
                 translator->translate(aBuilder, fieldCallback, diskFetchRow);
-                diskFetchRow = reinterpret_cast<const byte *>(diskFetchRowMb.toByteArray());
+                diskFetchRow = aBuilder.getSelf();
             }
             size32_t fetchReplySz = sizeof(FetchReplyHeader);
             if (helper->fetchMatch(fetchKey, diskFetchRow))

tools/dumpkey/dumpkey.cpp

@@ -323,7 +323,7 @@ int main(int argc, const char **argv)
                         MemoryBufferBuilder aBuilder(buf, 0);
                         if (translator->translate(aBuilder, callback, buffer))
                         {
-                            outmeta->toXML((const byte *) buf.toByteArray(), *writer.get());
+                            outmeta->toXML(aBuilder.getSelf(), *writer.get());
                             printf("%s\n", writer->str());
                             writer->clear();
                         }