Merge branch 'candidate-7.6.x' into candidate-7.8.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlfold.cpp

@@ -1353,7 +1353,7 @@ IValue * foldExternalCall(IHqlExpression* expr, unsigned foldOptions)
     void *funcptr = loadExternalEntryPoint(expr, foldOptions, library.str(), entry.str(), hDll);
     if (!funcptr)
     {
-        UERRLOG("Failed to load function %s", entry.str());
+        DBGLOG("Failed to load function %s", entry.str());
         return NULL;
     }
     return doFoldExternalCall(expr, foldOptions, library.str(), entry.str(), funcptr);

ecl/hql/hqlutil.cpp

@@ -10305,6 +10305,18 @@ static IFieldFilter * createIfBlockFilter(IRtlFieldTypeDeserializer &deserialize
     return extractor.createSingleFieldFilter(deserializer);
 }
 
+static void cleanupFields(unsigned num, const RtlFieldInfo * * fieldsArray)
+{
+    for (unsigned i = 0; i < num; i++)
+    {
+        const RtlFieldInfo * child = fieldsArray[i];
+
+        if (!isVirtualInitializer(child->initializer))
+            free((void *)child->initializer);
+        delete child;
+    }
+    delete [] fieldsArray;
+}
 
 unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned &idx, const RtlFieldInfo * * fieldsArray, IHqlExpression *record, IHqlExpression *rowRecord)
 {
@@ -10339,10 +10351,18 @@ unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned
                 unsigned numFields = getFlatFieldCount(record);
                 info.fieldsArray = new const RtlFieldInfo * [numFields+1];
                 unsigned idx = 0;
-                info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
-                info.fieldsArray[idx] = nullptr;
-
-                info.filter = createIfBlockFilter(deserializer, rowRecord, field);
+                try
+                {
+                    info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
+                    info.fieldsArray[idx] = nullptr;
+                    info.filter = createIfBlockFilter(deserializer, rowRecord, field);
+                }
+                catch (...)
+                {
+                    //Clean up all the fields that have been added (idx is incremented by buildRtlRecordFields)
+                    cleanupFields(idx, info.fieldsArray);
+                    throw;
+                }
 
                 type = deserializer.addType(info, key);
             }
@@ -10475,7 +10495,15 @@ const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeIn
             unsigned numFields = getFlatFieldCount(record);
             info.fieldsArray = new const RtlFieldInfo * [numFields+1];
             unsigned idx = 0;
-            info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
+            try
+            {
+                info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
+            }
+            catch (...)
+            {
+                cleanupFields(idx, info.fieldsArray);
+                throw;
+            }
             info.fieldsArray[idx] = nullptr;
             break;
         }

plugins/fileservices/fileservices.cpp

@@ -641,7 +641,7 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
 
         Owned<IClientGetDFUWorkunit> req = server.createGetDFUWorkunitRequest();
         req->setWuid(wuid);
-        Linked<IClientGetDFUWorkunitResponse> result = server.GetDFUWorkunit(req);
+        Owned<IClientGetDFUWorkunitResponse> result = server.GetDFUWorkunit(req);
 
         const IMultiException* excep = &result->getExceptions();
         if ((excep != NULL) && (excep->ordinality() > 0))

roxie/ccd/ccdactivities.cpp

@@ -1098,6 +1098,7 @@ public:
 
 protected:
     IHThorCsvReadArg *helper;
+    Owned<IDirectReader> ownedReader; // Ensure that the byte stream reader is released
     const IResolvedFile *datafile;
     size32_t maxRowSize;
 
@@ -1119,9 +1120,12 @@ public:
     {
         {
             CriticalBlock p(pcrit);
+            //Processor should be clear when this is called, but include inside the critical block to be safe
+            processor.clear();
+            ownedReader.setown(manager->createReader(postFilter, false, readPos, parallelPartNo, numParallel, translators));
             processor.setown(
                     createCsvRecordProcessor(*this,
-                                             manager->createReader(postFilter, false, readPos, parallelPartNo, numParallel, translators),
+                                             ownedReader,
                                              packet->queryHeader().channel==1 && !resent,
                                              varFileInfo ? varFileInfo.get() : datafile, maxRowSize));
         }
@@ -1145,6 +1149,7 @@ public:
 
 protected:
     IHThorXmlReadArg *helper;
+    Owned<IDirectReader> ownedReader; // Ensure that the byte stream reader is released
 
 public:
     CRoxieXmlReadActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
@@ -1164,7 +1169,9 @@ public:
     {
         {
             CriticalBlock p(pcrit);
-            processor.setown(createXmlRecordProcessor(*this, manager->createReader(postFilter, false, readPos, parallelPartNo, numParallel, translators)));
+            processor.clear();
+            ownedReader.setown(manager->createReader(postFilter, false, readPos, parallelPartNo, numParallel, translators));
+            processor.setown(createXmlRecordProcessor(*this, ownedReader));
         }
         unsigned __int64 rowLimit = helper->getRowLimit();
         unsigned __int64 stopAfter = helper->getChooseNLimit();

roxie/ccd/ccdqueue.cpp

@@ -1282,6 +1282,8 @@ public:
         {
             if (E->errorCode()!=ROXIE_ABORT_ERROR)
                 throwRemoteException(E, activity, packet, false);
+            else
+                E->Release();
         }
         catch (...)
         {

roxie/ccd/ccdserver.cpp

@@ -12254,7 +12254,9 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
         {
             StringBuffer name(nameLen, nameBuff);
             StringBuffer value(valueLen, valueBuff);
-            if(*nameBuff == '_' && !checkReservedMetadataName(name))
+            rtlFree(nameBuff);
+            rtlFree(valueBuff);
+            if(*name == '_' && !checkReservedMetadataName(name))
             {
                 OwnedRoxieString fname(helper.getFileName());
                 throw MakeStringException(0, "Invalid name %s in user metadata for index %s (names beginning with underscore are reserved)", name.str(), fname.get());

rtl/eclrtl/rtldynfield.cpp

@@ -903,6 +903,7 @@ extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
     catch (IException *E)
     {
         EXCLOG(E);
+        E->Release();
         return false;
     }
 }

rtl/eclrtl/rtlrecord.cpp

@@ -149,8 +149,8 @@ private:
 class RtlCondFieldStrInfo : public RtlFieldStrInfo
 {
 public:
-    RtlCondFieldStrInfo(const RtlFieldInfo &from, const IfBlockInfo &_ifblock)
-    : RtlFieldStrInfo(from.name, from.xpath, from.type, from.flags | (RFTMinifblock|RFTMdynamic), (const char *) from.initializer),
+    RtlCondFieldStrInfo(const RtlFieldInfo &from, const IfBlockInfo &_ifblock, bool forcePayload)
+    : RtlFieldStrInfo(from.name, from.xpath, from.type, from.flags | (forcePayload ? RFTMinifblock|RFTMdynamic|RFTMispayloadfield : RFTMinifblock|RFTMdynamic), (const char *) from.initializer),
       origField(from),ifblock(_ifblock)
     {
     }
@@ -159,7 +159,7 @@ public:
     const IfBlockInfo &ifblock;
 };
 
-static unsigned expandNestedRows(unsigned idx, unsigned startIdx, StringBuffer &prefix, StringBuffer &xPathPrefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names, const char * *xpaths, const IfBlockInfo *inIfBlock, ConstPointerArrayOf<IfBlockInfo> &ifblocks)
+static unsigned expandNestedRows(unsigned idx, unsigned startIdx, StringBuffer &prefix, StringBuffer &xPathPrefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names, const char * *xpaths, const IfBlockInfo *inIfBlock, ConstPointerArrayOf<IfBlockInfo> &ifblocks, bool forcePayload)
 {
     for (;*fields;fields++)
     {
@@ -188,7 +188,7 @@ static unsigned expandNestedRows(unsigned idx, unsigned startIdx, StringBuffer &
                     if (!isEmptyString(xpath))
                         xPathPrefix.append(xpath).append('/');
                 }
-                idx = expandNestedRows(idx, isIfBlock ? startIdx : idx, prefix, xPathPrefix, nested, target, names, xpaths, nestIfBlock, ifblocks);
+                idx = expandNestedRows(idx, isIfBlock ? startIdx : idx, prefix, xPathPrefix, nested, target, names, xpaths, nestIfBlock, ifblocks, forcePayload || (cur->flags & RFTMispayloadfield) != 0);
                 prefix.setLength(prevPrefixLength);
                 if (xpaths)
                     xPathPrefix.setLength(prevXPathPrefixLength);
@@ -215,7 +215,12 @@ static unsigned expandNestedRows(unsigned idx, unsigned startIdx, StringBuffer &
                     xpaths[idx] = nullptr;
             }
             if (inIfBlock && !(cur->flags & RFTMinifblock))
-                target[idx++] = new RtlCondFieldStrInfo(*cur, *inIfBlock);
+                target[idx++] = new RtlCondFieldStrInfo(*cur, *inIfBlock, forcePayload);
+            else if (forcePayload && !(cur->flags & RFTMispayloadfield))
+            {
+                dbgassertex((cur->flags & RFTMdynamic) == 0);
+                target[idx++] = new RtlFieldStrInfo(cur->name, cur->xpath, cur->type, cur->flags | (RFTMispayloadfield|RFTMdynamic), (const char * ) cur->initializer);
+            }
             else
             {
                 dbgassertex((cur->flags & RFTMdynamic) == 0);
@@ -272,7 +277,7 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
                 xpaths = new const char *[numFields];
             fields = allocated;
             StringBuffer prefix, xPathPrefix;
-            unsigned idx = expandNestedRows(0, 0, prefix, xPathPrefix, originalFields, allocated, names, xpaths, nullptr, _ifblocks);
+            unsigned idx = expandNestedRows(0, 0, prefix, xPathPrefix, originalFields, allocated, names, xpaths, nullptr, _ifblocks, false);
             ifblocks = _ifblocks.detach();
             assertex(idx == numFields);
             allocated[idx] = nullptr;
@@ -585,7 +590,7 @@ const RtlRecord *RtlRecord::queryNested(unsigned fieldId) const
 const RtlFieldInfo *RtlRecord::queryOriginalField(unsigned idx) const
 {
     const RtlFieldInfo *field = queryField(idx);
-    if (field->flags & RFTMdynamic)
+    if (field->flags & RFTMinifblock)
         return &static_cast<const RtlCondFieldStrInfo *>(field)->origField;
     else
         return field;

system/jhtree/jhtree.cpp

@@ -2705,8 +2705,8 @@ public:
         unsigned i;
         for (i = 0; i < numkeys; i++)
         {
-            keyCursor = keyset->queryPart(i)->getCursor(filter, logExcessiveSeeks);
-            keyCursor->reset();
+            Owned<IKeyCursor> cursor = keyset->queryPart(i)->getCursor(filter, logExcessiveSeeks);
+            cursor->reset();
             for (;;)
             {
                 bool found;
@@ -2716,30 +2716,32 @@ public:
                 {
                     if (seek)
                     {
-                        if (keyCursor->skipTo(seek, seekOffset, seeklen))
+                        if (cursor->skipTo(seek, seekOffset, seeklen))
                             lskips++;
                         else
                             lnullSkips++;
                     }
-                    found = keyCursor->lookup(true, stats);
-                    if (!found || !seek || memcmp(keyCursor->queryKeyBuffer() + seekOffset, seek, seeklen) >= 0)
+                    found = cursor->lookup(true, stats);
+                    if (!found || !seek || memcmp(cursor->queryKeyBuffer() + seekOffset, seek, seeklen) >= 0)
                         break;
                 }
                 stats.noteSkips(lskips, lnullSkips);
                 if (found)
                 {
-                    IKeyCursor *mergeCursor = LINK(keyCursor);
+                    IKeyCursor *mergeCursor;
                     if (sortFromSeg)
-                        mergeCursor = keyCursor->fixSortSegs(sortFieldOffset);
+                        mergeCursor = cursor->fixSortSegs(sortFieldOffset);
+                    else
+                        mergeCursor = LINK(cursor);
+
                     keyNoArray.append(i);
                     cursorArray.append(*mergeCursor);
                     mergeHeapArray.append(activekeys++);
-                    if (!sortFromSeg || !keyCursor->nextRange(sortFromSeg))
+                    if (!sortFromSeg || !cursor->nextRange(sortFromSeg))
                         break;
                 }
                 else
                 {
-                    keyCursor->Release();
                     break;
                 }
             }

system/jlib/jstats.cpp

@@ -1777,25 +1777,14 @@ public:
 
     CStatisticCollection * ensureSubScope(const StatsScopeId & search, bool hasChildren)
     {
-        //MORE: Implement hasChildren
-        return resolveSubScope(search, true, false);
-    }
+        //Once the CStatisicCollection is created it should not be replaced - so that returned pointers remain valid.
+        CStatisticCollection * match = children.find(&search);
+        if (match)
+            return match;
 
-    CStatisticCollection * resolveSubScope(const StatsScopeId & search, bool create, bool replace)
-    {
-        if (!replace)
-        {
-            CStatisticCollection * match = children.find(&search);
-            if (match)
-                return LINK(match);
-        }
-        if (create)
-        {
-            CStatisticCollection * ret = new CStatisticCollection(this, search);
-            children.add(*ret);
-            return LINK(ret);
-        }
-        return NULL;
+        CStatisticCollection * ret = new CStatisticCollection(this, search);
+        children.add(*ret);
+        return ret;
     }
 
     virtual void serialize(MemoryBuffer & out) const
@@ -2006,7 +1995,6 @@ public:
     }
     virtual void endScope() override
     {
-        scopes.tos().Release();
         scopes.pop();
     }
     virtual void addStatistic(StatisticKind kind, unsigned __int64 value) override