Merge branch 'candidate-7.4.x' into candidate-7.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlhtcpp.cpp

@@ -19403,6 +19403,7 @@ IHqlExpression * HqlCppTranslator::extractGlobalCSE(IHqlExpression * expr)
     HqlExprArray exprs;
     unwindCommaCompound(exprs, expr);
     transformer.analyseArray(exprs, 0);
+    transformer.analyseArray(exprs, 1);
     if (!transformer.worthTransforming())
         return LINK(expr);
 
@@ -19469,6 +19470,7 @@ void HqlCppTranslator::spotGlobalCSE(HqlExprArray & exprs)
     AutoScopeMigrateTransformer transformer(wu(), *this);
 
     transformer.analyseArray(exprs, 0);
+    transformer.analyseArray(exprs, 1);
     if (transformer.worthTransforming())
     {
         transformer.transformRoot(exprs, results);

ecl/hqlcpp/hqlresource.cpp

@@ -3905,7 +3905,18 @@ void EclResourcer::deriveUsageCounts(IHqlExpression * expr)
     if (info->numUses)
     {
         if (insideNeverSplit || insideSteppedNeverSplit)
+        {
             info->neverSplit = true;
+            //If this expression should never be split, ensure no input datasets are split, otherwise
+            //a input expression that has already been visited on another path may expect the result to be
+            //spilled, but no spill write will be generated.
+            IHqlExpression * cur = expr;
+            while (getNumActivityArguments(cur) == 1)
+            {
+                cur = cur->queryChild(0);
+                queryResourceInfo(cur)->neverSplit = true;
+            }
+        }
 
         if (info->isAlreadyInScope || info->isActivity || !info->containsActivity)
         {

ecl/hqlcpp/hqlttcpp.cpp

@@ -8504,7 +8504,7 @@ bool AutoScopeMigrateInfo::addGraph(unsigned graph)
 
 bool AutoScopeMigrateInfo::doAutoHoist(IHqlExpression * transformed, bool minimizeWorkunitTemporaries)
 {
-    if (useCount == 0)
+    if ((useCount == 0) || neverHoist)
         return false;
 
     node_operator op = original->getOperator();
@@ -8564,7 +8564,48 @@ AutoScopeMigrateTransformer::AutoScopeMigrateTransformer(IWorkUnit * _wu, HqlCpp
     globalTarget = NULL;
 }
 
-void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
+//Ensure all input activities are marked as never hoisting, but child activities are unaffected
+void AutoScopeMigrateTransformer::ensureNeverHoisted(IHqlExpression * expr)
+{
+    for (;;)
+    {
+        AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
+        if (extra->neverHoist)
+            return;
+        extra->neverHoist = true;
+        if (getNumActivityArguments(expr) != 1)
+            return;
+        expr = expr->queryChild(0);
+    }
+}
+
+//If an expression must be part of a compound source operation, ensure that the input
+//dataset is never hoisted, otherwise it will cause a later cannot key error.
+void AutoScopeMigrateTransformer::analysePass0(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_keyedlimit:
+        ensureNeverHoisted(expr);
+        break;
+    case no_hqlproject:
+    case no_newusertable:
+    case no_aggregate:
+    case no_newaggregate:
+        if (expr->hasAttribute(keyedAtom))
+            ensureNeverHoisted(expr);
+        break;
+    case no_filter:
+        if (filterIsKeyed(expr))
+            ensureNeverHoisted(expr);
+        break;
+    }
+
+    if (!alreadyVisited(expr))
+        NewHqlTransformer::analyseExpr(expr);
+}
+
+void AutoScopeMigrateTransformer::analysePass1(IHqlExpression * expr)
 {
     AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
     if (isConditional)
@@ -8587,6 +8628,19 @@ void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
     activityDepth = savedDepth;
 }
 
+void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
+{
+    switch (pass)
+    {
+    case 0:
+        analysePass0(expr);
+        break;
+    case 1:
+        analysePass1(expr);
+        break;
+    }
+}
+
 void AutoScopeMigrateTransformer::doAnalyseConditionalExpr(IHqlExpression * expr, unsigned firstConditional)
 {
     bool wasConditional = isConditional;

ecl/hqlcpp/hqlttcpp.ipp

@@ -709,19 +709,20 @@ private:
 class AutoScopeMigrateInfo : public NewTransformInfo
 {
 public:
-    AutoScopeMigrateInfo(IHqlExpression * _original) : NewTransformInfo(_original) { useCount = 0; condUseCount = 0; manyGraphs = false; firstUseIsConditional = false; firstUseIsSequential = false; globalInsideChild = false; lastGraph = 0; }
+    AutoScopeMigrateInfo(IHqlExpression * _original) : NewTransformInfo(_original) { }
 
     bool addGraph(unsigned graph);
     bool doAutoHoist(IHqlExpression * transformed, bool minimizeWorkunitTemporaries);
 
 public:
-    unsigned    useCount;
-    unsigned    condUseCount;
-    unsigned    lastGraph;
-    bool manyGraphs;
-    bool firstUseIsConditional;
-    bool firstUseIsSequential;
-    bool globalInsideChild;
+    unsigned    useCount = 0;
+    unsigned    condUseCount = 0;
+    unsigned    lastGraph = 0;
+    bool manyGraphs = false;
+    bool firstUseIsConditional = false;
+    bool firstUseIsSequential = false;
+    bool globalInsideChild = false;
+    bool neverHoist = false;
 };
 
 class AutoScopeMigrateTransformer : public NewHqlTransformer
@@ -738,6 +739,10 @@ protected:
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
     virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr) { return CREATE_NEWTRANSFORMINFO(AutoScopeMigrateInfo, expr); }
 
+    void analysePass0(IHqlExpression * expr);
+    void analysePass1(IHqlExpression * expr);
+
+    void ensureNeverHoisted(IHqlExpression * expr);
     IHqlExpression * hoist(IHqlExpression * expr, IHqlExpression * hoisted);
     IHqlExpression * transformCond(IHqlExpression * expr);
     void doAnalyseExpr(IHqlExpression * expr);

rtl/eclrtl/eclrtl.cpp

@@ -5152,7 +5152,7 @@ void rtlStrToUtf8X(size32_t & outlen, char * & out, size32_t inlen, const char *
     outlen = rtlUtf8Length(outsize, out);
 }
 
-#if U_ICU_VERSION_MAJOR_NUM<50
+#if U_ICU_VERSION_MAJOR_NUM<53
 static int rtlCompareUtf8Utf8ViaUnicode(size32_t llen, const char * left, size32_t rlen, const char * right, const char * locale)
 {
     rtlDataAttr uleft(llen*sizeof(UChar));
@@ -5166,7 +5166,7 @@ static int rtlCompareUtf8Utf8ViaUnicode(size32_t llen, const char * left, size32
 #ifdef _USE_ICU
 int rtlCompareUtf8Utf8(size32_t llen, const char * left, size32_t rlen, const char * right, const char * locale)
 {
-#if U_ICU_VERSION_MAJOR_NUM>=50
+#if U_ICU_VERSION_MAJOR_NUM>=53
     size_t lSize = rtlUtf8Size(llen, left);
     while (lSize && (left[lSize-1] == ' '))
         lSize--;

system/jlib/jsocket.cpp

@@ -6968,6 +6968,7 @@ public:
                 out.append(it.second);
             out.newline();
         }
+        out.newline().appendf("Whitelisting is currently: %s", enabled ? "enabled" : "disabled").newline();
         return out;
     }
     virtual void refresh() override

thorlcr/thorutil/thbuf.cpp

@@ -909,6 +909,7 @@ protected:
     Owned<CRowSet> inMemRows;
     CriticalSection crit;
     Linked<IOutputMetaData> meta;
+    Linked<IOutputRowSerializer> serializer;
     QueueOf<CRowSet, false> chunkPool;
     unsigned maxPoolChunks;
     bool reuseRowSets;
@@ -1042,7 +1043,8 @@ public:
 
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CSharedWriteAheadBase(CActivityBase *_activity, unsigned _outputCount, IThorRowInterfaces *rowIf) : activity(_activity), outputCount(_outputCount), meta(rowIf->queryRowMetaData())
+    CSharedWriteAheadBase(CActivityBase *_activity, unsigned _outputCount, IThorRowInterfaces *rowIf)
+        : activity(_activity), outputCount(_outputCount), meta(rowIf->queryRowMetaData()), serializer(rowIf->queryRowSerializer())
     {
         init();
         minChunkSize = 0x2000;
@@ -1193,7 +1195,6 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase
     QueueOf<Chunk, false> savedChunks;
     offset_t highOffset;
     Linked<IEngineRowAllocator> allocator;
-    Linked<IOutputRowSerializer> serializer;
     Linked<IOutputRowDeserializer> deserializer;
     IOutputMetaData *serializeMeta;
 
@@ -1501,7 +1502,7 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase
     }
 public:
     CSharedWriteAheadDisk(CActivityBase *activity, const char *spillName, unsigned outputCount, IThorRowInterfaces *rowIf, IDiskUsage *_iDiskUsage) : CSharedWriteAheadBase(activity, outputCount, rowIf),
-        allocator(rowIf->queryRowAllocator()), serializer(rowIf->queryRowSerializer()), deserializer(rowIf->queryRowDeserializer()), serializeMeta(meta->querySerializedDiskMeta()), iDiskUsage(_iDiskUsage)
+        allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer()), serializeMeta(meta->querySerializedDiskMeta()), iDiskUsage(_iDiskUsage)
     {
         assertex(spillName);
         spillFile.setown(createIFile(spillName));
@@ -1608,7 +1609,11 @@ class CSharedWriteAheadMem : public CSharedWriteAheadBase
     }
     virtual size32_t rowSize(const void *row)
     {
-        return meta->getRecordSize(row); // space in mem.
+        if (!row)
+            return 1; // eog;
+        CSizingSerializer ssz;
+        serializer->serialize(ssz, (const byte *)row);
+        return ssz.size();
     }
 public:
     CSharedWriteAheadMem(CActivityBase *activity, unsigned outputCount, IThorRowInterfaces *rowif, unsigned buffSize) : CSharedWriteAheadBase(activity, outputCount, rowif)