Merge branch 'candidate-7.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/go_gold.sh

@@ -79,23 +79,24 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   doit "git submodule update --init --recursive"
   HPCC_PROJECTS=hpcc-helm
   HPCC_NAME=HPCC
-  if [[ "$HPCC_MAJOR" == "7" ]] && [[ "$HPCC_MINOR" == "8" ]] ; then
+  if [[ "$HPCC_MAJOR" == "7" ]] && [[ "$HPCC_MINOR" == "10" ]] ; then
     doit "rm -rf ./helm"
     doit "cp -rf $HPCC_DIR/helm ./helm" 
     doit "rm ./helm/hpcc/*.bak" 
     doit "git add -A ./helm"
   fi
   cd docs
+  for f in `find ${HPCC_DIR}/helm/examples -name Chart.yaml` ; do 
+    doit "helm package ${f%/*}/"  
+  done
   doit "helm package ${HPCC_DIR}/helm/hpcc/"
   doit "helm repo index . --url https://hpcc-systems.github.io/helm-chart"
   doit "git add *.tgz"
   
-  doit "git commit -a -s -m \"$HPCC_NAME Helm Charts $HPCC_SHORT_TAG Release Candidate $HPCC_SEQUENCE\""
+  doit "git commit -a -s -m \"$HPCC_NAME Helm Charts $HPCC_SHORT_TAG\""
+  if [[ "$HPCC_MAJOR" == "7" ]] && [[ "$HPCC_MINOR" == "10" ]] ; then
+    doit "git tag $FORCE $HPCC_MAJOR.$HPCC_MINOR.$HPCC_POINT && git push $REMOTE $HPCC_MAJOR.$HPCC_MINOR.$HPCC_POINT $FORCE"
+  fi
   doit "git push $REMOTE master $FORCE"
   popd
 fi
-
-# upmerge back to minor closedown branch to ensure incr.sh works
-doit "git checkout candidate-$HPCC_MAJOR.$NEW_MINOR.x"
-doit "git merge --no-ff --no-commit -X ours candidate-$HPCC_MAJOR.$NEW_MINOR.$NEW_POINT"
-doit "if ! git diff HEAD --exit-code ; then git merge --abort ; else git commit -s && git push origin candidate-$HPCC_MAJOR.$NEW_MINOR.x ; fi" 

cmake_modules/go_rc.sh

@@ -106,8 +106,3 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   doit "git push $REMOTE master $FORCE"
   popd
 fi
-
-# upmerge back to minor closedown branch to ensure incr.sh works
-doit "git checkout candidate-$HPCC_MAJOR.$NEW_MINOR.x"
-doit "git merge --no-ff --no-commit -X ours candidate-$HPCC_MAJOR.$NEW_MINOR.$NEW_POINT"
-doit "if ! git diff HEAD --exit-code ; then git merge --abort ; else git commit -s && git push origin candidate-$HPCC_MAJOR.$NEW_MINOR.x ; fi" 

common/dllserver/thorplugin.cpp

@@ -731,6 +731,11 @@ extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuf
     return getResourceXMLFromFile(filename, "WORKUNIT", 1000, xml);
 }
 
+extern DLLSERVER_API bool getArchiveXMLFromFile(const char *filename, StringBuffer &xml)
+{
+    return getResourceXMLFromFile(filename, "ARCHIVE", 1000, xml);
+}
+
 extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuffer &xml)
 {
     return getResourceXMLFromFile(filename, "MANIFEST", 1000, xml);

common/dllserver/thorplugin.hpp

@@ -45,6 +45,7 @@ extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll);
 extern DLLSERVER_API bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id);
 extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml);
 extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuffer &xml);
+extern DLLSERVER_API bool getArchiveXMLFromFile(const char *filename, StringBuffer &xml);
 extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuffer &xml);
 
 extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result);

common/fileview2/fverror.hpp

@@ -56,6 +56,7 @@
 #define FVERR_PluginMismatch                    6728
 #define FVERR_RowTooLarge                       6729
 #define FVERR_CouldNotProcessSchema             6730
+#define FVERR_MaxLengthExceeded                 6731
 
 #define FVERR_CouldNotResolveX_Text             "Could not resolve file '%s' in DFS"
 #define FVERR_NoRecordDescription_Text          "DFS did not contain a record description for '%s'"

common/fileview2/fvsource.cpp

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 #include "platform.h"
+#include "limits.h"
 #include "jliball.hpp"
 #include "eclrtl.hpp"
 #include "rtlds_imp.hpp"
@@ -594,6 +595,18 @@ void DataSourceMetaData::serialize(MemoryBuffer & buffer) const
 
 size32_t DataSourceMetaData::getRecordSize(const void *rec)
 {
+    return calcRecordSize(UINT_MAX, rec);
+}
+
+static void checkReadPastEnd(size32_t offset, size32_t delta, size32_t maxLength)
+{
+    //Check for reading past the end of the buffer, or for the size wrapping 32bits.
+    if ((offset > maxLength) || (offset + delta < offset))
+        throw makeStringException(FVERR_MaxLengthExceeded, "Read past the end of a variable size block (MAXLENGTH exceeded)");
+}
+
+size32_t DataSourceMetaData::calcRecordSize(size32_t maxLength, const void *rec)
+{
     if (isStoredFixedWidth)
         return minRecordSize;
     if (!rec)
@@ -616,21 +629,27 @@ size32_t DataSourceMetaData::getRecordSize(const void *rec)
             case type_string:
             case type_table:
             case type_groupedtable:
+                checkReadPastEnd(curOffset, sizeof(unsigned), maxLength);
                 size = *((unsigned *)cur) + sizeof(unsigned);
                 break;
             case type_set:
+                checkReadPastEnd(curOffset, sizeof(bool) + sizeof(unsigned), maxLength);
                 size = *((unsigned *)(cur + sizeof(bool))) + sizeof(unsigned) + sizeof(bool);
                 break;
             case type_qstring:
+                checkReadPastEnd(curOffset, sizeof(unsigned), maxLength);
                 size = rtlQStrSize(*((unsigned *)cur)) + sizeof(unsigned);
                 break;
             case type_unicode:
+                checkReadPastEnd(curOffset, sizeof(unsigned), maxLength);
                 size = *((unsigned *)cur)*2 + sizeof(unsigned);
                 break;
             case type_utf8:
+                checkReadPastEnd(curOffset, sizeof(unsigned), maxLength);
                 size = sizeof(unsigned) + rtlUtf8Size(*(unsigned *)cur, cur+sizeof(unsigned));
                 break;
             case type_varstring:
+                //buffer overflow checking for the following will wait until code is reimplemented
                 size = strlen((char *)cur)+1;
                 break;
             case type_varunicode:
@@ -660,6 +679,7 @@ size32_t DataSourceMetaData::getRecordSize(const void *rec)
         else
             bitsRemaining = 0;
 
+        checkReadPastEnd(curOffset, size, maxLength);
         curOffset += size;
     }
     return curOffset;

common/fileview2/fvsource.ipp

@@ -137,12 +137,13 @@ public:
     virtual size32_t getFixedSize() const;
     virtual size32_t getRecordSize(unsigned maxLength, const void *rec)
     {
-        return getRecordSize(rec);
+        return calcRecordSize(maxLength, rec);
     }
     virtual size32_t getMinRecordSize() const;
 
 protected:
     void addSimpleField(const char * name, const char * xpath, ITypeInfo * type, unsigned flag=FVFFnone);
+    size32_t calcRecordSize(unsigned maxLength, const void *rec);
     void gatherFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
     void gatherChildFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
     void gatherAttributes();

common/thorhelper/thorcommon.hpp

@@ -96,13 +96,14 @@ enum RowReaderWriterFlags
     rw_buffered       = 0x80,
     rw_lzw            = 0x100, // if rw_compress
     rw_lz4            = 0x200, // if rw_compress
-    rw_sparse         = 0x400  // NB: mutually exclusive with rw_grouped
+    rw_sparse         = 0x400, // NB: mutually exclusive with rw_grouped
+    rw_lz4hc          = 0x800  // if rw_compress
 };
 #define DEFAULT_RWFLAGS (rw_buffered|rw_autoflush|rw_compressblkcrc)
 inline bool TestRwFlag(unsigned flags, RowReaderWriterFlags flag) { return 0 != (flags & flag); }
 
-#define COMP_MASK (rw_compress|rw_compressblkcrc|rw_fastlz|rw_lzw|rw_lz4)
-#define COMP_TYPE_MASK (rw_fastlz|rw_lzw|rw_lz4)
+#define COMP_MASK (rw_compress|rw_compressblkcrc|rw_fastlz|rw_lzw|rw_lz4|rw_lz4hc)
+#define COMP_TYPE_MASK (rw_fastlz|rw_lzw|rw_lz4|rw_lz4hc)
 inline void setCompFlag(const char *compStr, unsigned &flags)
 {
     flags &= ~COMP_TYPE_MASK;
@@ -112,7 +113,9 @@ inline void setCompFlag(const char *compStr, unsigned &flags)
             flags |= rw_fastlz;
         else if (0 == stricmp("LZW", compStr))
             flags |= rw_lzw;
-        else // not specifically FLZ or LZW so set to default LZ4
+        else if (0 == stricmp("LZ4HC", compStr))
+            flags |= rw_lz4hc;
+        else // not specifically FLZ, LZW, or FL4HC so set to default LZ4
             flags |= rw_lz4;
     }
     else // default is LZ4
@@ -126,6 +129,9 @@ inline unsigned getCompMethod(unsigned flags)
         compMethod = COMPRESS_METHOD_LZW;
     else if (TestRwFlag(flags, rw_fastlz))
         compMethod = COMPRESS_METHOD_FASTLZ;
+    else if (TestRwFlag(flags, rw_lz4hc))
+        compMethod = COMPRESS_METHOD_LZ4HC;
+
     return compMethod;
 }
 
@@ -138,6 +144,8 @@ inline unsigned getCompMethod(const char *compStr)
             compMethod = COMPRESS_METHOD_FASTLZ;
         else if (0 == stricmp("LZW", compStr))
             compMethod = COMPRESS_METHOD_LZW;
+        else if (0 == stricmp("LZ4HC", compStr))
+            compMethod = COMPRESS_METHOD_LZ4HC;
     }
     return compMethod;
 }

common/thorhelper/thorhelper.hpp

@@ -28,6 +28,10 @@
  #define THORHELPER_API DECL_IMPORT
 #endif
 
+#define SLAVEIDSTR "#SLAVEID#"
+#define THORMASTERLOGSEARCHSTR "thormaster."
+#define THORSLAVELOGSEARCHSTR "thorslave."
+
 interface IXmlToRawTransformer : extends IInterface
 {
     virtual IDataVal & transform(IDataVal & result, size32_t len, const void * text, bool isDataset) = 0;

common/workunit/workunit.cpp

@@ -4226,8 +4226,8 @@ public:
             { c->commit(); }
     virtual IWUException * createException()
             { return c->createException(); }
-    virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log)
-            { c->addProcess(type, instance, pid, log); }
+    virtual void addProcess(const char *type, const char *instance, unsigned pid, unsigned max, const char *pattern, bool singleLog, const char *log)
+            { c->addProcess(type, instance, pid, max, pattern, singleLog, log); }
     virtual void protect(bool protectMode)
             { c->protect(protectMode); }
     virtual void setAction(WUAction action)
@@ -7046,7 +7046,8 @@ bool CLocalWorkUnit::setDistributedAccessToken(const char * user)
     }
     else
     {
-        WARNLOG("Cannot sign Distributed Access Token, digisign signing not configured");
+        if (workUnitTraceLevel > 1)
+            WARNLOG("Cannot sign Distributed Access Token, digisign signing not configured");
         datoken.append(";");
     }
     p->setProp("@distributedAccessToken", datoken);
@@ -8181,7 +8182,8 @@ IStringIterator *CLocalWorkUnit::getProcesses(const char *type) const
     return new CStringPTreeTagIterator(p->getElements(xpath.str()));
 }
 
-void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid, const char *log)
+void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned pid,
+    unsigned max, const char *pattern, bool singleLog, const char *log)
 {
     VStringBuffer processType("Process/%s", type);
     VStringBuffer xpath("%s/%s", processType.str(), instance);
@@ -8194,6 +8196,12 @@ void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned
         node = node->addPropTree(instance);
         node->setProp("@log", log);
         node->setPropInt("@pid", pid);
+        if (max > 0)
+            node->setPropInt("@max", max);
+        if (!isEmptyString(pattern))
+            node->setProp("@pattern", pattern);
+        if (singleLog)
+            node->setPropBool("@singleLog", true);
     }
 }
 

common/workunit/workunit.hpp

@@ -1303,7 +1303,7 @@ interface IWorkUnit : extends IConstWorkUnit
     virtual void clearExceptions(const char *source=nullptr) = 0;
     virtual void commit() = 0;
     virtual IWUException * createException() = 0;
-    virtual void addProcess(const char *type, const char *instance, unsigned pid, const char *log=NULL) = 0;
+    virtual void addProcess(const char *type, const char *instance, unsigned pid, unsigned max, const char *pattern, bool singleLog, const char *log=nullptr) = 0;
     virtual void setAction(WUAction action) = 0;
     virtual void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite) = 0;
     virtual void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite) = 0;

common/workunit/workunit.ipp

@@ -354,7 +354,7 @@ public:
     void clearExceptions(const char *source=nullptr);
     void commit();
     IWUException *createException();
-    void addProcess(const char *type, const char *instance, unsigned pid, const char *log);
+    void addProcess(const char *type, const char *instance, unsigned pid, unsigned max, const char *pattern, bool singleLog, const char *log);
     void setAction(WUAction action);
     void setApplicationValue(const char * application, const char * propname, const char * value, bool overwrite);
     void setApplicationValueInt(const char * application, const char * propname, int value, bool overwrite);

common/wuanalysis/anacommon.cpp

@@ -66,7 +66,7 @@ void PerformanceIssue::createException(IWorkUnit * wu)
     StringBuffer s(comment);        // Append scope to comment as scope column is not visible in ECLWatch
     s.appendf(" (%s)", scope.str());
     we->setExceptionMessage(s.str());
-    we->setExceptionSource("Workunit Analyser");
+    we->setExceptionSource("Workunit Analyzer");
 }
 
 void PerformanceIssue::set(AnalyzerErrorCode _errorCode, stat_type _cost, const char * msg, ...)

common/wuanalysis/anacommon.hpp

@@ -23,11 +23,6 @@
 #include "eclhelper.hpp"
 #include "anaerrorcodes.hpp"
 
-#ifdef WUANALYSIS_EXPORTS
-    #define WUANALYSIS_API DECL_EXPORT
-#else
-    #define WUANALYSIS_API DECL_IMPORT
-#endif
 
 interface IWuScope
 {
@@ -82,6 +77,13 @@ private:
     StringBuffer comment;
 };
 
+typedef enum { watOptFirst=0, watOptMinInterestingTime=0, watOptMinInterestingCost, watOptSkewThreshold, watOptMinRowsPerNode, watPreFilteredKJThreshold, watOptMax } WutOptionType ;
+
+interface IAnalyserOptions
+{
+    virtual stat_type queryOption(WutOptionType opt) const = 0;
+};
+
 extern int compareIssuesCostOrder(CInterface * const * _l, CInterface * const * _r);
 
 #endif

common/wuanalysis/anaerrorcodes.hpp

@@ -9,7 +9,8 @@ typedef enum
     ANA_DISTRIB_SKEW_INPUT_ID,
     ANA_DISTRIB_SKEW_OUTPUT_ID,
     ANA_IOSKEW_RECORDS_ID,
-    ANA_IOSKEW_CHILDRECORDS_ID
+    ANA_IOSKEW_CHILDRECORDS_ID,
+    ANA_KJ_EXCESS_PREFILTER_ID
 } AnalyzerErrorCode;
 
 #endif /* COMMON_WUANALYSIS_ANAERRORCODES_HPP_ */

common/wuanalysis/anarule.cpp

@@ -41,16 +41,16 @@ class DistributeSkewRule : public ActivityKindRule
 public:
     DistributeSkewRule() : ActivityKindRule(TAKhashdistribute) {}
 
-    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const WuAnalyseOptions & options) override
+    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const IAnalyserOptions & options) override
     {
         IWuEdge * outputEdge = activity.queryOutput(0);
         if (!outputEdge)
             return false;
         stat_type rowsAvg = outputEdge->getStatRaw(StNumRowsProcessed, StAvgX);
-        if (rowsAvg < rowsThreshold)
+        if (rowsAvg < options.queryOption(watOptMinRowsPerNode))
             return false;
         stat_type rowsMaxSkew = outputEdge->getStatRaw(StNumRowsProcessed, StSkewMax);
-        if (rowsMaxSkew > options.skewThreshold)
+        if (rowsMaxSkew > options.queryOption(watOptSkewThreshold))
         {
             // Use downstream activity time to calculate approximate cost
             IWuActivity * targetActivity = outputEdge->queryTarget();
@@ -70,9 +70,6 @@ public:
         }
         return false;
     }
-
-protected:
-    static const stat_type rowsThreshold = 100;                // avg rows per node.
 };
 
 class IoSkewRule : public AActivityRule
@@ -128,12 +125,11 @@ public:
         return false;
     }
 
-    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const WuAnalyseOptions & options) override
+    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const IAnalyserOptions & options) override
     {
         stat_type ioAvg = activity.getStatRaw(stat, StAvgX);
         stat_type ioMaxSkew = activity.getStatRaw(stat, StSkewMax);
-
-        if (ioMaxSkew > options.skewThreshold)
+        if (ioMaxSkew > options.queryOption(watOptSkewThreshold))
         {
             stat_type timeMaxLocalExecute = activity.getStatRaw(StTimeLocalExecute, StMaxX);
             stat_type timeAvgLocalExecute = activity.getStatRaw(StTimeLocalExecute, StAvgX);
@@ -158,6 +154,36 @@ protected:
     const char * category;
 };
 
+class KeyedJoinExcessRejectedRowsRule : public ActivityKindRule
+{
+public:
+    KeyedJoinExcessRejectedRowsRule() : ActivityKindRule(TAKkeyedjoin) {}
+
+    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const IAnalyserOptions & options) override
+    {
+        stat_type preFiltered = activity.getStatRaw(StNumPreFiltered);
+        if (preFiltered)
+        {
+            IWuEdge * inputEdge = activity.queryInput(0);
+            stat_type rowscnt = inputEdge->getStatRaw(StNumRowsProcessed);
+            if (rowscnt)
+            {
+                stat_type preFilteredPer = statPercent( (double) preFiltered * 100.0 / rowscnt );
+                if (preFilteredPer > options.queryOption(watPreFilteredKJThreshold))
+                {
+                    IWuActivity * inputActivity = inputEdge->querySource();
+                    // Use input activity as the basis of cost because the rows generated from input activity is being filtered out
+                    stat_type timeAvgLocalExecute = inputActivity->getStatRaw(StTimeLocalExecute, StAvgX);
+                    stat_type cost = statPercentageOf(timeAvgLocalExecute, preFilteredPer);
+                    result.set(ANA_KJ_EXCESS_PREFILTER_ID, cost, "Large number of rows from left dataset rejected in keyed join");
+                    updateInformation(result, activity);
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+};
 
 void gatherRules(CIArrayOf<AActivityRule> & rules)
 {
@@ -165,4 +191,5 @@ void gatherRules(CIArrayOf<AActivityRule> & rules)
     rules.append(*new IoSkewRule(StTimeDiskReadIO, "disk read"));
     rules.append(*new IoSkewRule(StTimeDiskWriteIO, "disk write"));
     rules.append(*new IoSkewRule(StTimeSpillElapsed, "spill"));
+    rules.append(*new KeyedJoinExcessRejectedRowsRule);
 }

common/wuanalysis/anarule.hpp

@@ -27,7 +27,7 @@ class AActivityRule : public CInterface
 {
 public:
     virtual bool isCandidate(IWuActivity & activity) const = 0;
-    virtual bool check(PerformanceIssue & results, IWuActivity & activity, const WuAnalyseOptions & options) = 0;
+    virtual bool check(PerformanceIssue & results, IWuActivity & activity, const IAnalyserOptions & options) = 0;
     virtual void updateInformation(PerformanceIssue & result,  IWuActivity & activity)
     {
         StringBuffer def;

common/wuanalysis/anawu.cpp

@@ -95,15 +95,102 @@ protected:
 };
 
 //-----------------------------------------------------------------------------------------------------------
+typedef enum {wutOptValueTypeFirst=0, wutOptValueTypeMSec=0, wutOptValueTypeSeconds, wutOptValueTypePercent, wutOptValueTypeCount, wutOptValueTypeMax} WutOptValueType;
+
+struct WuOption
+{
+    WutOptionType option;
+    const char * name;
+    stat_type defaultValue;
+    WutOptValueType type;
+};
+
+constexpr struct WuOption wuOptionsDefaults[watOptMax]
+= { {watOptMinInterestingTime, "minInterestingTime", 1000, wutOptValueTypeMSec},
+    {watOptMinInterestingCost, "minInterestingCost", 30000, wutOptValueTypeMSec},
+    {watOptSkewThreshold, "skewThreshold", 20, wutOptValueTypePercent},
+    {watOptMinRowsPerNode, "minRowsPerNode", 1000, wutOptValueTypeCount},
+    {watPreFilteredKJThreshold, "preFilteredKJThreshold", 50, wutOptValueTypePercent} };
+
+constexpr bool checkWuOptionsDefaults(int i = watOptMax)
+{
+    return ((wuOptionsDefaults[i-1].name != nullptr && (wuOptionsDefaults[i-1].option == i-1) && wuOptionsDefaults[i-1].type < wutOptValueTypeMax ) && 
+           (i==1 || checkWuOptionsDefaults(i-1)));
+}
+static_assert(checkWuOptionsDefaults(), "wuOptionsDefaults[] not populated correctly");
+
+class WuAnalyserOptions : public IAnalyserOptions
+{
+public:
+   WuAnalyserOptions()
+    {
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+            setOptionValue(static_cast<WutOptionType>(opt), wuOptionsDefaults[opt].defaultValue);
+    }
+
+    void setOptionValue(WutOptionType opt, __int64 val)
+    {
+        assertex(opt<watOptMax);
+        switch(wuOptionsDefaults[opt].type)
+        {
+            case wutOptValueTypeMSec:
+                wuOptions[opt] = msecs2StatUnits(val);
+                break;
+            case wutOptValueTypeSeconds:
+                wuOptions[opt] = seconds2StatUnits(val);
+                break;
+            case wutOptValueTypePercent:
+                wuOptions[opt] = statPercent((stat_type)val);
+                break;
+            case wutOptValueTypeCount:
+                wuOptions[opt] = (stat_type) val;
+                break;
+            default:
+                throw MakeStringException(-1, "WuAnalyserOptions::setOptionValue - unknown wuOptionsDefaults[%d].type=%d", (int) opt, (int) wuOptionsDefaults[opt].type);
+        }
+    }
+
+    void applyConfig(IPropertyTree *options)
+    {
+        if (!options) return;
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+        {
+            StringBuffer wuOptionName("@");
+            wuOptionName.append(wuOptionsDefaults[opt].name);
+            __int64 val =  options->getPropInt64(wuOptionName, -1);
+            if (val!=-1)
+                setOptionValue(static_cast<WutOptionType>(opt), val);
+        }
+    }
+
+    void applyConfig(IConstWorkUnit * wu)
+    {
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+        {
+            StringBuffer wuOptionName("analyzer_");
+            wuOptionName.append(wuOptionsDefaults[opt].name);
+            __int64 val = wu->getDebugValueInt64(wuOptionName, -1);
+            if (val!=-1)
+                setOptionValue(static_cast<WutOptionType>(opt), val);
+        }
+    }
+    stat_type queryOption(WutOptionType opt) const override { return wuOptions[opt]; }
+private:
+    stat_type wuOptions[watOptMax];
+};
+//-----------------------------------------------------------------------------------------------------------
+
+
 class WorkunitAnalyser
 {
 public:
-    WorkunitAnalyser(WuAnalyseOptions & _options);
+    WorkunitAnalyser();
 
     void check(const char * scope, IWuActivity & activity);
     void analyse(IConstWorkUnit * wu);
     void print();
     void update(IWorkUnit *wu);
+    void applyConfig(IPropertyTree *cfg);
 
 protected:
     void collateWorkunitStats(IConstWorkUnit * workunit, const WuScopeFilter & filter);
@@ -112,7 +199,7 @@ protected:
     CIArrayOf<AActivityRule> rules;
     CIArrayOf<PerformanceIssue> issues;
     WuScope root;
-    WuAnalyseOptions options;
+    WuAnalyserOptions options;
 };
 
 //-----------------------------------------------------------------------------------------------------------
@@ -306,17 +393,23 @@ public:
 };
 
 
+
 //-----------------------------------------------------------------------------------------------------------
-WorkunitAnalyser::WorkunitAnalyser(WuAnalyseOptions & _options) : root("", nullptr), options(_options)
+
+WorkunitAnalyser::WorkunitAnalyser() : root("", nullptr)
 {
     gatherRules(rules);
 }
 
+void WorkunitAnalyser::applyConfig(IPropertyTree *cfg)
+{
+    options.applyConfig(cfg);
+}
+
 void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
 {
-    if (activity.getStatRaw(StTimeLocalExecute, StMaxX) < options.minInterestingTime)
+    if (activity.getStatRaw(StTimeLocalExecute, StMaxX) < options.queryOption(watOptMinInterestingTime))
         return;
-
     Owned<PerformanceIssue> highestCostIssue;
     ForEachItemIn(i, rules)
     {
@@ -325,7 +418,7 @@ void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
             Owned<PerformanceIssue> issue (new PerformanceIssue);
             if (rules.item(i).check(*issue, activity, options))
             {
-                if (issue->getCost() >= options.minCost)
+                if (issue->getCost() >= options.queryOption(watOptMinInterestingCost))
                 {
                     if (!highestCostIssue || highestCostIssue->getCost() < issue->getCost())
                         highestCostIssue.setown(issue.getClear());
@@ -344,6 +437,7 @@ void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
 
 void WorkunitAnalyser::analyse(IConstWorkUnit * wu)
 {
+    options.applyConfig(wu);
     WuScopeFilter filter;
     filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes);
     filter.finishedFilter();
@@ -407,22 +501,23 @@ WuScope * WorkunitAnalyser::selectFullScope(const char * scope)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, WuAnalyseOptions & options)
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options)
 {
-    WorkunitAnalyser analyser(options);
+    WorkunitAnalyser analyser;
+    analyser.applyConfig(options);
     analyser.analyse(wu);
     analyser.update(wu);
 }
 
-void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, WuAnalyseOptions & options, bool updatewu)
+void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, bool updatewu)
 {
-    WorkunitAnalyser analyser(options);
+    WorkunitAnalyser analyser;
     analyser.analyse(wu);
     analyser.print();
     if (updatewu)
     {
         Owned<IWorkUnit> lockedwu = &(wu->lock());
-        lockedwu->clearExceptions("Workunit Analyser");
+        lockedwu->clearExceptions("Workunit Analyzer");
         analyser.update(lockedwu);
     }
 }

common/wuanalysis/anawu.hpp

@@ -18,15 +18,12 @@
 #ifndef ANAWU_HPP
 #define ANAWU_HPP
 
-#include "anacommon.hpp"
-
-struct WuAnalyseOptions
-{
-    stat_type minInterestingTime = msecs2StatUnits(10);// ignore anything under 10 millisecond
-    stat_type minCost = seconds2StatUnits(30);          // only interested in costs of > 30s
-    stat_type skewThreshold = statSkewPercent(20);     // minimum interesting skew measurment
-};
+#ifdef WUANALYSIS_EXPORTS
+    #define WUANALYSIS_API DECL_EXPORT
+#else
+    #define WUANALYSIS_API DECL_IMPORT
+#endif
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, WuAnalyseOptions & options);
-void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, WuAnalyseOptions & options, bool updatewu);
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options);
+void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, bool updatewu);
 #endif

dali/base/dadiags.cpp

@@ -256,11 +256,12 @@ public:
                     bool success = querySDSServer().setSDSDebug(arr, reply);
                     mb.append(success).append(reply);
                 }
-                else if (0 == stricmp(id, "whitelist")) {
+                else if (strieq(id, "whitelist") || strieq(id, "allowlist")) // NB: "whitelist" is deprecated
+                {
                     Owned<IMPServer> mpServer = getMPServer();
-                    IWhiteListHandler *whiteListHandler = mpServer->queryWhiteListCallback();
-                    if (whiteListHandler)
-                        mb.append(whiteListHandler->getWhiteList(buf).str());
+                    IAllowListHandler *allowListHandler = mpServer->queryAllowListCallback();
+                    if (allowListHandler)
+                        mb.append(allowListHandler->getAllowList(buf).str());
                 }
                 else
                     mb.append(StringBuffer("UNKNOWN OPTION: ").append(id).str());

dali/base/dasds.cpp

@@ -6484,9 +6484,9 @@ void CCovenSDSManager::saveDelta(const char *path, IPropertyTree &changeTree)
         if (startsWith(path, "/Environment") || (streq(path, "/") && changeTree.hasProp("*[@name=\"Environment\"]")))
         {
             Owned<IMPServer> mpServer = getMPServer();
-            IWhiteListHandler *whiteListHandler = mpServer->queryWhiteListCallback();
-            if (whiteListHandler)
-                whiteListHandler->refresh();
+            IAllowListHandler *allowListHandler = mpServer->queryAllowListCallback();
+            if (allowListHandler)
+                allowListHandler->refresh();
             PROGLOG("Dali Environment updated, path = %s", path);
             return;
         }

dali/dalidiag/dalidiag.cpp

@@ -42,7 +42,7 @@ void usage(const char *exe)
     printf("-sdsstats           -- SDS statistics\n");
     printf("-sdssubscribers     -- list active SDS subscribers\n");
     printf("-connections        -- list SDS connections\n");
-    printf("-whitelist          -- list white list\n");
+    printf("-allowlist          -- list entries in allowlist\n");
     printf("-threads            -- running threads\n");
     printf("-mpqueue            -- list waiting MP queue items\n");
     printf("-clients            -- list connected Dali clients\n");

dali/sasha/saverify.cpp

@@ -199,6 +199,7 @@ public:
 class CFileCrcList
 {
     CIpTable dafilesrvips;
+    Owned<IUserDescriptor> udesc;
 public:
     bool &stopped;
     CIArrayOf<CFileCrcItem> list;
@@ -206,6 +207,10 @@ public:
     CFileCrcList(bool &_stopped)
         : stopped(_stopped)
     {
+        StringBuffer userName;
+        serverConfig->getProp("@sashaUser", userName);
+        udesc.setown(createUserDescriptor());
+        udesc->set(userName.str(), nullptr);
     }
 
     void add(RemoteFilename &filename,unsigned partno,unsigned copy,unsigned crc)
@@ -221,7 +226,7 @@ public:
 
     void verifyFile(const char *name,CDateTime *cutoff)
     {
-        Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser);
+        Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,udesc,false,false,false,nullptr,defaultPrivilegedUser);
         if (!file)
             return;
         IPropertyTree &fileprops = file->queryAttributes();
@@ -343,7 +348,7 @@ public:
             }
         }
         if (!stopped) {
-            file.setown(queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser));
+            file.setown(queryDistributedFileDirectory().lookup(name,udesc,false,false,false,nullptr,defaultPrivilegedUser));
             if (!file)
                 return;
             if (afor.ok) {
@@ -365,6 +370,7 @@ class CSashaVerifierServer: public ISashaServer, public Thread
 
     bool stopped;
     Semaphore stopsem;
+    Owned<IUserDescriptor> udesc;
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -372,6 +378,11 @@ public:
         : Thread("CSashaVerifierServer")
     {
         stopped = false;
+
+        StringBuffer userName;
+        serverConfig->getProp("@sashaUser", userName);
+        udesc.setown(createUserDescriptor());
+        udesc->set(userName.str(), nullptr);
     }
 
     ~CSashaVerifierServer()
@@ -411,7 +422,7 @@ public:
             try {
                 PROGLOG("VERIFIER: Started");
                 CFileCrcList filelist(stopped);
-                Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator("*",UNKNOWN_USER,true,false);//MORE:Pass IUserDescriptor
+                Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator("*",udesc,true,false);
                 if (iter) {
                     CDateTime mincutoff;
                     mincutoff.setNow();

dali/sasha/saxref.cpp

@@ -672,7 +672,7 @@ public:
     CLargeMemoryAllocator mem;
     bool verbose;
     unsigned numuniqnodes = 0;
-
+    Owned<IUserDescriptor> udesc;
 
     CNewXRefManager(unsigned maxMb=DEFAULT_MAXMEMORY)
         : mem(0x100000*((memsize_t)maxMb),0x10000,true)
@@ -687,6 +687,11 @@ public:
         orphansbranch.setown(createPTree("Orphans"));
         dirbranch.setown(createPTree("Directories"));
         log("Max memory = %d MB", maxMb);
+
+        StringBuffer userName;
+        serverConfig->getProp("@sashaUser", userName);
+        udesc.setown(createUserDescriptor());
+        udesc->set(userName.str(), nullptr);
     }
 
     ~CNewXRefManager()
@@ -1208,7 +1213,7 @@ public:
         CDfsLogicalFileName lfn;
         if (lfn.setFromMask(mask.str(),rootdir)) { // orphans are only orphans if there doesn't exist a valid file
             try {
-                if (queryDistributedFileDirectory().exists(lfn.get(),UNKNOWN_USER,true,false)) {
+                if (queryDistributedFileDirectory().exists(lfn.get(),udesc,true,false)) {
                     warn(mask.str(),"Orphans ignored as %s exists",lfn.get());
                     return;
                 }
@@ -1498,7 +1503,7 @@ public:
             }
             Owned<IDistributedFile> file;
             try {
-                file.setown(queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser));
+                file.setown(queryDistributedFileDirectory().lookup(lfn,udesc,false,false,false,nullptr,defaultPrivilegedUser));
             }
             catch (IException *e) {
                 EXCLOG(e,"CNewXRefManager::listLost");
@@ -2257,6 +2262,7 @@ class CSashaExpiryServer: public ISashaServer, public Thread
     bool stopped;
     Semaphore stopsem;
     Mutex runmutex;
+    Owned<IUserDescriptor> udesc;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -2265,6 +2271,11 @@ public:
         : Thread("CSashaExpiryServer")
     {
         stopped = false;
+
+        StringBuffer userName;
+        serverConfig->getProp("@sashaUser", userName);
+        udesc.setown(createUserDescriptor());
+        udesc->set(userName.str(), nullptr);
     }
 
     ~CSashaExpiryServer()
@@ -2300,7 +2311,7 @@ public:
         unsigned defaultExpireDays = serverConfig->getPropInt("DfuExpiry/@expiryDefault", DEFAULT_EXPIRYDAYS);
         unsigned defaultPersistExpireDays = serverConfig->getPropInt("DfuExpiry/@persistExpiryDefault", DEFAULT_PERSISTEXPIRYDAYS);
         StringArray expirylist;
-        Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator("*",UNKNOWN_USER,true,false);//MORE:Pass IUserDescriptor
+        Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator("*",udesc,true,false);
         ForEach(*iter)
         {
             IPropertyTree &attr=iter->query();
@@ -2352,7 +2363,7 @@ public:
                 /* NB: 0 timeout, meaning fail and skip, if there is any locking contention.
                  * If the file is locked, it implies it is being accessed.
                  */
-                queryDistributedFileDirectory().removeEntry(lfn, UNKNOWN_USER, NULL, 0, true); //MORE:Pass IUserDescriptor
+                queryDistributedFileDirectory().removeEntry(lfn, udesc, NULL, 0, true);
                 PROGLOG(LOGPFX2 "Deleted %s",lfn);
             }
             catch (IException *e) // may want to just detach if fails

dali/server/daserver.cpp

@@ -142,7 +142,7 @@ void usage(void)
 /* NB: Ideally this belongs within common/environment,
  * however, that would introduce a circular dependency.
  */
-static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
+static bool populateAllowListFromEnvironment(IAllowListWriter &writer)
 {
     if (isContainerized())
         return false;
@@ -151,14 +151,20 @@ static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
     if (!conn->queryRoot()->hasProp("Software/DaliServerProcess"))
         return false;
 
-    // only ever expecting 1 DaliServerProcess and 1 WhiteList
-    const IPropertyTree *whiteListTree = conn->queryRoot()->queryPropTree("Software/DaliServerProcess[1]/WhiteList[1]");
+    // only ever expecting 1 DaliServerProcess and 1 AllowList
+    const IPropertyTree *allowListTree = conn->queryRoot()->queryPropTree("Software/DaliServerProcess[1]/AllowList[1]");
+    if (!allowListTree)
+    {
+        // deprecated, but for backward compatibility..
+        allowListTree = conn->queryRoot()->queryPropTree("Software/DaliServerProcess[1]/WhiteList[1]");
+    }
+
     bool enabled = true;
-    if (whiteListTree)
+    if (allowListTree)
     {
-        enabled = whiteListTree->getPropBool("@enabled", true); // on by default
-        // Default for now is to allow clients that send no role (legacy) to connect if their IP is whitelisted.
-        writer.setAllowAnonRoles(whiteListTree->getPropBool("@allowAnonRoles", true));
+        enabled = allowListTree->getPropBool("@enabled", true); // on by default
+        // Default for now is to allow clients that send no role (legacy) to connect if their IP is in allowlist.
+        writer.setAllowAnonRoles(allowListTree->getPropBool("@allowAnonRoles", true));
     }
 
     std::unordered_map<std::string, std::string> machineMap;
@@ -311,12 +317,12 @@ static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
         }
     }
 
-    if (whiteListTree)
+    if (allowListTree)
     {
-        Owned<IPropertyTreeIterator> whiteListIter = whiteListTree->getElements("Entry");
-        ForEach(*whiteListIter)
+        Owned<IPropertyTreeIterator> allowListIter = allowListTree->getElements("Entry");
+        ForEach(*allowListIter)
         {
-            const IPropertyTree &entry = whiteListIter->query();
+            const IPropertyTree &entry = allowListIter->query();
             StringArray hosts, roles;
             hosts.appendListUniq(entry.queryProp("@hosts"), ",");
             roles.appendListUniq(entry.queryProp("@roles"), ",");
@@ -607,8 +613,8 @@ int main(int argc, const char* argv[])
         unsigned short myport = epa.item(myrank).port;
         startMPServer(DCR_DaliServer, myport, true, true);
         Owned<IMPServer> mpServer = getMPServer();
-        Owned<IWhiteListHandler> whiteListHandler = createWhiteListHandler(populateWhiteListFromEnvironment, formatDaliRole);
-        mpServer->installWhiteListCallback(whiteListHandler);
+        Owned<IAllowListHandler> allowListHandler = createAllowListHandler(populateAllowListFromEnvironment, formatDaliRole);
+        mpServer->installAllowListCallback(allowListHandler);
 #ifndef _CONTAINERIZED
         setMsgLevel(fileMsgHandler, serverConfig->getPropInt("SDS/@msgLevel", 100));
 #endif

dockerfiles/buildall.sh

@@ -25,6 +25,7 @@ BUILD_TAG=$(git describe --exact-match --tags)  # The git tag for the images we
 BUILD_LABEL=${BUILD_TAG}                        # The docker hub label for all other components
 BUILD_USER=hpcc-systems                         # The github repo owner
 BUILD_TYPE=                                     # Set to Debug for a debug build, leave blank for default (RelWithDebInfo)
+USE_CPPUNIT=1
 
 # These values are set in a GitHub workflow build
 
@@ -38,6 +39,7 @@ if [[ -n ${INPUT_BUILDTYPE} ]] ; then
 else
   BUILD_LABEL=${BUILD_TAG}
   BUILD_TYPE=RelWithDebInfo
+  USE_CPPUNIT=0
 fi
 
 if [[ -n ${INPUT_USERNAME} ]] ; then
@@ -62,6 +64,7 @@ pushd $DIR 2>&1 > /dev/null
 
 . ../cmake_modules/parse_cmake.sh
 parse_cmake
+set_tag
 
 if [[ "$HPCC_MATURITY" = "release" ]] && [[ "$INPUT_LATEST" = "1" ]] ; then
   LATEST=1
@@ -70,7 +73,7 @@ fi
 build_image() {
   local name=$1
   local label=$2
-  [[ -z ${label} ]] && label=$BUILD_LABEL
+  [[ -z ${label} ]] && label=$HPCC_SHORT_TAG
 
   if ! docker pull hpccsystems/${name}:${label} ; then
     docker image build -t hpccsystems/${name}:${label} \
@@ -80,6 +83,7 @@ build_image() {
        --build-arg BUILD_LABEL=${BUILD_LABEL} \
        --build-arg BUILD_USER=${BUILD_USER} \
        --build-arg BUILD_TYPE=${BUILD_TYPE} \
+       --build-arg USE_CPPUNIT=${USE_CPPUNIT} \
        --build-arg BUILD_THREADS=${BUILD_THREADS} \
        ${name}/ 
     if [ "$LATEST" = "1" ] ; then

dockerfiles/cleanup.sh

@@ -22,8 +22,13 @@
 [[ "$1" == "-all" ]] && docker rm $(docker ps -q -f 'status=exited')
 docker rmi $(docker images -q -f "dangling=true")
 
-HEAD=$(git rev-parse --short HEAD)
-PREV=$(git describe --abbrev=0 --tags)
+if [[ "$1" == "-all" ]] ; then
+  HEAD=NOSUCHTHING
+  PREV=NOSUCHTHING
+else
+  HEAD=$(git rev-parse --short HEAD)
+  PREV=$(git describe --abbrev=0 --tags)
+fi
 for f in `docker images  --format "{{.Repository}}:{{.Tag}}" | grep hpccsystems/ | grep -v $HEAD | grep -v $PREV` ; do
   docker rmi $f
 done

dockerfiles/incr.sh

@@ -77,9 +77,9 @@ if [[ -z "$FORCE" ]] ; then
 
   # If not found above, look for latest tagged
   if [[ -z ${PREV} ]] ; then
-    PREV=$(git describe --abbrev=0 --tags)
+    PREV=$(git log --oneline --no-merges | grep "^[0-9a-f]* Split off " | head -1 | sed "s/.*Split off //" | head -1)-rc1
     if [[ ! "${BUILD_TYPE}" =~ "Release" ]] ; then
-      PREVE=${PREV}-${BUILD_TYPE}
+      PREV=${PREV}-${BUILD_TYPE}
     fi
     echo Finding image based on most recent git tag: ${PREV}
     INCR_DOCKER_REPO=hpccsystems

dockerfiles/platform-build-incremental/Dockerfile

@@ -54,6 +54,5 @@ RUN echo Building with $(cat ~/build_threads) threads
 RUN make -j$(cat ~/build_threads)
 
 USER root
-RUN cmake /hpcc-dev/HPCC-Platform -DUSE_PYTHON2=0
 RUN make -j$(cat ~hpcc/build_threads) install
 

dockerfiles/platform-build/Dockerfile

@@ -49,7 +49,8 @@ RUN mkdir build
 WORKDIR /hpcc-dev/build
 
 ARG BUILD_TYPE=RelWithDebInfo
-RUN cmake /hpcc-dev/HPCC-Platform -Wno-dev -DCONTAINERIZED=1 -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_PYTHON2=0 -DSUPPRESS_SPARK=1
+ARG USE_CPPUNIT=1
+RUN cmake /hpcc-dev/HPCC-Platform -Wno-dev -DCONTAINERIZED=1 -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_PYTHON2=0 -DSUPPRESS_SPARK=1 -DUSE_CPPUNIT=${USE_CPPUNIT}
 
 ARG BUILD_THREADS
 RUN if [ -n "${BUILD_THREADS}" ] ; then echo ${BUILD_THREADS} > ~/build_threads; else echo $(nproc) > ~/build_threads ; fi

docs/EN_US/ECLLanguageReference/ECLR_mods/Recrd-DATASET.xml

@@ -638,8 +638,12 @@ PtblE := DATASET('~Thor400::RTTEMP::TestFileEncrypted',
             role="bold">)</emphasis></entry>
 
             <entry><para>Optional. Maximum record length in the
-            <emphasis>file</emphasis>. If omitted, the default is
-            4096.</para></entry>
+            <emphasis>file</emphasis> in bytes. If omitted, the default is
+            4096. There is a hard limit of 10MB but that can be overridden
+            using <emphasis>#OPTION(maxCSVRowSizeMb,nn) </emphasis>where
+            <emphasis>nn</emphasis> is the maximum size in MB. The maximum
+            record size should be set as conservatively as possible.
+            </para></entry>
           </row>
 
           <row>

docs/EN_US/ECLLanguageReference/ECLR_mods/Templ-OPTION.xml

@@ -1478,6 +1478,66 @@
         </tgroup>
       </informaltable></para>
 
+    <para><emphasis role="bold">The following </emphasis><emphasis
+    role="bold">options</emphasis><emphasis role="bold"> are for the workunit
+    analyzer:</emphasis></para>
+
+    <informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="3">
+        <colspec colwidth="178.45pt" />
+
+        <colspec colwidth="78.60pt" />
+
+        <colspec />
+
+        <tbody>
+          <row>
+            <entry><emphasis>analyzeWorkunit</emphasis></entry>
+
+            <entry>Default: true</entry>
+
+            <entry>If set to FALSE, disables analysis of the workunit</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>analyzer_minInterestingTime</emphasis></entry>
+
+            <entry>Default: 1000</entry>
+
+            <entry>Analyze activities that exceed this minimum time to execute
+            (milliseconds)</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>analyzer_minInterestingCost </emphasis></entry>
+
+            <entry>Default: 30000</entry>
+
+            <entry>Report issues where the time penalty exceeds this value
+            (milliseconds)</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>analyzer_skewThreshold </emphasis></entry>
+
+            <entry>Default: 20</entry>
+
+            <entry>Report skew related issues that exceed this
+            threshold</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>analyzer_minRowsPerNode </emphasis></entry>
+
+            <entry>Default: 1000</entry>
+
+            <entry>Ignore activities that have this average number of rows per
+            node</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable>
+
     <para>Example:</para>
 
     <programlisting>  #OPTION('traceRowXml', TRUE);

docs/EN_US/HPCCSystemAdmin/SA-Mods/DaliLDAP.xml

@@ -99,26 +99,26 @@
   </sect2>
 
   <sect2 id="WhitelistInDali" role="brk">
-    <title>Whitelist in Dali</title>
+    <title>The AllowList in Dali</title>
 
     <para>The Dali server has the ability to restrict access to only those
     nodes that are associated with a role in the environment definition
-    (environment.xml) or explicitly added to the
-    <emphasis>Whitelist</emphasis> in Dali's configuration. The Whitelist is
-    implicitly populated with the server components and their roles as defined
-    in the environment. You can explicitly add additional nodes and their
-    approved role(s) to a supplementary Whitelist in the environment.xml file
-    as shown in the following example:</para>
+    (environment.xml) or explicitly added to the <emphasis>AllowList
+    </emphasis> in Dali's configuration. The AllowList is implicitly populated
+    with the server components and their roles as defined in the environment.
+    You can explicitly add additional nodes and their approved role(s) to a
+    supplementary AllowList in the environment.xml file as shown in the
+    following example:</para>
 
     <para><programlisting>&lt;Environment&gt;
 ...
  &lt;Software&gt;
  ...
    &lt;DaliServerProcess&gt;
-     &lt;WhiteList&gt;
+     &lt;AllowList&gt;
       &lt;Entry hosts="adminnode1,192.168.0.101" roles="DaliDiag,DaliAdmin"/&gt;
       &lt;Entry hosts="adminnode3" roles="DaliDiag"/&gt;
-     &lt;/WhiteList&gt;
+     &lt;/AllowList&gt;
    ...
    &lt;/DaliServerProcess&gt;
  ...
@@ -196,12 +196,12 @@
 SashaServer, RoxieMaster, and ThorMaster </programlisting></para>
 
     <para>These roles correspond to components in the environment, which
-    should always be implicitly whitelisted by inclusion in the environment.
+    should always be implicitly allowlisted by inclusion in the environment.
     If nodes in a foreign environment need to communicate with Dali, then
     those nodes should be explicitly added.</para>
 
     <para>In addition, the following administrative tool roles are
-    automatically whitelisted to run on the Dali server node:</para>
+    automatically allowlisted to run on the Dali server node:</para>
 
     <para><programlisting>Config, DaFsControl, DaliAdmin, DaliDiag, ScheduleAdmin, SwapNode, Testing, 
 TreeView, UpdateEnv, XRef, Monitoring</programlisting></para>
@@ -214,44 +214,44 @@ TreeView, UpdateEnv, XRef, Monitoring</programlisting></para>
     <para><programlisting>Config, DaFsControl, DaliAdmin, DaliDiag, SchedulerAdmin, SwapNode, Testing, 
 TreeView, UpdateEnv, XRef</programlisting></para>
 
-    <para>To disable the Whitelist feature entirely, you can add:</para>
+    <para>To disable the AllowList feature entirely, you can add:</para>
 
-    <para><programlisting>&lt;WhiteList enabled="false"/&gt;</programlisting><emphasis
+    <para><programlisting>&lt;AllowList enabled="false"/&gt;</programlisting><emphasis
     role="bold">This is not recommended for production
     environments.</emphasis></para>
 
-    <para>When enabling Dali whitelisting on a cluster where there are other
+    <para>When enabling the Dali AllowList on a cluster where there are other
     external environments interacting, it can be useful to initially disable
-    whitelisting and audit the external components that are trying to connect
+    the AllowList and audit the external components that are trying to connect
     with it by viewing the DaServer log files. Each client that would be
-    refused access if whitelisting were enabled, creates log entries similar
+    refused access if the AllowList was enabled, creates log entries similar
     to this:</para>
 
     <para><programlisting>00000017 Operator 2019-11-20 16:58:39.617 17056 17074 
-         "WhiteListing is disabled, ignoring: Access denied! 
-         [client ip=192.168.9.12, role=DaliDiag] not whitelisted"</programlisting></para>
+         "AllowList is disabled, ignoring: Access denied! 
+         [client ip=192.168.9.12, role=DaliDiag] not AllowListed"</programlisting></para>
 
     <para>To allow legacy components that do not provide a role to connect,
-    you can specify the allowAnonRoles option. The component MUST be on a
-    whitelisted IP.</para>
+    you can specify the allowAnonRoles option. The component MUST be on an
+    allowlisted IP.</para>
 
-    <para><programlisting>&lt;WhiteList allowAnonRoles="false"/&gt;</programlisting></para>
+    <para><programlisting>&lt;AllowList allowAnonRoles="false"/&gt;</programlisting></para>
 
     <sect3>
-      <title>Retrieve the Whitelist</title>
+      <title>Retrieve the AllowList</title>
 
-      <para>To retrieve the entire whitelist (implicit and explicit), use the
+      <para>To retrieve the entire AllowList (implicit and explicit), use the
       <emphasis role="bold">dalidiag</emphasis> command line tool</para>
 
       <para>(found in /opt/HPCCSystems/bin/)</para>
 
-      <para><programlisting>dalidiag &lt;dali-ip&gt; -whitelist</programlisting></para>
+      <para><programlisting>dalidiag &lt;dali-ip&gt; -AllowList</programlisting></para>
     </sect3>
 
     <sect3>
       <title>Update Dali without restarting</title>
 
-      <para>To update whitelist information in Dali without restarting, use
+      <para>To update AllowList information in Dali without restarting, use
       the <emphasis role="bold">updtdalienv</emphasis> command line
       tool</para>
 
@@ -262,32 +262,32 @@ TreeView, UpdateEnv, XRef</programlisting></para>
 
     <sect3 role="brk">
       <title>Use envmod to add or remove entries in the supplementary
-      Whitelist</title>
+      AllowList</title>
 
       <para>We recommend using the <emphasis role="bold">envmod</emphasis>
-      command line tool to add Whitelist entries to your environment.xml file.
+      command line tool to add AllowList entries to your environment.xml file.
       The envmod utility can be found in /opt/HPCCSystems/bin/.</para>
 
       <para>Use a template file such as the following example:</para>
 
       <para><programlisting>{
-    "name" : "AddWhiteList",
-    "description" : "Add whilelist to environment",
+    "name" : "AddAllowList",
+    "description" : "Add AllowList to environment",
     "type" : "modification",
     "operations" : [
         {
             "action" : "find",
-            "target_path" : "/Environment/Software/DaliServerProcess/WhiteList",
+            "target_path" : "/Environment/Software/DaliServerProcess/AllowList",
             "data" : {
                 "create_if_not_found" : true,
                 "save" : {
-                    "name" : "whiteListNodeId"
+                    "name" : "AllowListNodeId"
                 }
             }
         },
         {
             "action" : "create",
-            "target_nodeid" : "{{whiteListNodeId}}",
+            "target_nodeid" : "{{AllowListNodeId}}",
             "data" : {
                 "node_type" : "Entry",
                 "attributes" : [
@@ -304,7 +304,7 @@ TreeView, UpdateEnv, XRef</programlisting></para>
         },
 {
             "action" : "create",
-            "target_nodeid" : "{{whiteListNodeId}}",
+            "target_nodeid" : "{{AllowListNodeId}}",
             "data" : {
                 "node_type" : "Entry",
                 "attributes" : [
@@ -330,10 +330,10 @@ TreeView, UpdateEnv, XRef</programlisting></para>
       <para>Sample command line:</para>
 
       <programlisting>sudo /opt/HPCCSystems/bin/envmod \
-     -t myWhitelistTemplate.json \
+     -t myAllowListTemplate.json \
      -e /etc/HPCCSystems/source/environment.xml \
      -d /opt/HPCCSystems/componentfiles/configschema/xsd \
-     -o /etc/HPCCSystems/source/environmentWithWhitelist.xml</programlisting>
+     -o /etc/HPCCSystems/source/environmentWithAllowList.xml</programlisting>
 
       <para>The -t (or --template) parameter is the location of the
       template.</para>
@@ -357,7 +357,7 @@ TreeView, UpdateEnv, XRef</programlisting></para>
 
       <para><programlisting>{
     “action” : “delete”,
-    “target_path” : “/Environment/Software/DaliServerProcess/WhiteList/Entry[@hosts=’node3’]”
+    “target_path” : “/Environment/Software/DaliServerProcess/AllowList/Entry[@hosts=’node3’]”
 }
 </programlisting></para>
     </sect3>

ecl/ecl-bundle/ecl-bundle.cpp

@@ -1024,6 +1024,8 @@ protected:
         unsigned retCode = doPipeCommand(output, queryEclccPath(optVerbose), "--nologfile -showpaths", NULL);
         if (retCode == START_FAILURE)
             throw makeStringExceptionV(0, "FATAL: Could not locate eclcc command");
+        if (optVerbose)
+            printf("eclcc output:\n%s\n", output.str());
         extractValueFromEnvOutput(bundlePath, output, ECLCC_ECLBUNDLE_PATH);
         extractValueFromEnvOutput(hooksPath, output, HPCC_FILEHOOKS_PATH);
     }

ecl/eclagent/eclagent.cpp

@@ -1882,7 +1882,7 @@ void EclAgent::doProcess()
                 traceLevel = w->getDebugValueInt("traceLevel", 10);
             w->setTracingValue("EclAgentBuild", BUILD_TAG);
             if (agentTopology->hasProp("@name"))
-                w->addProcess("EclAgent", agentTopology->queryProp("@name"), GetCurrentProcessId(), logname.str());
+                w->addProcess("EclAgent", agentTopology->queryProp("@name"), GetCurrentProcessId(), 0, nullptr, false, logname.str());
 
             eclccCodeVersion = w->getCodeVersion();
             if (eclccCodeVersion == 0)
@@ -2013,8 +2013,8 @@ void EclAgent::doProcess()
         {
             if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true)))
             {
-                WuAnalyseOptions options;  // TODO: read options from configuration file
-                analyseWorkunit(w.get(), options);
+                IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions");
+                analyseWorkunit(w.get(), analyzerOptions);
             }
         }
         if(w->queryEventScheduledCount() > 0)

ecl/eclagent/eclgraph.cpp

@@ -1560,7 +1560,7 @@ void EclAgent::updateWULogfile()
             rlf.getRemotePath(logname.clear());
 
             Owned <IWorkUnit> w = updateWorkUnit();
-            w->addProcess("EclAgent", agentTopology->queryProp("@name"), GetCurrentProcessId(), logname.str());
+            w->addProcess("EclAgent", agentTopology->queryProp("@name"), GetCurrentProcessId(), 0, nullptr, false, logname.str());
         }
         else
         {

ecl/eclcc/eclcc.cpp

@@ -59,7 +59,7 @@
 #include "reservedwords.hpp"
 #include "eclcc.hpp"
 
-#ifndef CONTAINERIZED
+#ifndef _CONTAINERIZED
 #include "environment.hpp"
 #endif
 
@@ -852,6 +852,20 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 }
                 generator->setSaveGeneratedFiles(optSaveCpp);
 
+                if (optSaveQueryArchive && instance.wu && instance.archive)
+                {
+                    StringBuffer buf;
+                    toXML(instance.archive, buf);
+                    if (optWorkUnit)
+                    {
+                        Owned<IWUQuery> q = instance.wu->updateQuery();
+                        q->setQueryText(buf);
+                    }
+                    else
+                    {
+                        generator->addArchiveAsResource(buf);
+                    }
+                }
                 bool generateOk = generator->processQuery(instance.query, target);  // NB: May clear instance.query
                 instance.stats.cppSize = generator->getGeneratedSize();
                 if (generateOk && !optNoCompile)
@@ -1433,14 +1447,6 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     if (syntaxChecking || optGenerateMeta || optEvaluateResult)
         return;
 
-    if (optSaveQueryArchive && instance.wu && instance.archive)
-    {
-        Owned<IWUQuery> q = instance.wu->updateQuery();
-        StringBuffer buf;
-        toXML(instance.archive, buf);
-        q->setQueryText(buf);
-    }
-
     StringBuffer targetFilename;
     const char * outputFilename = instance.outputFilename;
     if (!outputFilename)
@@ -2323,7 +2329,7 @@ bool EclCC::checkDaliConnected() const
 unsigned EclCC::lookupClusterSize() const
 {
     CriticalBlock b(dfsCrit);  // Overkill at present but maybe one day codegen will start threading? If it does the stack is also iffy!
-#ifndef CONTAINERIZED
+#ifndef _CONTAINERIZED
     if (!optDFS || disconnectReported || !checkDaliConnected())
         return 0;
 #endif
@@ -2334,7 +2340,7 @@ unsigned EclCC::lookupClusterSize() const
         prevClusterSize = 0;
     else
     {
-#ifdef CONTAINERIZED
+#ifdef _CONTAINERIZED
         VStringBuffer xpath("queues[@name=\"%s\"]", cluster);
         IPropertyTree * queue = configuration->queryPropTree(xpath);
         if (queue)

ecl/eclccserver/eclccserver.cpp

@@ -430,7 +430,10 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 if (!workunit->getDebugValueBool("obfuscateOutput", false))
                 {
                     Owned<IWUQuery> query = workunit->updateQuery();
-                    query->setQueryText(eclQuery.s.str());
+                    if (getArchiveXMLFromFile(realdllfilename, wuXML.clear()))  // MORE - if what was submitted was an archive, this is probably pointless?
+                        query->setQueryText(wuXML.str());
+                    else
+                        query->setQueryText(eclQuery.s.str());
                 }
 
                 createUNCFilename(realdllfilename.str(), dllurl);

ecl/hql/hqlexpr.cpp

@@ -9193,6 +9193,18 @@ bool CHqlMergedScope::isPlugin() const
     return false;
 }
 
+bool CHqlMergedScope::isEquivalentScope(const IHqlScope & other) const
+{
+    if (this == &other)
+        return true;
+
+    ForEachItemIn(i, mergedScopes)
+    {
+        if (other.isEquivalentScope(mergedScopes.item(i)))
+            return true;
+    }
+    return false;
+}
 
 
 

ecl/hql/hqlexpr.hpp

@@ -1092,6 +1092,7 @@ interface IHqlScope : public IInterface
     virtual ISourcePath * querySourcePath() const = 0;
     virtual bool hasBaseClass(IHqlExpression * searchBase) = 0;
     virtual bool allBasesFullyBound() const = 0;
+    virtual bool isEquivalentScope(const IHqlScope & other) const = 0;
 
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols) = 0;
     virtual IHqlScope * queryConcreteScope() = 0;

ecl/hql/hqlexpr.ipp

@@ -1155,6 +1155,7 @@ public:
     virtual ISourcePath * querySourcePath() const override { throwUnexpected(); }
     virtual bool hasBaseClass(IHqlExpression * searchBase) override;
     virtual bool allBasesFullyBound() const override { return false; } // Assume the worst
+    virtual bool isEquivalentScope(const IHqlScope & other) const override { return this == &other; }
 
     virtual void ensureSymbolsDefined(HqlLookupContext & ctx) override { }
 
@@ -1217,6 +1218,7 @@ public:
     virtual const char * queryFullName() const override  { return fullName; }
     virtual IIdAtom * queryFullContainerId() const override { return containerId; }
     virtual ISourcePath * querySourcePath() const override { return text ? text->querySourcePath() : NULL; }
+    virtual bool isEquivalentScope(const IHqlScope & other) const override { return this == &other; }
 
     virtual void ensureSymbolsDefined(HqlLookupContext & ctx) override { }
     virtual bool isImplicit() const override { return false; }
@@ -1390,6 +1392,7 @@ public:
     virtual bool allBasesFullyBound() const override;
     virtual bool isImplicit() const override;
     virtual bool isPlugin() const override;
+    virtual bool isEquivalentScope(const IHqlScope & other) const override;
     virtual IHqlScope * queryConcreteScope() override { return this; }
 
 protected:
@@ -1614,6 +1617,7 @@ public:
     virtual ISourcePath * querySourcePath() const override { return NULL; }
     virtual bool hasBaseClass(IHqlExpression * searchBase) override;
     virtual bool allBasesFullyBound() const override { return false; }
+    virtual bool isEquivalentScope(const IHqlScope & other) const override { return this == &other; }
 
     virtual IHqlScope * clone(HqlExprArray & children, HqlExprArray & symbols) override;
     virtual IHqlScope * queryConcreteScope() override;

ecl/hql/hqlgram2.cpp

@@ -3786,7 +3786,7 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
                 else
                     reportError(ERR_OBJ_NOSUCHFIELD, errpos, "Object does not have a member named '%s'", str(searchName));
             }
-            else if ((modScope != containerScope) && !isExported(resolved))
+            else if (!isExported(resolved) && !modScope->isEquivalentScope(*containerScope))
                 reportError(HQLERR_CannotAccessShared, errpos, "Cannot access SHARED symbol '%s' in another module", str(searchName));
             return resolved.getClear();
         }

ecl/hql/hqlir.cpp

@@ -762,6 +762,7 @@ inline type_t getRequiredTypeCode(node_operator op)
     case no_type:
     case no_libraryscopeinstance:
     case no_forwardscope:
+    case no_mergedscope:
         return type_alias; // type is an alias of itself.
     }
     return type_none;
@@ -2037,6 +2038,7 @@ id_t ExpressionIRPlayer::doProcessExpr(IHqlExpression * expr)
         break;
     case no_virtualscope:
     case no_concretescope:
+    case no_mergedscope:
         {
             HqlExprArray scopeSymbols;
             expr->queryScope()->getSymbols(scopeSymbols);

ecl/hql/hqlutil.cpp

@@ -9118,7 +9118,7 @@ extern HQL_API bool allParametersHaveDefaults(IHqlExpression * function)
     return true;
 }
 
-extern HQL_API bool expandMissingDefaultsAsStoreds(HqlExprArray & args, IHqlExpression * function)
+extern HQL_API bool expandParametersAsStoreds(HqlExprArray & args, IHqlExpression * function)
 {
     assertex(function->isFunction());
     IHqlExpression * formals = queryFunctionParameters(function);
@@ -9128,21 +9128,16 @@ extern HQL_API bool expandMissingDefaultsAsStoreds(HqlExprArray & args, IHqlExpr
         ForEachChild(idx, formals)
         {
             IHqlExpression *formal = formals->queryChild(idx);
-            IHqlExpression * defvalue = queryDefaultValue(defaults, idx);
-            if (defvalue)
-            {
-                args.append(*LINK(defvalue));
-            }
-            else
-            {
-                OwnedHqlExpr nullValue = createNullExpr(formal->queryType());
-                OwnedHqlExpr storedName = createConstant(str(formal->queryName()));
-                OwnedHqlExpr stored = createValue(no_stored, makeVoidType(), storedName.getClear());
-                HqlExprArray colonArgs;
-                colonArgs.append(*LINK(nullValue));
-                colonArgs.append(*LINK(stored));
-                args.append(*createWrapper(no_colon, formal->queryType(), colonArgs));
-            }
+            Linked<IHqlExpression> defvalue = queryDefaultValue(defaults, idx);
+            if (!defvalue)
+                defvalue.setown(createNullExpr(formal->queryType()));
+
+            OwnedHqlExpr storedName = createConstant(str(formal->queryId()));
+            OwnedHqlExpr stored = createValue(no_stored, makeVoidType(), storedName.getClear());
+            HqlExprArray colonArgs;
+            colonArgs.append(*LINK(defvalue));
+            colonArgs.append(*LINK(stored));
+            args.append(*createWrapper(no_colon, formal->queryType(), colonArgs));
         }
     }
     catch (IException * e)
@@ -9759,7 +9754,7 @@ static IHqlExpression * transformAttributeToQuery(IHqlExpression * expr, HqlLook
         HqlExprArray actuals;
         if (!allParametersHaveDefaults(expr))
         {
-            if (!expandMissingDefaultsAsStoreds(actuals, expr))
+            if (!expandParametersAsStoreds(actuals, expr))
             {
                 //For each parameter that doesn't have a default, create a stored variable of the appropriate type
                 //with a null value as the default value, and use that.

ecl/hqlcpp/hqlecl.cpp

@@ -193,6 +193,7 @@ protected:
     void addCppName(const char * filename, unsigned minActivity, unsigned maxActivity);
     void addLibrariesToCompiler();
     void addWorkUnitAsResource();
+    void addArchiveAsResource(StringBuffer &buf);
     void calculateHash(IHqlExpression * expr);
     bool doCompile(ICppCompiler * compiler);
     void doExpand(HqlCppTranslator & translator);
@@ -608,6 +609,10 @@ void HqlDllGenerator::addWorkUnitAsResource()
     code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 
+void HqlDllGenerator::addArchiveAsResource(StringBuffer &buf)
+{
+    code->addCompressResource("ARCHIVE", buf.length(), buf.str(), nullptr, 1000);
+}
 
 void HqlDllGenerator::insertStandAloneCode()
 {

ecl/hqlcpp/hqlecl.hpp

@@ -49,6 +49,7 @@ public:
     virtual void addManifestsFromArchive(IPropertyTree *archive) = 0;
     virtual void addWebServiceInfo(IPropertyTree *wsinfo) = 0;
     virtual void setSaveGeneratedFiles(bool value) = 0;
+    virtual void addArchiveAsResource(StringBuffer &buf) = 0;
 };
 
 extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, ClusterType targetClusterType, ICodegenContextCallback * ctxCallback, bool checkForLocalFileUploads, bool okToAbort);

ecl/regress/issue24482.ecl

@@ -0,0 +1,24 @@
+<Archive build="internal_7.10.3-closedown0Debug"
+         eclVersion="7.11.0"
+         legacyImport="0"
+         legacyWhen="0">
+ <Query attributePath="B"/>
+ <Module key="" name="">
+  <Attribute key="b"
+             name="B"
+             sourcePath="/home/gavin/temp/24482/B.ecl"
+             ts="1595584384000000">
+   import $;
+EXPORT B := $.A;&#10;&#10;
+  </Attribute>
+  <Attribute key="a"
+             name="a"
+             sourcePath="/home/gavin/temp/24482/./A.ecl"
+             ts="1595584417000000">
+   SHARED A := 42;&#10;&#10;
+  </Attribute>
+ </Module>
+ <Option name="debugquery" value="1"/>
+ <Option name="savecpptempfiles" value="1"/>
+ <Option name="eclcc_compiler_version" value="7.11.0"/>
+</Archive>

ecl/regress/issue24485.eclxml

@@ -0,0 +1,27 @@
+<Archive>
+<!--
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+ <Module name="myModule">
+  <Attribute name="myAttr">
+rec := { unsigned age; };
+EXPORT myAttr(string myName, unsigned age = 99, SET OF unsigned ids = [], DATASET(rec) AGES) := FUNCTION
+    RETURN output ('Hello ' + myName + ', age ' + (string)age + ' (' + (string)count(ids) + ':' + (string)count(ages) + ')');
+END;
+    </Attribute>
+  </Module>
+ <Query attributePath="myModule.myAttr"/>
+</Archive>

esp/applications/CMakeLists.txt

@@ -16,3 +16,4 @@
 HPCC_ADD_SUBDIRECTORY (eclwatch)
 HPCC_ADD_SUBDIRECTORY (eclservices)
 HPCC_ADD_SUBDIRECTORY (eclqueries)
+HPCC_ADD_SUBDIRECTORY (sql2ecl)

esp/applications/sql2ecl/CMakeLists.txt

@@ -0,0 +1,28 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( ESP_APPLICATION_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
+)
+
+FOREACH( iFile ${ESP_APPLICATION_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/sql2ecl COMPONENT Runtime )
+ENDFOREACH ( iFile )

esp/applications/sql2ecl/application.yaml

@@ -0,0 +1,4 @@
+application:
+  name: sql2ecl
+  services:
+   - ws_sql

esp/applications/sql2ecl/directories.yaml

@@ -0,0 +1,11 @@
+directories:
+  log: "/var/log/[NAME]/[INST]"
+  run: "/var/lib/[NAME]/[INST]"
+  conf: "/etc/[NAME]/[INST]"
+  temp: "/var/lib/[NAME]/[INST]/temp"
+  data: "/var/lib/[NAME]/hpcc-data/[COMPONENT]"
+  data2: "/var/lib/[NAME]/hpcc-data2/[COMPONENT]"
+  data3: "/var/lib/[NAME]/hpcc-data3/[COMPONENT]"
+  mirror: "/var/lib/[NAME]/hpcc-mirror/[COMPONENT]"
+  query: "/var/lib/[NAME]/queries/[INST]"
+  lock: "/var/lock/[NAME]/[INST]"

esp/applications/sql2ecl/esp.yaml

@@ -0,0 +1,27 @@
+esp:
+   instance: sql2ecl
+   description: SQL2ECL Service
+   daliServers: dali
+   auth: ldap
+   tls: true
+   port: 8880
+   enableSEHMapping: true
+   httpConfigAccess: true
+   logLevel: 1
+   maxBacklogQueueSize: 200
+   portalurl: http://hpccsystems.com/download
+   logDir: "-"
+
+   tls_config:
+     certificate: /etc/HPCCSystems/certificates/{$instance}/server.crt
+     privatekey: /etc/HPCCSystems/certificates/{$instance}/private.key
+     passphrase: JHjdi6DDptMwYmJNTZsqjg==
+     cipherList:
+       verify:
+          enable: false
+          address_match: false
+          accept_selfsigned: true
+          ca_certificates:
+            - path: "ca.pem"
+          trusted_peers:
+            - anyone

esp/applications/sql2ecl/ldap.yaml

@@ -0,0 +1,25 @@
+ldap:
+   objname: ldapserver
+   description: LDAP server process
+   ldapProtocol: ldaps
+   authMethod: kerberos
+   localDomain: localdomain
+   ldapPort: 389
+   ldapSecurePort: 636
+   systemCommonName: hpcc_admin2
+   systemPassword: ""
+   systemUser: hpcc_admin2
+   adminGroupName: HPCCAdmin
+   maxConnections: 10
+   passwordExpirationWarningDays: 10
+   cacheTimeout: 5
+   ldapTimeoutSecs: 131
+   sharedCache: true
+   checkViewPermissions: ''
+   filesBasedn: ou=files,ou=ecl
+   groupsBasedn: ou=groups,ou=ecl
+   sudoersBasedn: ou=SUDOers
+   systemBasedn: cn=Users
+   usersBasedn: ou=users,ou=ecl
+   resourcesBasedn: ou=WsEcl,ou=EspServices,ou=ecl
+   workunitsBasedn: ou=workunits,ou=ecl

esp/applications/sql2ecl/ldap_authorization_map.yaml

@@ -0,0 +1,11 @@
+ldap:
+   root_access:
+      resource: WsSQLAccess
+      required: Read
+      description: Base access to EclWatch
+   resource_map:
+      ws_ecl:
+         Feature:
+         -  path: WsSQLAccess
+            resource: WsSQLAccess
+            description: Base access to sql2ecl services

esp/applications/sql2ecl/plugins.yaml

@@ -0,0 +1,9 @@
+protocol_plugins:
+  http_protocol: esphttp
+  secure_http_protocol: esphttp
+
+service_plugins:
+  ws_sql: ws_sql
+
+binding_plugins:
+  ws_sql: ws_sql

esp/logging/logginglib/logthread.cpp

@@ -27,6 +27,7 @@ const char* const PropMaxLogQueueLength = "MaxLogQueueLength";
 const char* const PropQueueSizeSignal = "QueueSizeSignal";
 const char* const PropMaxTriesRS = "MaxTriesRS";
 const char* const PropFailSafe = "FailSafe";
+const char* const PropDisableFailSafe = "DisableFailSafe";
 const char* const PropAckedFiles = "AckedFiles";
 const char* const PropDefaultAckedFiles = "AckedFiles";
 const char* const PropAckedLogRequests = "AckedLogRequests";
@@ -73,9 +74,14 @@ CLogThread::CLogThread(IPropertyTree* _cfg , const char* _service, const char* _
     maxLogQueueLength = _cfg->getPropInt(PropMaxLogQueueLength, MaxLogQueueLength);
     signalGrowingQueueAt = _cfg->getPropInt(PropQueueSizeSignal, QueueSizeSignal);
     maxLogRetries = _cfg->getPropInt(PropMaxTriesRS, DefaultMaxTriesRS);
-    ensureFailSafe = _cfg->getPropBool(PropFailSafe);
+    //For decoupled logging, the fail safe is not needed because the logging agent always
+    //picks up the logging requests from tank file.
+    ensureFailSafe = _cfg->getPropBool(PropFailSafe) && !_cfg->getPropBool(PropDisableFailSafe, false);
     if(ensureFailSafe)
+    {
         logFailSafe.setown(createFailSafeLogger(_cfg, _service, _agentName));
+        PROGLOG("FailSafe ensured for %s", agentName.get());
+    }
 
     time_t tNow;
     time(&tNow);

esp/logging/loggingmanager/loggingmanager.cpp

@@ -40,8 +40,9 @@ bool CLoggingManager::init(IPropertyTree* cfg, const char* service)
         return false;
     }
 
+    decoupledLogging = cfg->getPropBool("DecoupledLogging", false);
     oneTankFile = cfg->getPropBool("FailSafe", true);
-    if (oneTankFile)
+    if (oneTankFile || decoupledLogging)
     {
         logFailSafe.setown(createFailSafeLogger(cfg, service, cfg->queryProp("@name")));
         logContentFilter.readAllLogFilters(cfg);
@@ -227,7 +228,7 @@ bool CLoggingManager::updateLog(IEspContext* espContext, IEspUpdateLogRequestWra
         if (espContext)
             espContext->addTraceSummaryTimeStamp(LogMin, "LMgr:startQLog");
 
-        if (oneTankFile)
+        if (oneTankFile || decoupledLogging)
         {
             Owned<CLogRequestInFile> reqInFile = new CLogRequestInFile();
             if (!saveToTankFile(req, reqInFile))
@@ -245,12 +246,15 @@ bool CLoggingManager::updateLog(IEspContext* espContext, IEspUpdateLogRequestWra
             Owned<IEspUpdateLogRequest> logRequest = new CUpdateLogRequest("", "");
             logRequest->setOption(reqInFile->getOption());
             logRequest->setLogContent(logContent);
-            for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+            if (!decoupledLogging)
             {
-                IUpdateLogThread* loggingThread = loggingAgentThreads[x];
-                if (loggingThread->hasService(LGSTUpdateLOG))
+                for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
                 {
-                    loggingThread->queueLog(logRequest);
+                    IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+                    if (loggingThread->hasService(LGSTUpdateLOG))
+                    {
+                        loggingThread->queueLog(logRequest);
+                    }
                 }
             }
         }

esp/logging/loggingmanager/loggingmanager.hpp

@@ -76,7 +76,7 @@ class CLoggingManager : implements ILoggingManager, public CInterface
 {
     typedef std::vector<IUpdateLogThread*> LOGGING_AGENTTHREADS;
     LOGGING_AGENTTHREADS  loggingAgentThreads;
-    bool oneTankFile = false, initialized;
+    bool oneTankFile = false, decoupledLogging = false, initialized = false;
     Owned<ILogFailSafe> logFailSafe;
     CLogContentFilter logContentFilter;
 

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -2253,6 +2253,7 @@ bool CWsWorkunitsEx::getQueryFiles(IEspContext &context, const char* wuid, const
         if (!cw)
             return false;
 
+        double version = context.getClientVersion();
         StringArray superFileNames;
         Owned<IHpccPackageSet> ps = createPackageSet(process.str());
         Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(),
@@ -2266,8 +2267,10 @@ bool CWsWorkunitsEx::getQueryFiles(IEspContext &context, const char* wuid, const
             const char *lfn = rf.getLogicalName();
             if (lfn && *lfn)
             {
-                logicalFiles.append(lfn);
-                if (respSuperFiles && (rf.getFlags() & RefFileSuper))
+                bool isSuper = rf.getFlags() & RefFileSuper;
+                if (!isSuper || (version < 1.78))
+                    logicalFiles.append(lfn);
+                if (respSuperFiles && isSuper)
                     readSuperFiles(context, &rf, lfn, wufiles, respSuperFiles);
             }
         }

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -396,7 +396,7 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
     xpath.setf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/WUResultMaxSizeMB", process, service);
     unsigned wuResultMaxSizeMB = cfg->getPropInt(xpath);
     if (wuResultMaxSizeMB > 0)
-        wuResultMaxSize = wuResultMaxSizeMB * 1000000;
+        wuResultMaxSize = wuResultMaxSizeMB * 0x100000;
 
     xpath.setf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/ZAPEmail", process, service);
     IPropertyTree *zapEmail = cfg->queryPropTree(xpath.str());
@@ -2844,6 +2844,21 @@ INewResultSet* createFilteredResultSet(INewResultSet* result, IArrayOf<IConstNam
     return filter->create();
 }
 
+
+static bool isResultRequestSzTooBig(unsigned __int64 start, unsigned requestCount, unsigned __int64 resultSz, unsigned resultRows, unsigned __int64 limitSz)
+{
+    if (0 == requestCount)
+        return resultSz > limitSz;
+    else
+    {
+        if (start+requestCount > resultRows)
+            requestCount = resultRows-start;
+        unsigned __int64 avgRecSize = resultSz / resultRows;
+        unsigned __int64 estSize = requestCount * avgRecSize;
+        return estSize > limitSz;
+    }
+}
+
 void CWsWorkunitsEx::getWsWuResult(IEspContext &context, const char *wuid, const char *name, const char *logical, unsigned index, __int64 start,
     unsigned &count, __int64 &total, IStringVal &resname, bool bin, IArrayOf<IConstNamedValue> *filterBy, MemoryBuffer &mb,
     WUState &wuState, bool xsd)
@@ -2876,10 +2891,6 @@ void CWsWorkunitsEx::getWsWuResult(IEspContext &context, const char *wuid, const
     if (!result)
         throw MakeStringException(ECLWATCH_CANNOT_GET_WU_RESULT,"Cannot open the workunit result.");
 
-    if (result->getResultRawSize(nullptr, nullptr) > wuResultMaxSize)
-        throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result for %s. The size is bigger than %lld.",
-            wuid, wuResultMaxSize);
-
     if (!resname.length())
         result->getResultName(resname);
 
@@ -2894,7 +2905,29 @@ void CWsWorkunitsEx::getWsWuResult(IEspContext &context, const char *wuid, const
     else
         rs.setown(resultSetFactory->createNewResultSet(result, wuid));
     if (!filterBy || !filterBy->length())
+    {
+        unsigned __int64 resultSz;
+
+        if (0 == logicalName.length()) // could be a workunit owned file (OUTPUT, THOR)
+            result->getResultFilename(logicalName);
+        if (logicalName.length())
+        {
+            Owned<IDistributedFile> df = lookupLogicalName(context, logicalName.str());
+            if (!df)
+                throw makeStringExceptionV(ECLWATCH_FILE_NOT_EXIST, "Cannot find file %s.", logicalName.str());
+            resultSz = df->getDiskSize(true, false);
+        }
+        else
+            resultSz = result->getResultRawSize(nullptr, nullptr);
+
+        if (isResultRequestSzTooBig(start, count, resultSz, rs->getNumRows(), wuResultMaxSize))
+        {
+            throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result for %s. The size is bigger than %lld MB.",
+                                       wuid, wuResultMaxSize/0x100000);
+        }
+
         appendResultSet(mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns());
+    }
     else
     {
         Owned<INewResultSet> filteredResult = createFilteredResultSet(rs, filterBy);
@@ -3233,16 +3266,22 @@ void CWsWorkunitsEx::getFileResults(IEspContext &context, const char *logicalNam
     Owned<IDistributedFile> df = lookupLogicalName(context, logicalName);
     if (!df)
         throw makeStringExceptionV(ECLWATCH_FILE_NOT_EXIST, "Cannot find file %s.", logicalName);
-    if (df->getDiskSize(true, false) > wuResultMaxSize)
-        throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result from file %s. The size is bigger than %lld.",
-            logicalName, wuResultMaxSize);
 
     Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
     Owned<INewResultSet> result(resultSetFactory->createNewFileResultSet(df, cluster));
     if (!filterBy || !filterBy->length())
+    {
+        if (isResultRequestSzTooBig(start, count, df->getDiskSize(true, false), result->getNumRows(), wuResultMaxSize))
+        {
+            throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result from file %s. The size is bigger than %lld MB.",
+                                       logicalName, wuResultMaxSize/0x100000);
+        }
+    
         appendResultSet(buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), NULL);
+    }
     else
     {
+        // NB: this could be still be very big, appendResultSet should be changed to ensure filtered result doesn't grow bigger than wuResultMaxSize
         Owned<INewResultSet> filteredResult = createFilteredResultSet(result, filterBy);
         appendResultSet(buf, filteredResult, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), NULL);
     }

esp/src/eclwatch/GraphsWUWidget.js

@@ -49,6 +49,10 @@ define([
             if (params.Wuid) {
                 var context = this;
                 this.wu = ESPWorkunit.Get(params.Wuid);
+                this.wu.fetchActivities().then(function (_) {
+                    context._activitiesData = _;
+                    context.updateGrid();
+                });
 
                 this.timeline = new srcTimings.WUTimelineEx()
                     .target(this.id + "TimelinePane")
@@ -138,7 +142,7 @@ define([
                         return (hours < 10 ? "0" : "") + hours + ":" + (minutes < 10 ? "0" : "") + minutes + ":" + (seconds < 10 ? "0" : "") + seconds;
                     }
                 },
-                Type: { label: this.i18n.Type, width: 72, sortable: true }
+                activityCount: { label: this.i18n.Activities, width: 72, sortable: true }
             };
         },
 
@@ -154,10 +158,16 @@ define([
         refreshGrid: function (args) {
             this._timelineData = null;
             this._graphsData = null;
+            this._activitiesData = null;
 
             this.timeline.refresh();
 
             var context = this;
+            this.wu.fetchActivities().then(function (_) {
+                context._activitiesData = _;
+                context.updateGrid();
+            });
+
             this.wu.getInfo({
                 onGetTimers: function (timers) {
                     //  Required to calculate Graphs Total Time  ---
@@ -170,7 +180,7 @@ define([
         },
 
         updateGrid: function () {
-            if (this._timelineData && this._graphsData) {
+            if (this._timelineData && this._graphsData && this._activitiesData) {
                 var context = this;
                 this.store.setData(this._graphsData.map(function (row) {
                     var timelineData = context._timelineData[row.Name];
@@ -178,6 +188,10 @@ define([
                         row.WhenStarted = timelineData.started;
                         row.WhenFinished = timelineData.finished;
                     }
+                    const activities = context._activitiesData[row.Name];
+                    if (activities) {
+                        row.activityCount = activities.length;
+                    }
                     return row;
                 }));
                 this.grid.refresh();

esp/src/eclwatch/PackageMapValidateContentWidget.js

@@ -208,6 +208,8 @@ define([
                     }
                     context.validateButton.set("disabled", false);
                     return response;
+                } else {
+                    context.validateButton.set("disabled", false);
                 }
             }, function (err) {
                 context.showErrors(err);

esp/src/eclwatch/PackageMapValidateWidget.js

@@ -218,6 +218,8 @@ define([
                     }
                         context.validateButton.set("disabled", false);
                         return response;
+                } else {
+                    context.validateButton.set("disabled", false);
                 }
             }, function (err) {
                 context.showErrors(err);

esp/src/eclwatch/TimingPageWidget.js

@@ -163,6 +163,9 @@ define([
                                 case "activity":
                                 case "edge":
                                     return "<a href='#" + cell + "' class='dgrid-row-url'>" + cell + "</a>";
+                                case "function":
+                                    const activityScopeID = context._timings.activityScopeID(cell);
+                                    return "<a href='#" + cell + "' class='dgrid-row-url'>" + activityScopeID + "</a>" + cell.substring(activityScopeID.length);
                             }
                             return cell;
                         }
@@ -198,6 +201,11 @@ define([
                             SubGraphId = context._timings.subgraphID(row.name);
                             ActivityId = row.id;
                             break;
+                        case "function":
+                            GraphName = context._timings.graphID(row.name);
+                            SubGraphId = context._timings.subgraphID(row.name);
+                            ActivityId = context._timings.activityID(row.name);
+                            break;
                         case "edge":
                             GraphName = context._timings.graphID(row.name);
                             SubGraphId = context._timings.subgraphID(row.name);

esp/src/eclwatch/nls/fr/hpcc.js

@@ -467,7 +467,7 @@ define(
         noDataMessage: "...Zéro lignes...",
         Node: "Noeud",
         NodeGroup: "Groupe de nœuds",
-        NoErrorFound: "Aucune erreur trouvée \ n",
+        NoErrorFound: "Aucune erreur trouvée \n",
         NoFilterCriteriaSpecified: "Aucun critère de filtre spécifié.",
         None: "Aucun",
         NoPublishedSize: "Aucune taille publiée",
@@ -478,7 +478,7 @@ define(
         NothingSelected: "Rien a été sélectionné",
         NotInSuperfiles: "Pas dans les Superfichiers",
         NotSuspendedbyUser: "Non suspendu par l'utilisateur",
-        NoWarningFound: "Aucun avertissement trouvé \ n",
+        NoWarningFound: "Aucun avertissement trouvé \n",
         NumberofParts: "Nombre de parties",
         NumberofSlaves: "Nombre d'esclaves",
         OK: "Ok",
@@ -887,7 +887,7 @@ define(
         ValidateActivePackageMap: "Valider la carte du package actif",
         ValidatePackageContent: "Valider le contenu du package",
         ValidatePackageMap: "Valider la carte des packages",
-        ValidateResult: "===== Valider le résultat ===== \ n \ n",
+        ValidateResult: "===== Valider le résultat ===== \n\n",
         ValidateResultHere: "(Résultat de validation)",
         Validating: "En cours de valider",
         Value: "Valeur",

esp/src/package-lock.json

@@ -44,37 +44,37 @@
       "integrity": "sha512-fxfMSBMX3tlIbKUdtGKxqB1fyrH6gVrX39Gsv3y8lRYKUqlgDt3UMqQyGnR1bQMa2B8aGnhLZokZgg8vT0Le+A=="
     },
     "@hpcc-js/api": {
-      "version": "2.8.19",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.8.19.tgz",
-      "integrity": "sha512-JqfqrPF5Cv0jTmNb0mnoAqCgJfWLniKseFnqzFKU0Efk+389/3orsrQZW9lAKclKe0W1CRwM/S1VqpqAIrsEhw==",
+      "version": "2.8.32",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.8.32.tgz",
+      "integrity": "sha512-h31do5IMVv2E+YAg6RH6HcJxLXyMdeTDxMDerThSGHd8kTWVEKe7kBv29jL5zdikBqTDkFjXfx1zDLd31Up1hA==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0"
+        "@hpcc-js/common": "^2.37.0"
       }
     },
     "@hpcc-js/chart": {
-      "version": "2.31.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.31.0.tgz",
-      "integrity": "sha512-kEX98VEYlZUEsSacmwoDzTLrEogQPtq7WFDh3QeDMRT+5HrscvnLSvwRt6zESvXKJR/Uu4vcWgfC7p2qKPRu3A==",
+      "version": "2.44.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.44.0.tgz",
+      "integrity": "sha512-IB1Ge9ikOsssLp2xP0lolLU1ax2i5ch+7jDR1YkVXWVMCO+kYoJ7hXz5ugD6eR8D9hoACFwc3sY7hbvLVmcBvA==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/util": "^2.13.0"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/codemirror": {
-      "version": "2.15.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.15.0.tgz",
-      "integrity": "sha512-YYGSy79LoZIGbL6SP1SbFhfKfMAyuI4K7C+Vh6+PMdVj7FAPBKr+foQPazcZfoXxWcp8dMpM3Kos9MdrKYEcoQ==",
+      "version": "2.27.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.27.0.tgz",
+      "integrity": "sha512-fFg99jQ1fsp4Jul2U2WNpLDKEDR+PVScEhrXi/H4JIU/H1ISPpWlBnRj6NdegUGW8RPz/wY2rLFsI7f37GOjhQ==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0"
+        "@hpcc-js/common": "^2.37.0"
       }
     },
     "@hpcc-js/common": {
-      "version": "2.25.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.25.0.tgz",
-      "integrity": "sha512-ZcfmN7IgkHebSkzGxL+KrGCCLHiLpxZqyW6JCO0l+nuwtHmeaScxPPUuJE1pFIx0diGoyfYrskRX280OROh/bg==",
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.37.0.tgz",
+      "integrity": "sha512-oZ73IwE5Lxvo8I3sjYXNfMuXmyHnPDhZslrVAN14GetTsqyhlUSHsGM6SCxBdYrhwr4tsLUweZOXzRP+fbwK/g==",
       "requires": {
-        "@hpcc-js/util": "^2.13.0",
+        "@hpcc-js/util": "^2.24.0",
         "@types/d3-array": "1.2.6",
         "@types/d3-brush": "1.0.10",
         "@types/d3-collection": "1.0.8",
@@ -93,12 +93,12 @@
       }
     },
     "@hpcc-js/comms": {
-      "version": "2.13.12",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.13.12.tgz",
-      "integrity": "sha512-KT0EH5xvS706CtjKGWGmg0qI9CE/Xc+hLnRnEz34TEcAuRsE2Czi8pqhA0WVamwnYvsta/93QFdHvqqFMo835A==",
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.22.0.tgz",
+      "integrity": "sha512-GmMzJ5YuR0qgbA+2QOKewQHVsCZFYf630ky0z1n2mIrqxwaz+tz+qwA/m39wB3xUGcrVnMIq/ug/HgHAHibTYw==",
       "requires": {
-        "@hpcc-js/ddl-shim": "^2.17.8",
-        "@hpcc-js/util": "^2.13.0",
+        "@hpcc-js/ddl-shim": "^2.17.14",
+        "@hpcc-js/util": "^2.24.0",
         "node-fetch": "2.3.0",
         "safe-buffer": "5.1.2",
         "tmp": "0.0.33",
@@ -106,82 +106,82 @@
       }
     },
     "@hpcc-js/ddl-shim": {
-      "version": "2.17.8",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/ddl-shim/-/ddl-shim-2.17.8.tgz",
-      "integrity": "sha512-MDwLMsSJV6pDlGKguSoP7Xoy4ly92I9NgjcH2N65oQhi5gTYlCb3zmIr47gAXHyyjvxOWvC3kC2GrZUaVMp+Lw==",
+      "version": "2.17.14",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/ddl-shim/-/ddl-shim-2.17.14.tgz",
+      "integrity": "sha512-mNZ2tNsTWOPTDOQKasL9AENM5OH8Lc3rZLnhIbpr1YJqF4JYea5LUX/87dYeOa7U6jQxpl1XlJ8+GEwgbBwS6w==",
       "requires": {
         "ajv": "6.10.0"
       }
     },
     "@hpcc-js/dgrid": {
-      "version": "2.8.17",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.8.17.tgz",
-      "integrity": "sha512-IyfyjRLGOjeRYQ25xCOvOD3PKtRqCgbtZdig+eqZ42bR9cGedJy+SSCH5PZAfEU4wa1DY06v06Fpqz623EAjYQ==",
+      "version": "2.8.30",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.8.30.tgz",
+      "integrity": "sha512-cHpyPiMFDDA6nGF0vMe4RHjgKXKDMybFpwSe8CQU2mZkihaAOFl6xCkeStg6TTDmSxQ8HO3cJ92cKqkA0v3t5Q==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/ddl-shim": "^2.17.8",
-        "@hpcc-js/dgrid-shim": "^2.11.13",
-        "@hpcc-js/util": "^2.13.0"
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/ddl-shim": "^2.17.14",
+        "@hpcc-js/dgrid-shim": "^2.11.21",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/dgrid-shim": {
-      "version": "2.11.13",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.11.13.tgz",
-      "integrity": "sha512-JZbxSk/2HJq9TaHC4lWUWfri1uly7zpUsh0Bj/dYa055nEVgGIr6RMXjOufkqm881egmjww/RLtKl7FQ1GEHdg=="
+      "version": "2.11.21",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.11.21.tgz",
+      "integrity": "sha512-zPcJ5wyZvbIkPZBJEpjPeTWu7HfE9JvoqkTXTRCkEF8rcg8LEwCyw4gyxKZt2lCDJfUtpGco7bZgANTVYfRIxA=="
     },
     "@hpcc-js/eclwatch": {
-      "version": "2.8.29",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.8.29.tgz",
-      "integrity": "sha512-Jd0V8GY0GjRMW24lPPLwPbROIqBukeuo8eYYmaMr65m/xbAeVmociuM74um/z/wMCkbB+b/rpnCSc2+t9Ty/rQ==",
-      "requires": {
-        "@hpcc-js/codemirror": "^2.15.0",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/comms": "^2.13.12",
-        "@hpcc-js/dgrid": "^2.8.17",
-        "@hpcc-js/graph": "^2.20.0",
-        "@hpcc-js/layout": "^2.16.28",
-        "@hpcc-js/phosphor": "^2.14.11",
-        "@hpcc-js/timeline": "^2.7.25",
-        "@hpcc-js/tree": "^2.12.10",
-        "@hpcc-js/util": "^2.13.0"
+      "version": "2.8.48",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.8.48.tgz",
+      "integrity": "sha512-Yp7thXM6NTCYq4Ya+xf5rsaKyy9YjjS4sMxlIFjCTD3o67uAmEHbYbRg9yH5v4bMq0HOG1lPqedwqyuX+Fl7aA==",
+      "requires": {
+        "@hpcc-js/codemirror": "^2.27.0",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/comms": "^2.22.0",
+        "@hpcc-js/dgrid": "^2.8.30",
+        "@hpcc-js/graph": "^2.36.0",
+        "@hpcc-js/layout": "^2.16.42",
+        "@hpcc-js/phosphor": "^2.14.25",
+        "@hpcc-js/timeline": "^2.10.0",
+        "@hpcc-js/tree": "^2.12.23",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/graph": {
-      "version": "2.20.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.20.0.tgz",
-      "integrity": "sha512-jCr222nhBwB+BVWqXV0rSuEomZ/fmnhLNjIIrmb8UPWCxvtUz60cIQA8GjS4/CC5qc4VUHSTN0P2O8SFDow6xw==",
+      "version": "2.36.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.36.0.tgz",
+      "integrity": "sha512-95kES8udG9Y0cmFOoUkr6pV8P1JogMpBG6ibRstH2j5JOzdRip5/xfp7jxLLIZ4oDv2xeExpUjC46wbvd6VdpQ==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/react": "^2.9.11",
-        "@hpcc-js/util": "^2.13.0",
-        "@hpcc-js/wasm": "0.3.12"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/html": "^2.10.0",
+        "@hpcc-js/react": "^2.18.0",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/html": {
-      "version": "2.8.13",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.8.13.tgz",
-      "integrity": "sha512-SAUz2DqdGIvPNpO6+8HVf3oVmOkw6kycA6BUsIAQ1gQBLZkpUqmc3FZhCwL0mvvE/zjpyXxYJHprS5/sGmt6tA==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.10.0.tgz",
+      "integrity": "sha512-d1bjZwwvjMVd59qHB5RlBmnOoMK0iU13JyMkDFvH/XXTB1q4N/WR72CcqCgV5BQzKNndAevPk2Uy4HAbeGH0Gg==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/preact-shim": "^2.13.6",
-        "@hpcc-js/util": "^2.13.0"
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/preact-shim": "^2.13.12",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/layout": {
-      "version": "2.16.28",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.16.28.tgz",
-      "integrity": "sha512-nfP8K0y1lXx8BCK4tEeA5Wx3vpUuXke/5MI34Ybxf59IX21j3vaRzIn0BJ5f+C9C9s5gnzaE2eyPHgWkulwmkg==",
+      "version": "2.16.42",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.16.42.tgz",
+      "integrity": "sha512-CVL7NOsn60AlSvpmVtI3YAJskaNJfA7LSTlRj+mkhzt9xElBqDoj4JAjamVEdlFYH+tkGFDB2F6KO5UUtadk9A==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/dgrid": "^2.8.17"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/dgrid": "^2.8.30"
       }
     },
     "@hpcc-js/leaflet-shim": {
-      "version": "2.1.7",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/leaflet-shim/-/leaflet-shim-2.1.7.tgz",
-      "integrity": "sha512-pAGyXJRH9HZH4b5BLgSRzQMxttde8FZtgL5djeI0M7sNiGfNdLLbnj7m6dv20pyayq6PPtYCgQHFlUKfU87Dlw==",
+      "version": "2.1.13",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/leaflet-shim/-/leaflet-shim-2.1.13.tgz",
+      "integrity": "sha512-jvRWg75vRR3FQkQ9sWIeCilmOnMRVQPLOBJ8tWD92xut8pTwQkfX2ref8bhla2uljI3mIL9Pa2sjzU8WwENdzg==",
       "requires": {
         "@types/leaflet": "1.5.1",
         "leaflet": "1.5.1",
@@ -189,44 +189,44 @@
       }
     },
     "@hpcc-js/map": {
-      "version": "2.14.32",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.14.32.tgz",
-      "integrity": "sha512-G4W9Beeat4WCFR5RcK5dvvt4t5z/suQAz/C/b1m9wYZZvEQ4BUWqrij375OBMcC5LjfkpUVJWTWqzpnp1BhqzA==",
+      "version": "2.26.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.26.0.tgz",
+      "integrity": "sha512-+fy40t3apqVluOemLI9p31g5IiPxRoRqpe+cLi9IKUGM/WanSDVpmhfG2bzV3wiI654Ste17C8pDdpUzI7P04Q==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/graph": "^2.20.0",
-        "@hpcc-js/layout": "^2.16.28",
-        "@hpcc-js/leaflet-shim": "^2.1.7",
-        "@hpcc-js/other": "^2.13.31",
-        "@hpcc-js/util": "^2.13.0"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/graph": "^2.36.0",
+        "@hpcc-js/layout": "^2.16.42",
+        "@hpcc-js/leaflet-shim": "^2.1.13",
+        "@hpcc-js/other": "^2.13.45",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/other": {
-      "version": "2.13.31",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.13.31.tgz",
-      "integrity": "sha512-YK8VHWDkg9Xcd/aTEtKYDZ8BuS2nrlYVPWHAIwAjEgJCmgXNaT57wqk2vbxvfac9LeRKSpNYzCZsz9bP5QCUBQ==",
+      "version": "2.13.45",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.13.45.tgz",
+      "integrity": "sha512-Fi7bF3gG2tZN6wrOVQfzl4NZrZ64PjehB5Bk4Hys3ONd7km2kPvdCQfEW40Z754BsVyg/DTgncUV2vJHxyCmzg==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/layout": "^2.16.28"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/layout": "^2.16.42"
       }
     },
     "@hpcc-js/phosphor": {
-      "version": "2.14.11",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.14.11.tgz",
-      "integrity": "sha512-kDO/iOVzGlOXR+U6X9uqPdgeTXmBZR3sAdoTstdUP9NyYAeqKI9lFM0O775YfKGoOeqNeIz99wRfPrGVz3iXeA==",
+      "version": "2.14.25",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.14.25.tgz",
+      "integrity": "sha512-vecX0pPDaZWDl7FS6Z8Obtr+pSlpw+Gqsks6i6bqk54VUz5ZqdPDOS+h3aFBBwU+DewthH2ieLUBR37KxDIXcw==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/other": "^2.13.31",
-        "@hpcc-js/phosphor-shim": "^2.11.10",
-        "@hpcc-js/util": "^2.13.0"
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/other": "^2.13.45",
+        "@hpcc-js/phosphor-shim": "^2.11.16",
+        "@hpcc-js/util": "^2.24.0"
       }
     },
     "@hpcc-js/phosphor-shim": {
-      "version": "2.11.10",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.11.10.tgz",
-      "integrity": "sha512-q26AtTylDyBSbTjV4OrJXhInGAzCMcLsNJ4A7pseN9uQldQbHUbo/5vmia6pqyWJQxMJuUy9cfnJWM1ysnzugA==",
+      "version": "2.11.16",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.11.16.tgz",
+      "integrity": "sha512-MvFCpdwhR+KyI3F1k6y2qHdWdIjqBhbFef04sRYIkZwAS/b94PAStt10dVDP72fxNUfTMZ1TLwDjg6SaJLbyow==",
       "requires": {
         "@phosphor/algorithm": "1.1.2",
         "@phosphor/commands": "1.5.0",
@@ -235,50 +235,55 @@
       }
     },
     "@hpcc-js/preact-shim": {
-      "version": "2.13.6",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.13.6.tgz",
-      "integrity": "sha512-GdnLfmjlrsi16RZnJiQ/Bl2UvnbBprNRtDY5p6t5TUerUtVgKcxEOblkF1CFEA/T3SO0KAn2EedHHusoJOr9sQ==",
+      "version": "2.13.12",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.13.12.tgz",
+      "integrity": "sha512-XB5eboxjHTgtcSCQOlGSDU1sEeMuK2cSLcRa4jN6HmyFzfsvXhMa6kTIviUYcgENcXxBfU0Kf5/Ev6I9HwfiJA==",
       "requires": {
         "preact": "10.1.1"
       }
     },
     "@hpcc-js/react": {
-      "version": "2.9.11",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.9.11.tgz",
-      "integrity": "sha512-f3fAH3yUsFxhx4N0pjJqX8RJ3F10KNJTEIvi9vLbWHyNqo2VLZtlcMS5YA2GfeSaNcRo0vYKBRBKLEZ1dr7Qvg==",
+      "version": "2.18.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.18.0.tgz",
+      "integrity": "sha512-BsJiIQtzySq8082bnItuuw4DjstRIYj3Qp58Y6a1ClUyj5UnAIvzzOFtzU8cKpD/egZMUYpRjMIcd/xPCicIlg==",
       "requires": {
-        "@hpcc-js/common": "^2.25.0",
-        "@hpcc-js/preact-shim": "^2.13.6"
+        "@hpcc-js/common": "^2.37.0",
+        "@hpcc-js/preact-shim": "^2.13.12"
       }
     },
     "@hpcc-js/timeline": {
-      "version": "2.7.25",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.7.25.tgz",
-      "integrity": "sha512-HI10lR9kS/aZNxvRgncq9oKIxOQfIHo7+e4BWt2dT65KCl+i+cAjOxOwNUJ8FCk9VaPZL6z9mJ0Nz9q55PbDjA==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.10.0.tgz",
+      "integrity": "sha512-6+UXCny+JlBAGia/6ia4dQ+zq7iuDDRzuTiWnl71wO4sZ87hIuVDAqittmKIp6ImvwtAjR93vUgv9UU26z33LA==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/chart": "^2.31.0",
-        "@hpcc-js/common": "^2.25.0"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/chart": "^2.44.0",
+        "@hpcc-js/common": "^2.37.0"
       }
     },
     "@hpcc-js/tree": {
-      "version": "2.12.10",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.12.10.tgz",
-      "integrity": "sha512-rqMEN+Imm5P1QnPj8KmkGsHFWkgkSnbQmykG/3/qrvsIk0LGtzu4edBOK9M1rBQavUMnCqu2BaV/N9WeORuxTQ==",
+      "version": "2.12.23",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.12.23.tgz",
+      "integrity": "sha512-rzpm0aOIaPHlJdo96icBKwCZ85O2N/tGRXNdMKmutOyjBzIBngzY+zB7y7W02p5hEtU5wq7y14MVhctVbR0B1w==",
       "requires": {
-        "@hpcc-js/api": "^2.8.19",
-        "@hpcc-js/common": "^2.25.0"
+        "@hpcc-js/api": "^2.8.32",
+        "@hpcc-js/common": "^2.37.0"
       }
     },
     "@hpcc-js/util": {
-      "version": "2.13.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.13.0.tgz",
-      "integrity": "sha512-KknlyQzKwrjXBbhCB4Y7J0Hep3BzaCOWNU5H5KtEXrdlnX8digoQCywACZmDCpbNFP5neD2kFotKOwzgqgf9Ag=="
-    },
-    "@hpcc-js/wasm": {
-      "version": "0.3.12",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-0.3.12.tgz",
-      "integrity": "sha512-oqY3j7VBqVpnkVDlcuHsJ6QG4e9r1t9AtqHFeSp1j1ERA8R8prnen+7gk/GmSecfBL4Zn4uK7D3C/C10egX7Sw=="
+      "version": "2.24.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.24.0.tgz",
+      "integrity": "sha512-z8d/lhMcaelHm6cvvt3dC5n/UaUOpeim04MEGtSZk6l7V9TkC1ps2zZ2Q8V1nh3PGo0FzQx4LdzBMvtMBP7tYA==",
+      "requires": {
+        "tslib": "1.10.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "1.10.0",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
+          "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ=="
+        }
+      }
     },
     "@koa/cors": {
       "version": "3.0.0",

esp/src/package.json

@@ -30,20 +30,20 @@
   },
   "main": "src/stub.js",
   "dependencies": {
-    "@hpcc-js/chart": "2.31.0",
-    "@hpcc-js/codemirror": "2.15.0",
-    "@hpcc-js/common": "2.25.0",
-    "@hpcc-js/comms": "2.13.12",
-    "@hpcc-js/eclwatch": "2.8.29",
-    "@hpcc-js/graph": "2.20.0",
-    "@hpcc-js/html": "2.8.13",
-    "@hpcc-js/layout": "2.16.28",
-    "@hpcc-js/map": "2.14.32",
-    "@hpcc-js/other": "2.13.31",
-    "@hpcc-js/phosphor": "2.14.11",
-    "@hpcc-js/react": "2.9.11",
-    "@hpcc-js/tree": "2.12.10",
-    "@hpcc-js/util": "2.13.0",
+    "@hpcc-js/chart": "2.44.0",
+    "@hpcc-js/codemirror": "2.27.0",
+    "@hpcc-js/common": "2.37.0",
+    "@hpcc-js/comms": "2.22.0",
+    "@hpcc-js/eclwatch": "2.8.48",
+    "@hpcc-js/graph": "2.36.0",
+    "@hpcc-js/html": "2.10.0",
+    "@hpcc-js/layout": "2.16.42",
+    "@hpcc-js/map": "2.26.0",
+    "@hpcc-js/other": "2.13.45",
+    "@hpcc-js/phosphor": "2.14.25",
+    "@hpcc-js/react": "2.18.0",
+    "@hpcc-js/tree": "2.12.23",
+    "@hpcc-js/util": "2.24.0",
     "@material-ui/core": "4.8.3",
     "@material-ui/icons": "4.9.1",
     "@material-ui/lab": "4.0.0-alpha.47",

esp/src/src/DiskUsage.ts

@@ -20,6 +20,8 @@ interface CompontentT {
 
 type DetailsT = GetTargetClusterUsageEx.TargetClusterUsage | CompontentT;
 
+const calcPct = (val, tot) => Math.round((val / tot) * 100);
+
 export class Summary extends FlexGrid {
 
     private _loadingMsg;
@@ -118,7 +120,8 @@ export class Summary extends FlexGrid {
                                 details.rowCount++;
                                 details.inUse += du.InUse;
                                 details.total += du.Total;
-                                details.max = details.max < du.InUse ? du.InUse : details.max;
+                                const usage = calcPct(du.InUse, du.Total);
+                                details.max = details.max < usage ? usage : details.max;
                             });
                         });
                     });
@@ -136,9 +139,9 @@ export class Summary extends FlexGrid {
                         const totalMean = details.total / details.rowCount;
                         const inUseMean = details.inUse / details.rowCount;
                         this._usage[key].gauge
-                            .value((details.max / totalMean))
+                            .value(details.max / 100)
                             .valueDescription(nlsHPCC.Max)
-                            .tickValue((inUseMean / totalMean))
+                            .tickValue(inUseMean / totalMean)
                             .tickValueDescription(nlsHPCC.Mean)
                             .tooltip(key)
                             ;
@@ -182,7 +185,7 @@ export class Details extends Table {
         this._details.ComponentUsages.forEach(cu => {
             cu.MachineUsages.forEach(mu => {
                 mu.DiskUsages.forEach(du => {
-                    data.push([du.PercentUsed, cu.Name, du.Name, mu.NetAddress !== "." ? mu.NetAddress : mu.Name, du.Path, du.InUse, du.Total]);
+                    data.push([calcPct(du.InUse, du.Total), cu.Name, du.Name, mu.NetAddress !== "." ? mu.NetAddress : mu.Name, du.Path, du.InUse, du.Total]);
                 });
             });
         });

esp/src/src/ESPWorkunit.ts

@@ -863,6 +863,49 @@ const Workunit = declare([ESPUtil.Singleton], {  // jshint ignore:line
             onGetTimers: onFetchTimers
         });
     },
+    fetchActivities() {
+        return (this._hpccWU as HPCCWorkunit).fetchDetails({
+            ScopeFilter: {
+                MaxDepth: 999999,
+                ScopeTypes: ["graph"]
+            },
+            ScopeOptions: {
+                IncludeMatchedScopesInResults: true,
+                IncludeScope: true,
+                IncludeId: true,
+                IncludeScopeType: true
+            },
+            PropertyOptions: {
+                IncludeName: false,
+                IncludeRawValue: false,
+                IncludeFormatted: false,
+                IncludeMeasure: false,
+                IncludeCreator: false,
+                IncludeCreatorType: false
+            },
+            NestedFilter: {
+                Depth: 999999,
+                ScopeTypes: ["activity"]
+            },
+            PropertiesToReturn: {
+                AllStatistics: false,
+                AllAttributes: false,
+                AllHints: false,
+                AllProperties: false,
+                AllScopes: true
+            }
+        }).then(response => {
+            const retVal = {};
+            response.forEach(scope => {
+                const graphID = scope.ScopeName.split(":")[1];
+                if (!retVal[graphID]) {
+                    retVal[graphID] = [];
+                }
+                retVal[graphID].push(scope.Id);
+            });
+            return retVal;
+        });
+    },
     fetchGraphs(onFetchGraphs) {
         if (this.graphs && this.graphs.length) {
             onFetchGraphs(this.graphs);

esp/src/src/Timings.ts

@@ -147,12 +147,34 @@ export class Timings {
         return retVal;
     }
 
+    activityID(id: string): string {
+        let retVal: string;
+        this.walkScopeName(id, partialID => {
+            retVal = this._activityLookup[partialID];
+            if (retVal) return true;
+        });
+        return retVal;
+    }
+
+    activityScopeID(id: string): string {
+        let retVal: string;
+        this.walkScopeName(id, partialID => {
+            retVal = this._activityLookup[partialID];
+            if (retVal) {
+                retVal = partialID;
+                return true;
+            }
+        });
+        return retVal;
+    }
+
     _rawColumns = {};
     _scopeFilter: string = "";
     _metricSelectLabel: string = "";
     _metricSelectValues: string[] = ["TimeElapsed"];
     _graphLookup: { [id: string]: string } = {};
     _subgraphLookup: { [id: string]: string } = {};
+    _activityLookup: { [id: string]: string } = {};
     fetchDetailsNormalizedPromise;
     refresh(force: boolean = false) {
         if (force) {
@@ -268,9 +290,11 @@ export class Timings {
             this._rawColumns = response.columns;
             this._graphLookup = {};
             this._subgraphLookup = {};
+            this._activityLookup = {};
             const rawData = response.data.filter(row => {
                 if (row.type === "graph") this._graphLookup[row.name] = row.id;
                 if (row.type === "subgraph") this._subgraphLookup[row.name] = row.id;
+                if (row.type === "activity") this._activityLookup[row.name] = row.id;
                 if (!row.id) return false;
                 if (this._scopeFilter && row.name !== this._scopeFilter && row.name.indexOf(`${this._scopeFilter}:`) !== 0) return false;
                 if (this._metricSelectValues.every(m => row[m] === undefined)) return false;

esp/src/src/WUScopeController.ts

@@ -358,7 +358,7 @@ export abstract class WUScopeControllerBase<ISubgraph, IVertex, IEdge, IGraphDat
         });
     }
 
-    calcTooltip(scope: BaseScope, parentScope?: BaseScope, term = "") {
+    calcTooltipTable(scope: BaseScope, parentScope?: BaseScope, term = "") {
         const [findScope, findTerm] = this.splitTerm(term);
 
         function highlightText(key: string, _text: any) {
@@ -389,11 +389,21 @@ export abstract class WUScopeControllerBase<ISubgraph, IVertex, IEdge, IGraphDat
             }
         }
 
-        return `<div class="eclwatch_WUGraph_Tooltip" style="max-width:480px">
-            <h4 align="center">${highlightText("Label", label)}</h4>
+        const funcTooltips: string[] = [];
+        scope.children().forEach(row => {
+            funcTooltips.push(this.calcTooltipTable(row));
+        })
+
+        return `<h4 align="center">${highlightText("Label", label)}</h4>
             <table>
                 ${rows.join("")}
-            </table>
+            </table>${funcTooltips.length ? `<br>${funcTooltips.join("<br>")}` : ""}`;
+    }
+
+    calcTooltip(scope: BaseScope, parentScope?: BaseScope, term = "") {
+
+        return `<div class="eclwatch_WUGraph_Tooltip" style="max-width:480px">
+            ${this.calcTooltipTable(scope, parentScope, term)}
         </div>`;
     }
 

fs/dafsclient/rmtfile.cpp

@@ -2031,6 +2031,8 @@ void installDefaultFileHooks(IPropertyTree * config)
     {
         getPackageFolder(hookdir);
         addPathSepChar(hookdir).append("filehooks");
+        if (!checkFileExists(hookdir))
+            return;
     }
     installFileHooks(hookdir);
 }

helm/hpcc/values.yaml

@@ -144,6 +144,14 @@ esp:
   public: true
   servicePort: 8002
 
+- name: sql2ecl
+  application: sql2ecl
+  auth: none
+  tls: off
+  replicas: 1
+  public: true
+  servicePort: 8510
+
 roxie:
 - name: roxie-cluster
   disabled: false

initfiles/componentfiles/configxml/@temp/esp_service_DynamicESDL.xsl

@@ -72,6 +72,9 @@
                 </xsl:if>
 
                 <LoggingManager name="{$managerNode/@name}">
+                    <xsl:if test="string($managerNode/@DecoupledLogging) != ''">
+                        <DecoupledLogging><xsl:value-of select="$managerNode/@DecoupledLogging"/></DecoupledLogging>
+                    </xsl:if>
                     <xsl:if test="string($managerNode/@FailSafe) != ''">
                         <FailSafe><xsl:value-of select="$managerNode/@FailSafe"/></FailSafe>
                     </xsl:if>
@@ -91,6 +94,7 @@
                         <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
                         <xsl:variable name="espLoggingAgentNode" select="/Environment/Software/ESPLoggingAgent[@name=$agentName]"/>
                         <xsl:variable name="wsLogServiceESPAgentNode" select="/Environment/Software/WsLogServiceESPAgent[@name=$agentName]"/>
+                        <xsl:variable name="disableFailSafe" select="$managerNode/@DecoupledLogging"/>
                         <xsl:choose>
                             <xsl:when test="($espLoggingAgentNode)">
                                 <xsl:call-template name="ESPLoggingAgent">
@@ -101,6 +105,7 @@
                             <xsl:when test="($wsLogServiceESPAgentNode)">
                                 <xsl:call-template name="WsLogServiceESPAgent">
                                     <xsl:with-param name="agentName" select="$agentName"/>
+                                    <xsl:with-param name="disableFailSafe" select="$disableFailSafe"/>
                                     <xsl:with-param name="agentNode" select="$wsLogServiceESPAgentNode"/>
                                 </xsl:call-template>
                             </xsl:when>

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -27,6 +27,7 @@ xmlns:set="http://exslt.org/sets">
     <xsl:template name="WsLogServiceESPAgent" type="DefaultLoggingAgent">
         <xsl:param name="agentName"/>
         <xsl:param name="agentNode"/>
+        <xsl:param name="disableFailSafe"/>
         <xsl:if test="not($agentNode)">
             <xsl:message terminate="yes">An WsLogService ESP Logging Agent <xsl:value-of select="$agentName"/> is undefined!</xsl:message>
         </xsl:if>
@@ -56,6 +57,9 @@ xmlns:set="http://exslt.org/sets">
             <xsl:call-template name="EspLoggingAgentBasic">
                 <xsl:with-param name="agentNode" select="$agentNode"/>
             </xsl:call-template>
+            <xsl:if test="string($disableFailSafe) != ''">
+                <DisableFailSafe><xsl:value-of select="$disableFailSafe"/></DisableFailSafe>
+            </xsl:if>
             <xsl:if test="string($agentNode/@TransactionSeedType) != ''">
                 <TransactionSeedType><xsl:value-of select="$agentNode/@TransactionSeedType"/></TransactionSeedType>
             </xsl:if>

initfiles/componentfiles/configxml/agentexec.xsl

@@ -111,6 +111,7 @@
           <xsl:value-of select="@httpGlobalIdHeader"/>
         </xsl:attribute>
       </xsl:if>
+      <xsl:copy-of select="analyzerOptions"/>
       <xsl:copy-of select="/Environment/Software/Directories"/>  
 
     </agentexec>

initfiles/componentfiles/configxml/loggingmanager.xsd

@@ -43,6 +43,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="DecoupledLogging" type="xs:boolean" use="optional" default="false">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Enable Decoupled Logging functionality.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="FailSafe" type="xs:boolean" use="optional" default="true">
         <xs:annotation>
           <xs:appinfo>

initfiles/componentfiles/configxml/sasha.xsd

@@ -196,6 +196,15 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+
+            <xs:attribute name="sashaUser" type="xs:string" use="optional" default="">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Specifies the Sasha Username used for authorization.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+
         </xs:complexType>
     </xs:element>
     <xs:attributeGroup name="Archiver">

initfiles/componentfiles/configxml/sasha.xsl

@@ -54,7 +54,11 @@
             <xsl:attribute name="logDir">
                <xsl:value-of select="translate(@logDir, $oldPathChars, $newPathChars)"/>
             </xsl:attribute>
-            
+
+            <xsl:attribute name="sashaUser">
+                <xsl:value-of select="@sashaUser"/>
+            </xsl:attribute>
+
             <xsl:attribute name="enableSNMP">
                <xsl:call-template name="outputBool">
                   <xsl:with-param name="val" select="@enableSNMP"/>
@@ -83,7 +87,7 @@
                    <xsl:value-of select="@LDSroot"/>
                 </xsl:attribute>
             </xsl:element>
-            
+
             <xsl:element name="Archiver">
                 <xsl:element name="WorkUnits"> 
                     <xsl:attribute name="limit">

plugins/cassandra/CMakeLists.txt

@@ -70,6 +70,9 @@ if(USE_CASSANDRA)
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy")
     endif ()
   endif()
+  if (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
+    remove_definitions (-fsanitize=undefined -fno-sanitize=alignment -fsanitize-trap=undefined)
+  endif()
   set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cpp-driver/cmake/modules/")
   set(CASS_ROOT_DIR "${PROJECT_SOURCE_DIR}/cpp-driver")
   add_subdirectory(cpp-driver ${PROJECT_BINARY_DIR}/cassandra)

plugins/cassandra/cassandraembed.cpp

@@ -349,7 +349,7 @@ CassandraPrepared *CassandraClusterSession::prepareStatement(const char *query,
     preparedCache.setValue(query, cached); // NOTE - this links parameter
     return cached.getClear();
 }
-CassandraStatementInfo *CassandraClusterSession::createStatementInfo(const char *script, unsigned numParams, CassBatchType batchMode, unsigned pageSize) const
+CassandraStatementInfo *CassandraClusterSession::createStatementInfo(const char *script, unsigned numParams, OptionalCassBatchType batchMode, unsigned pageSize) const
 {
     Owned<CassandraPrepared> prepared = prepareStatement(script, false); // We could make tracing selectable
     return new CassandraStatementInfo(session, prepared, numParams, batchMode, pageSize, semaphore, maxRetries);
@@ -499,7 +499,7 @@ void CassandraRetryingFuture::signaller(CassFuture *future, void *data)
 
 //----------------------
 
-CassandraStatementInfo::CassandraStatementInfo(CassandraSession *_session, CassandraPrepared *_prepared, unsigned _numBindings, CassBatchType _batchMode, unsigned pageSize, Semaphore *_semaphore, unsigned _maxRetries)
+CassandraStatementInfo::CassandraStatementInfo(CassandraSession *_session, CassandraPrepared *_prepared, unsigned _numBindings, OptionalCassBatchType _batchMode, unsigned pageSize, Semaphore *_semaphore, unsigned _maxRetries)
     : session(_session), prepared(_prepared), numBindings(_numBindings), batchMode(_batchMode), semaphore(_semaphore), maxRetries(_maxRetries)
 {
     assertex(prepared && *prepared);
@@ -545,8 +545,8 @@ bool CassandraStatementInfo::next()
 }
 void CassandraStatementInfo::startStream()
 {
-    if (batchMode != (CassBatchType) -1)
-        batch.setown(new CassandraBatch(batchMode));
+    if (batchMode != OptionalCassBatchType::nobatch)
+        batch.setown(new CassandraBatch((CassBatchType) batchMode));
     statement.setown(new CassandraStatement(cass_prepared_bind(*prepared)));
     inBatch = true;
 }
@@ -1320,7 +1320,7 @@ class CassandraEmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>
 {
 public:
     CassandraEmbedFunctionContext(const IContextLogger &_logctx, const IThorActivityContext *_activityCtx, unsigned _flags, const char *options)
-      : logctx(_logctx), activityCtx(_activityCtx), flags(_flags), nextParam(0), numParams(0), batchMode((CassBatchType) -1), pageSize(0)
+      : logctx(_logctx), activityCtx(_activityCtx), flags(_flags), nextParam(0), numParams(0), batchMode(OptionalCassBatchType::nobatch), pageSize(0)
     {
         StringArray opts;
         opts.appendList(options, ",");
@@ -1332,11 +1332,11 @@ public:
             {
                 const char *val=opt+6;
                 if (stricmp(val, "LOGGED")==0)
-                    batchMode = CASS_BATCH_TYPE_LOGGED;
+                    batchMode = OptionalCassBatchType::logged;
                 else if (stricmp(val, "UNLOGGED")==0)
-                    batchMode = CASS_BATCH_TYPE_UNLOGGED;
+                    batchMode = OptionalCassBatchType::unlogged;
                 else if (stricmp(val, "COUNTER")==0)
-                    batchMode = CASS_BATCH_TYPE_COUNTER;
+                    batchMode = OptionalCassBatchType::counter;
                 opts.remove(idx);
             }
             else if (strnicmp(opt, "pagesize=", 9)==0)
@@ -1939,7 +1939,7 @@ protected:
     unsigned nextParam;
     unsigned numParams;
     StringBuffer queryString;
-    CassBatchType batchMode;
+    OptionalCassBatchType batchMode;
     unsigned pageSize;
 };
 

plugins/cassandra/cassandraembed.hpp

@@ -26,6 +26,14 @@
 
 namespace cassandraembed {
 
+enum class OptionalCassBatchType
+{
+    logged = CASS_BATCH_TYPE_LOGGED,
+    unlogged = CASS_BATCH_TYPE_UNLOGGED,
+    counter = CASS_BATCH_TYPE_COUNTER,
+    nobatch = 255
+};
+
 __declspec(noreturn) extern void UNSUPPORTED(const char *feature) __attribute__((noreturn));
 __declspec(noreturn) extern void failx(const char *msg, ...) __attribute__((format(printf, 1, 2), noreturn));
 __declspec(noreturn) extern void fail(const char *msg) __attribute__((noreturn));
@@ -120,7 +128,7 @@ public:
     void connect();
     void disconnect();
     CassandraPrepared *prepareStatement(const char *query, bool trace) const;
-    CassandraStatementInfo *createStatementInfo(const char *script, unsigned numParams, CassBatchType batchMode, unsigned pageSize) const;
+    CassandraStatementInfo *createStatementInfo(const char *script, unsigned numParams, OptionalCassBatchType batchMode, unsigned pageSize) const;
     void executeAsync(CIArrayOf<CassandraStatement> &batch, const char *what) const;
 
 private:
@@ -432,7 +440,7 @@ private:
 class CASSANDRAEMBED_API CassandraStatementInfo : public CInterface
 {
 public:
-    CassandraStatementInfo(CassandraSession *_session, CassandraPrepared *_prepared, unsigned _numBindings, CassBatchType _batchMode, unsigned pageSize, Semaphore *_semaphore, unsigned _maxRetries);
+    CassandraStatementInfo(CassandraSession *_session, CassandraPrepared *_prepared, unsigned _numBindings, OptionalCassBatchType _batchMode, unsigned pageSize, Semaphore *_semaphore, unsigned _maxRetries);
     ~CassandraStatementInfo();
     void stop();
     bool next();
@@ -469,7 +477,7 @@ protected:
     Semaphore *semaphore;
     unsigned maxRetries;
     bool inBatch;
-    CassBatchType batchMode;
+    OptionalCassBatchType batchMode;
 };
 
 extern CASSANDRAEMBED_API bool getBooleanResult(const RtlFieldInfo *field, const CassValue *value);

plugins/py3embed/py3embed.cpp

@@ -48,6 +48,10 @@
   #define Py_TPFLAGS_HAVE_ITER 0
 #endif
 
+#if PY_MINOR_VERSION < 7
+  #define USE_CUSTOM_NAMEDTUPLES
+#endif
+
 static const char * compatibleVersions[] = {
     "Python3.x Embed Helper 1.0.0",
     NULL };
@@ -146,6 +150,105 @@ static void checkPythonError()
     }
 }
 
+#ifdef USE_CUSTOM_NAMEDTUPLES
+// In Python3.6 we can't use the standard namedtuple if we have > 255 params. So we need to use our own based on similar techniques.
+// The code below is copied from the Python 3.6 collections module source, with minor modifications:
+// - pass in the initial tuple values as a tuple rather than as separate parameters, to avoid the 256 param limit
+// - remove some options we were not using
+
+static constexpr const char * _py36code = R"!!(
+import sys as _sys
+from keyword import iskeyword as _iskeyword
+_class_template = """\
+from builtins import property as _property, tuple as _tuple
+from operator import itemgetter as _itemgetter
+from collections import OrderedDict
+class {typename}(tuple):
+    '{typename}({arg_list})'
+    __slots__ = ()
+    _fields = {field_names!r}
+    def __new__(_cls, t):
+        'Create new instance of {typename}({arg_list})'
+        return _tuple.__new__(_cls, t)
+    @classmethod
+    def _make(cls, iterable, new=tuple.__new__, len=len):
+        'Make a new {typename} object from a sequence or iterable'
+        result = new(cls, iterable)
+        if len(result) != {num_fields:d}:
+            raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result))
+        return result
+    def _replace(_self, **kwds):
+        'Return a new {typename} object replacing specified fields with new values'
+        result = _self._make(map(kwds.pop, {field_names!r}, _self))
+        if kwds:
+            raise ValueError('Got unexpected field names: %r' % list(kwds))
+        return result
+    def __repr__(self):
+        'Return a nicely formatted representation string'
+        return self.__class__.__name__ + '({repr_fmt})' % self
+    def _asdict(self):
+        'Return a new OrderedDict which maps field names to their values.'
+        return OrderedDict(zip(self._fields, self))
+    def __getnewargs__(self):
+        'Return self as a plain tuple.  Used by copy and pickle.'
+        return tuple(self)
+{field_defs}
+"""
+
+_repr_template = '"{name}=%r";'
+
+_field_template = '''\
+    {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}')
+'''
+
+def namedtuple(typename, field_names):
+
+    # Validate the field names, and replace any invalid ones with a valid name.
+    field_names = field_names.replace(',', ' ').split()
+    field_names = list(map(str, field_names))
+    typename = str(typename)
+    for index, name in enumerate(field_names):
+        if (not name.isidentifier()
+            or _iskeyword(name)
+            or name.startswith('_')):
+            field_names[index] = '_%d' % index
+
+    # Fill-in the class template
+    class_definition = _class_template.format(
+        typename = typename,
+        field_names = tuple(field_names),
+        num_fields = len(field_names),
+        arg_list = repr(tuple(field_names)).replace("'", "")[1:-1],
+        repr_fmt = ', '.join(_repr_template.format(name=name)
+                             for name in field_names),
+        field_defs = '\n'.join(_field_template.format(index=index, name=name)
+                               for index, name in enumerate(field_names))
+    )
+
+    # Execute the template string in a temporary namespace and support
+    # tracing utilities by setting a value for frame.f_globals['__name__']
+    namespace = dict(__name__='namedtuple_%s' % typename)
+    exec(class_definition, namespace)
+    result = namespace[typename]
+
+    # For pickling to work, the __module__ variable needs to be set to the frame
+    # where the named tuple is created.  Bypass this step in environments where
+    # sys._getframe is not defined (Jython for example) or sys._getframe is not
+    # defined for arguments greater than 0 (IronPython), or where the user has
+    # specified a particular module.
+    module = None
+    try:
+        module = _sys._getframe(1).f_globals.get('__name__', '__main__')
+    except (AttributeError, ValueError):
+        pass
+    if module is not None:
+        result.__module__ = module
+
+    return result
+)!!";
+
+#endif
+
 // The Python Global Interpreter Lock (GIL) won't know about C++-created threads, so we need to
 // call PyGILState_Ensure() and PyGILState_Release at the start and end of every function.
 // Wrapping them in a class like this ensures that the release always happens even if
@@ -298,6 +401,13 @@ public:
             initialized = false;
             return;
         }
+        StringBuffer py2modname;
+        if  (findLoadedModule(py2modname, "libpy2embed."))
+        {
+            initialized = false;
+            multiPython = true;
+            return;
+        }
 #endif
 #ifndef _WIN32
         // We need to ensure all symbols in the python3.x so are loaded - due to bugs in some distro's python installations
@@ -305,8 +415,8 @@ public:
         // Therefore on systems where both are present, do NOT do this - people using centos systems that suffer from issue
         // https://bugs.centos.org/view.php?id=6063 will need to choose which version of python plugin to install but not both
 
-        StringBuffer modname, py2modname;
-        if  (findLoadedModule(modname, "libpython3.") && !findLoadedModule(py2modname, "libpython2."))
+        StringBuffer modname;
+        if  (findLoadedModule(modname, "libpython3."))
             pythonLibrary = dlopen(modname.str(), RTLD_NOW|RTLD_GLOBAL);
 #endif
         // Initialize the Python Interpreter
@@ -346,6 +456,13 @@ public:
 
         }
     }
+    void checkInitialized()
+    {
+        if (multiPython)
+            rtlFail(0, "Python3 not initialized as Python2 already loaded");
+        else if (!initialized)
+            rtlFail(0, "Python3 not initialized");
+    }
     bool isInitialized()
     {
         return initialized;
@@ -394,10 +511,19 @@ public:
         if (!namedtuple)
         {
             namedtupleTypes.setown(PyDict_New());
+#ifdef USE_CUSTOM_NAMEDTUPLES
+            OwnedPyObject temp_namespace(PyDict_New());
+            PyDict_SetItemString(temp_namespace, "__builtins__",  PyEval_GetBuiltins());  // required for import to work
+            checkPythonError();
+            OwnedPyObject ran = PyRun_String(_py36code, Py_file_input, temp_namespace, temp_namespace);
+            checkPythonError();
+            namedtuple.set(PyDict_GetItemString(temp_namespace, "namedtuple"));   // NOTE - returns borrowed reference
+#else
             OwnedPyObject pName = PyUnicode_FromString("collections");
             OwnedPyObject collections = PyImport_Import(pName);
             checkPythonError();
             namedtuple.setown(PyObject_GetAttrString(collections, "namedtuple"));
+#endif
             checkPythonError();
             assertex(PyCallable_Check(namedtuple));
         }
@@ -566,6 +692,7 @@ protected:
     }
     PyThreadState *tstate = nullptr;
     bool initialized = false;
+    bool multiPython = false;
     bool skipPythonCleanup = true; // Tensorflow seems to often lockup in the python cleanup process.
     HINSTANCE pythonLibrary = 0;
     OwnedPyObject namedtuple;      // collections.namedtuple
@@ -605,8 +732,7 @@ static void checkThreadContext()
 {
     if (!threadContext)
     {
-        if (!globalState.isInitialized())
-            rtlFail(0, "Python not initialized");
+        globalState.checkInitialized();
         threadContext = new PythonThreadContext;
         addThreadTermFunc(releaseContext);
     }
@@ -748,7 +874,6 @@ static void getStringResult(const RtlFieldInfo *field, PyObject *obj, size32_t &
             size_t lenBytes = PyBytes_Size(obj);
             rtlStrToStrX(chars, result, lenBytes, text);
         }
-        return;
     }
     typeError("string", field);
 }
@@ -1197,7 +1322,13 @@ public:
     PyObject *getTuple(const RtlTypeInfo *type)
     {
         OwnedPyObject mynamedtupletype = sharedCtx ? sharedCtx->getNamedTupleType(type) : globalState.getNamedTupleType(type);
+#ifdef USE_CUSTOM_NAMEDTUPLES
+        OwnedPyObject argsTuple = PyTuple_New(1);
+        Py_INCREF(args);
+        PyTuple_SET_ITEM((PyTupleObject *) argsTuple.get(), 0, args);
+#else
         OwnedPyObject argsTuple = PyList_AsTuple(args);
+#endif
         OwnedPyObject mynamedtuple = PyObject_CallObject(mynamedtupletype, argsTuple);  // Creates a namedtuple from the supplied tuple
         checkPythonError();
         return mynamedtuple.getClear();
@@ -1968,20 +2099,23 @@ extern DECL_EXPORT IEmbedContext* getEmbedContext()
 extern DECL_EXPORT void syntaxCheck(size32_t & __lenResult, char * & __result, const char *funcname, size32_t charsBody, const char * body, const char *argNames, const char *compilerOptions, const char *persistOptions)
 {
     StringBuffer result;
-    // NOTE - compilation of a script does not actually resolve imports - so the fact that the manifest is not on the path does not matter
-    // This does mean that many errors cannot be caught until runtime, but that's Python for you...
-    try
-    {
-        checkThreadContext();
-        Owned<Python3xEmbedScriptContext> ctx = new Python3xEmbedScriptContext(threadContext, nullptr);
-        ctx->setargs(argNames);
-        ctx->compileEmbeddedScript(charsBody, body);
-    }
-    catch (IException *E)
+    if (globalState.isInitialized())
     {
-        StringBuffer msg;
-        result.append(E->errorMessage(msg));
-        E->Release();
+        // NOTE - compilation of a script does not actually resolve imports - so the fact that the manifest is not on the path does not matter
+        // This does mean that many errors cannot be caught until runtime, but that's Python for you...
+        try
+        {
+            checkThreadContext();
+            Owned<Python3xEmbedScriptContext> ctx = new Python3xEmbedScriptContext(threadContext, nullptr);
+            ctx->setargs(argNames);
+            ctx->compileEmbeddedScript(charsBody, body);
+        }
+        catch (IException *E)
+        {
+            StringBuffer msg;
+            result.append(E->errorMessage(msg));
+            E->Release();
+        }
     }
     __lenResult = result.length();
     __result = result.detach();

plugins/pyembed/pyembed.cpp

@@ -292,6 +292,13 @@ public:
             initialized = false;
             return;
         }
+        StringBuffer py3modname;
+        if  (findLoadedModule(py3modname, "libpy3embed."))
+        {
+            initialized = false;
+            multiPython = true;
+            return;
+        }
 #endif
 #ifndef _WIN32
         // We need to ensure all symbols in the python2.x so are loaded - due to bugs in some distro's python installations
@@ -299,8 +306,8 @@ public:
         // Therefore on systems where both are present, do NOT do this - people using centos systems that suffer from issue
         // https://bugs.centos.org/view.php?id=6063 will need to choose which version of python plugin to install but not both
 
-        StringBuffer modname, py3modname;
-        if  (findLoadedModule(modname, "libpython2.") && !findLoadedModule(py3modname, "libpython3."))
+        StringBuffer modname;
+        if  (findLoadedModule(modname, "libpython2."))
             pythonLibrary = dlopen(modname.str(), RTLD_NOW|RTLD_GLOBAL);
 #endif
         // Initialize the Python Interpreter
@@ -340,6 +347,13 @@ public:
 
         }
     }
+    void checkInitialized()
+    {
+        if (multiPython)
+            rtlFail(0, "Python2 not initialized as Python3 already loaded");
+        else if (!initialized)
+            rtlFail(0, "Python2 not initialized");
+    }
     bool isInitialized()
     {
         return initialized;
@@ -562,6 +576,7 @@ protected:
     }
     PyThreadState *tstate = nullptr;
     bool initialized = false;
+    bool multiPython = false;
     bool skipPythonCleanup = true; // Tensorflow seems to often lockup in the python cleanup process.
     HINSTANCE pythonLibrary = 0;
     OwnedPyObject namedtuple;      // collections.namedtuple
@@ -602,8 +617,7 @@ static void checkThreadContext()
 {
     if (!threadContext)
     {
-        if (!globalState.isInitialized())
-            rtlFail(0, "Python not initialized");
+        globalState.checkInitialized();
         threadContext = new PythonThreadContext;
         addThreadTermFunc(releaseContext);
     }
@@ -1961,20 +1975,23 @@ extern DECL_EXPORT IEmbedContext* getEmbedContext()
 extern DECL_EXPORT void syntaxCheck(size32_t & __lenResult, char * & __result, const char *funcname, size32_t charsBody, const char * body, const char *argNames, const char *compilerOptions, const char *persistOptions)
 {
     StringBuffer result;
-    // NOTE - compilation of a script does not actually resolve imports - so the fact that the manifest is not on the path does not matter
-    // This does mean that many errors cannot be caught until runtime, but that's Python for you...
-    try
-    {
-        checkThreadContext();
-        Owned<Python27EmbedScriptContext> ctx = new Python27EmbedScriptContext(threadContext, nullptr);
-        ctx->setargs(argNames);
-        ctx->compileEmbeddedScript(charsBody, body);
-    }
-    catch (IException *E)
+    if (globalState.isInitialized())
     {
-        StringBuffer msg;
-        result.append(E->errorMessage(msg));
-        E->Release();
+        // NOTE - compilation of a script does not actually resolve imports - so the fact that the manifest is not on the path does not matter
+        // This does mean that many errors cannot be caught until runtime, but that's Python for you...
+        try
+        {
+            checkThreadContext();
+            Owned<Python27EmbedScriptContext> ctx = new Python27EmbedScriptContext(threadContext, nullptr);
+            ctx->setargs(argNames);
+            ctx->compileEmbeddedScript(charsBody, body);
+        }
+        catch (IException *E)
+        {
+            StringBuffer msg;
+            result.append(E->errorMessage(msg));
+            E->Release();
+        }
     }
     __lenResult = result.length();
     __result = result.detach();

system/jlib/jlz4.cpp

@@ -19,6 +19,7 @@
 #include "jfcmp.hpp"
 #include "jlz4.hpp"
 #include "lz4.h"
+#include "lz4hc.h"
 
 /* Format:
     size32_t totalexpsize;
@@ -28,7 +29,9 @@
 
 class jlib_decl CLZ4Compressor : public CFcmpCompressor
 {
-    virtual void setinmax()
+    bool hc;
+protected:
+    virtual void setinmax() override
     {
         inmax = blksz-outlen-sizeof(size32_t);
         if (inmax<256)
@@ -45,7 +48,7 @@ class jlib_decl CLZ4Compressor : public CFcmpCompressor
         }
     }
 
-    virtual void flushcommitted()
+    virtual void flushcommitted() override
     {
         // only does non trailing
         if (trailing)
@@ -75,7 +78,10 @@ class jlib_decl CLZ4Compressor : public CFcmpCompressor
         size32_t *cmpsize = (size32_t *)(outbuf+outlen);
         byte *out = (byte *)(cmpsize+1);
 
-        *cmpsize = LZ4_compress_default((const char *)inbuf, (char *)out, toflush, LZ4_COMPRESSBOUND(toflush));
+        if (hc)
+            *cmpsize = LZ4_compress_HC((const char *)inbuf, (char *)out, toflush, LZ4_COMPRESSBOUND(toflush), LZ4HC_CLEVEL_DEFAULT);
+        else
+            *cmpsize = LZ4_compress_default((const char *)inbuf, (char *)out, toflush, LZ4_COMPRESSBOUND(toflush));
         if (*cmpsize && *cmpsize<toflush)
         {
             *(size32_t *)outbuf += toflush;
@@ -92,7 +98,10 @@ class jlib_decl CLZ4Compressor : public CFcmpCompressor
         }
         trailing = true;
     }
-
+public:
+    CLZ4Compressor(bool _hc) : hc(_hc)
+    {        
+    }
 };
 
 
@@ -229,9 +238,9 @@ void LZ4DecompressToBuffer(MemoryAttr & out, MemoryBuffer & in)
 }
 
 
-ICompressor *createLZ4Compressor()
+ICompressor *createLZ4Compressor(bool hc)
 {
-    return new CLZ4Compressor;
+    return new CLZ4Compressor(hc);
 }
 
 IExpander *createLZ4Expander()

system/jlib/jlz4.hpp

@@ -22,7 +22,7 @@
 
 #define LZ4COMPRESSEDFILEBLOCKSIZE (0x100000)
 
-extern jlib_decl ICompressor *createLZ4Compressor();
+extern jlib_decl ICompressor *createLZ4Compressor(bool hc=false);
 extern jlib_decl IExpander   *createLZ4Expander();
 
 extern jlib_decl void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src);

system/jlib/jlzw.cpp

@@ -2130,10 +2130,12 @@ public:
         curblockpos = 0;
         curblocknum = (unsigned)-1; // relies on wrap
         compMethod = _compMethod;
-        if (mode!=ICFread) {
+        if (mode!=ICFread)
+        {
             if (!_fileio&&_mmfile)
                 throw MakeStringException(-1,"Compressed Write not supported on memory mapped files");
-            if (trailer.recordSize) {
+            if (trailer.recordSize)
+            {
                 if ((trailer.recordSize>trailer.blockSize/4) || // just too big
                     (trailer.recordSize<10))                    // or too small
                     trailer.recordSize = 0;
@@ -2142,43 +2144,58 @@ public:
             }
             compblkptr = (byte *)compblk.allocate(trailer.blockSize+trailer.recordSize*2+16); // over estimate!
             compblklen = 0;
-            if (trailer.recordSize==0) {
+            if (trailer.recordSize==0)
+            {
                 if (!compressor)
                 {
-                    if (compMethod == COMPRESS_METHOD_FASTLZ)
-                        compressor.setown(createFastLZCompressor());
-                    else if (compMethod == COMPRESS_METHOD_LZ4)
-                        compressor.setown(createLZ4Compressor());
-                    else // fallback
+                    switch (compMethod)
                     {
-                        compMethod = COMPRESS_METHOD_LZW;
-                        trailer.compressedType = COMPRESSEDFILEFLAG;
-                        compressor.setown(createLZWCompressor(true));
+                        case COMPRESS_METHOD_FASTLZ:
+                            compressor.setown(createFastLZCompressor());
+                            break;
+                        case COMPRESS_METHOD_LZ4:
+                            compressor.setown(createLZ4Compressor(false));
+                            break;
+                        case COMPRESS_METHOD_LZ4HC:
+                            compressor.setown(createLZ4Compressor(true));
+                            break;
+                        default:
+                            compMethod = COMPRESS_METHOD_LZW;
+                            trailer.compressedType = COMPRESSEDFILEFLAG;
+                            compressor.setown(createLZWCompressor(true));
+                            break;
                     }
                 }
                 compressor->open(compblkptr, trailer.blockSize);
             }
         }
-        if (mode!=ICFcreate) {
+        if (mode!=ICFcreate)
+        {
             unsigned nb = trailer.numBlocks();
             size32_t toread = sizeof(offset_t)*nb;
-            if (fileio) {
+            if (fileio)
+            {
                 size32_t r = fileio->read(trailer.indexPos,toread,indexbuf.reserveTruncate(toread));
                 assertex(r==toread);
             }
-            else {
+            else
+            {
                 assertex((memsize_t)trailer.indexPos==trailer.indexPos);
                 memcpy(indexbuf.reserveTruncate(toread),mmfile->base()+(memsize_t)trailer.indexPos,toread);
             }
-            if (mode==ICFappend) {
+            if (mode==ICFappend)
+            {
                 curblocknum = nb-1;
-                if (setcrc) {
+                if (setcrc)
+                {
                     trailer.crc = trailer.datacrc;
                     trailer.datacrc = ~0U;
                 }
             }
-            if (trailer.recordSize==0) {
-                if (!expander) {
+            if (trailer.recordSize==0)
+            {
+                if (!expander)
+                {
                     if (compMethod == COMPRESS_METHOD_FASTLZ)
                         expander.setown(createFastLZExpander());
                     else if (compMethod == COMPRESS_METHOD_LZ4)
@@ -2466,7 +2483,7 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio, bool append, size
             trailer.blockSize = FASTCOMPRESSEDFILEBLOCKSIZE;
             trailer.recordSize = 0;
         }
-        else if (_compMethod == COMPRESS_METHOD_LZ4)
+        else if ((_compMethod == COMPRESS_METHOD_LZ4) || (_compMethod == COMPRESS_METHOD_LZ4HC))
         {
             trailer.compressedType = LZ4COMPRESSEDFILEFLAG;
             trailer.blockSize = LZ4COMPRESSEDFILEBLOCKSIZE;
@@ -2735,6 +2752,14 @@ public:
     }
 } compressors;
 
+typedef IIteratorOf<ICompressHandler> ICompressHandlerIterator;
+
+ICompressHandlerIterator *getCompressHandlerIterator()
+{
+    return new ArrayIIteratorOf<IArrayOf<ICompressHandler>, ICompressHandler, ICompressHandlerIterator>(compressors);
+}
+
+
 
 bool addCompressorHandler(ICompressHandler *handler)
 {
@@ -2776,7 +2801,14 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     {
     public:
         CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { }
-        virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(); }
+        virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(false); }
+        virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); }
+    };
+    class CLZ4HCCompressHandler : public CCompressHandlerBase
+    {
+    public:
+        CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { }
+        virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(true); }
         virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); }
     };
     class CAESCompressHandler : public CCompressHandlerBase
@@ -2812,6 +2844,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     addCompressorHandler(new CDiffCompressHandler());
     addCompressorHandler(new CLZWCompressHandler());
     addCompressorHandler(new CFLZCompressHandler());
+    addCompressorHandler(new CLZ4HCCompressHandler());    
     ICompressHandler *lz4Compressor = new CLZ4CompressHandler();
     addCompressorHandler(lz4Compressor);
     defaultCompressor.set(lz4Compressor);

system/jlib/jlzw.hpp

@@ -99,6 +99,7 @@ extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const voi
 #define COMPRESS_METHOD_FASTLZ 3
 #define COMPRESS_METHOD_LZMA   4
 #define COMPRESS_METHOD_LZ4    5
+#define COMPRESS_METHOD_LZ4HC  6
 
 interface ICompressedFileIO: extends IFileIO
 {
@@ -135,6 +136,8 @@ interface ICompressHandler : extends IInterface
     virtual ICompressor *getCompressor(const char *options=NULL) = 0;
     virtual IExpander *getExpander(const char *options=NULL) = 0;
 };
+typedef IIteratorOf<ICompressHandler> ICompressHandlerIterator;
+extern jlib_decl ICompressHandlerIterator *getCompressHandlerIterator();
 extern jlib_decl void setDefaultCompressor(const char *type);
 extern jlib_decl ICompressHandler *queryCompressHandler(const char *type);
 extern jlib_decl ICompressHandler *queryDefaultCompressHandler();
@@ -157,6 +160,8 @@ inline unsigned translateToCompMethod(const char *compStr)
             compMethod = COMPRESS_METHOD_ROWDIF;
         else if (strieq("LZMA", compStr))
             compMethod = COMPRESS_METHOD_LZMA;
+        else if (strieq("LZ4HC", compStr))
+            compMethod = COMPRESS_METHOD_LZ4HC;
         //else // default is LZ4
     }
     return compMethod;
@@ -174,6 +179,8 @@ inline const char *translateFromCompMethod(unsigned compMethod)
             return "FLZ";
         case COMPRESS_METHOD_LZ4:
             return "LZ4";
+        case COMPRESS_METHOD_LZ4HC:
+            return "LZ4HC";
         case COMPRESS_METHOD_LZMA:
             return "LZMA";
         default:

system/jlib/jsocket.cpp

@@ -6954,9 +6954,9 @@ bool isIPAddress(const char *ip)
 }
 
 
-class CWhiteListHandler : public CSimpleInterfaceOf<IWhiteListHandler>, implements IWhiteListWriter
+class CAllowListHandler : public CSimpleInterfaceOf<IAllowListHandler>, implements IAllowListWriter
 {
-    typedef CSimpleInterfaceOf<IWhiteListHandler> PARENT;
+    typedef CSimpleInterfaceOf<IAllowListHandler> PARENT;
 
     struct PairHasher
     {
@@ -6969,11 +6969,11 @@ class CWhiteListHandler : public CSimpleInterfaceOf<IWhiteListHandler>, implemen
         }
     };
 
-    using WhiteListHT = std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher>;
-    WhiteListPopulateFunction populateFunc;
-    WhiteListFormatFunction roleFormatFunc;
-    std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher> whiteList;
-    std::unordered_set<std::string> IPOnlyWhiteList;
+    using AllowListHT = std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher>;
+    AllowListPopulateFunction populateFunc;
+    AllowListFormatFunction roleFormatFunc;
+    std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher> allowList;
+    std::unordered_set<std::string> IPOnlyAllowList;
     bool allowAnonRoles = false;
     mutable CriticalSection populatedCrit;
     mutable bool populated = false;
@@ -6985,17 +6985,17 @@ class CWhiteListHandler : public CSimpleInterfaceOf<IWhiteListHandler>, implemen
         if (populated)
             return;
         // NB: want to keep this method const, as used by isXX functions that are const, but if need to refresh it's effectively mutable
-        enabled = populateFunc(* const_cast<IWhiteListWriter *>((const IWhiteListWriter *)this));
+        enabled = populateFunc(* const_cast<IAllowListWriter *>((const IAllowListWriter *)this));
         populated = true;
     }
 public:
     IMPLEMENT_IINTERFACE_O_USING(PARENT);
 
-    CWhiteListHandler(WhiteListPopulateFunction _populateFunc, WhiteListFormatFunction _roleFormatFunc) : populateFunc(_populateFunc), roleFormatFunc(_roleFormatFunc)
+    CAllowListHandler(AllowListPopulateFunction _populateFunc, AllowListFormatFunction _roleFormatFunc) : populateFunc(_populateFunc), roleFormatFunc(_roleFormatFunc)
     {
     }
-// IWhiteListHandler impl.
-    virtual bool isWhiteListed(const char *ip, unsigned __int64 role, StringBuffer *responseText) const override
+// IAllowListHandler impl.
+    virtual bool isAllowListed(const char *ip, unsigned __int64 role, StringBuffer *responseText) const override
     {
         CriticalBlock block(populatedCrit);
         ensurePopulated();
@@ -7003,15 +7003,15 @@ public:
         {
             if (allowAnonRoles)
             {
-                const auto &it = IPOnlyWhiteList.find(ip);
-                if (it != IPOnlyWhiteList.end())
+                const auto &it = IPOnlyAllowList.find(ip);
+                if (it != IPOnlyAllowList.end())
                     return true;
             }
         }
         else
         {
-            const auto &it = whiteList.find({ip, role});
-            if (it != whiteList.end())
+            const auto &it = allowList.find({ip, role});
+            if (it != allowList.end())
                 return true;
         }
 
@@ -7034,22 +7034,22 @@ public:
                 else
                     responseText->append(role);
             }
-            responseText->append("] not whitelisted");
+            responseText->append("] not in allowlist");
         }
 
         if (enabled)
             return false;
         else
         {
-            OWARNLOG("WhiteListing is disabled, ignoring: %s", responseText->str());
+            OWARNLOG("Allowlist is disabled, ignoring: %s", responseText->str());
             return true;
         }
     }
-    virtual StringBuffer &getWhiteList(StringBuffer &out) const override
+    virtual StringBuffer &getAllowList(StringBuffer &out) const override
     {
         CriticalBlock block(populatedCrit);
         ensurePopulated();
-        for (const auto &it: whiteList)
+        for (const auto &it: allowList)
         {
             out.append(it.first.c_str()).append(", ");
             if (roleFormatFunc)
@@ -7058,7 +7058,7 @@ public:
                 out.append(it.second);
             out.newline();
         }
-        out.newline().appendf("Whitelisting is currently: %s", enabled ? "enabled" : "disabled").newline();
+        out.newline().appendf("Allowlist is currently: %s", enabled ? "enabled" : "disabled").newline();
         return out;
     }
     virtual void refresh() override
@@ -7068,17 +7068,17 @@ public:
          */
         CriticalBlock block(populatedCrit);
         enabled = true;
-        whiteList.clear();
-        IPOnlyWhiteList.clear();
+        allowList.clear();
+        IPOnlyAllowList.clear();
         populated = false;
     }
-// IWhiteListWriter impl.
+// IAllowListWriter impl.
     virtual void add(const char *ip, unsigned __int64 role) override
     {
         // NB: called via populateFunc, which is called whilst populatedCrit is locked.
-        whiteList.insert({ ip, role });
+        allowList.insert({ ip, role });
         if (allowAnonRoles)
-            IPOnlyWhiteList.insert(ip);
+            IPOnlyAllowList.insert(ip);
     }
     virtual void setAllowAnonRoles(bool tf) override
     {
@@ -7087,9 +7087,9 @@ public:
     }
 };
 
-IWhiteListHandler *createWhiteListHandler(WhiteListPopulateFunction populateFunc, WhiteListFormatFunction roleFormatFunc)
+IAllowListHandler *createAllowListHandler(AllowListPopulateFunction populateFunc, AllowListFormatFunction roleFormatFunc)
 {
-    return new CWhiteListHandler(populateFunc, roleFormatFunc);
+    return new CAllowListHandler(populateFunc, roleFormatFunc);
 }
 
 static_assert(sizeof(IpAddress) == 16, "check size of IpAddress");

system/jlib/jsocket.hpp

@@ -646,22 +646,22 @@ extern jlib_decl bool isIPV4(const char *ip);
 extern jlib_decl bool isIPV6(const char *ip);
 extern jlib_decl bool isIPAddress(const char *ip);
 
-interface IWhiteListHandler : extends IInterface
+interface IAllowListHandler : extends IInterface
 {
-    virtual bool isWhiteListed(const char *ip, unsigned __int64 role, StringBuffer *responseText=nullptr) const = 0;
-    virtual StringBuffer &getWhiteList(StringBuffer &out) const = 0;
+    virtual bool isAllowListed(const char *ip, unsigned __int64 role, StringBuffer *responseText=nullptr) const = 0;
+    virtual StringBuffer &getAllowList(StringBuffer &out) const = 0;
     virtual void refresh() = 0;
 };
 
-interface IWhiteListWriter : extends IInterface
+interface IAllowListWriter : extends IInterface
 {
     virtual void add(const char *ip, unsigned __int64 role) = 0;
     virtual void setAllowAnonRoles(bool tf) = 0;
 };
 
-typedef std::function<bool(IWhiteListWriter &)> WhiteListPopulateFunction;
-typedef std::function<StringBuffer &(StringBuffer &, unsigned __int64)> WhiteListFormatFunction;
-extern jlib_decl IWhiteListHandler *createWhiteListHandler(WhiteListPopulateFunction populateFunc, WhiteListFormatFunction roleFormatFunc = {}); // format function optional
+typedef std::function<bool(IAllowListWriter &)> AllowListPopulateFunction;
+typedef std::function<StringBuffer &(StringBuffer &, unsigned __int64)> AllowListFormatFunction;
+extern jlib_decl IAllowListHandler *createAllowListHandler(AllowListPopulateFunction populateFunc, AllowListFormatFunction roleFormatFunc = {}); // format function optional
 
 #endif
 

system/jlib/jstats.h

@@ -36,9 +36,10 @@ inline constexpr stat_type msecs2StatUnits(stat_type ms) { return ms * 1000000;
 inline constexpr stat_type statUnits2seconds(stat_type stat) {return stat / 1000000000; }
 inline constexpr stat_type statUnits2msecs(stat_type stat) {return stat / 1000000; }
 
-inline constexpr stat_type statSkewPercent(int value) { return (stat_type)value * 100; }            // Since 1 = 0.01% skew
-inline constexpr stat_type statSkewPercent(long double value) { return (stat_type)(value * 100); }
-inline constexpr stat_type statSkewPercent(stat_type  value) { return (stat_type)(value * 100); }
+inline constexpr stat_type statPercent(int value) { return (stat_type)value * 100; }            // Since 1 = 0.01% skew
+inline constexpr stat_type statPercent(double value) { return (stat_type)(value * 100); }
+inline constexpr stat_type statPercent(stat_type  value) { return (stat_type)(value * 100); }
+inline constexpr stat_type statPercentageOf(stat_type value, stat_type per) { return value * per / 10000; }
 
 inline StatisticKind queryStatsVariant(StatisticKind kind) { return (StatisticKind)(kind & ~StKindMask); }
 inline cost_type money2cost_type(double money) { return money * 1E6; }

system/jlib/jutil.cpp

@@ -2747,6 +2747,13 @@ const char * queryCurrentProcessPath()
         default:
             break;
         }
+        const char *canon = realpath(processPath.str(), nullptr);
+        if (canon)
+        {
+            processPath.clear();
+            processPath.set(canon);
+            free((void *) canon);
+        }
 #else
         char path[PATH_MAX + 1];
         ssize_t len = readlink("/proc/self/exe", path, PATH_MAX);

system/lz4_sm/CMakeLists.txt

@@ -27,6 +27,7 @@ project( lz4 )
 
 set ( SRCS
         lz4/lib/lz4.c
+        lz4/lib/lz4hc.c
 )
 
 include_directories (

system/mp/mpcomm.cpp

@@ -441,7 +441,7 @@ class CMPConnectThread: public Thread
     CMPServer *parent;
     int mpSoMaxConn;
     unsigned mpTraceLevel;
-    Owned<IWhiteListHandler> whiteListCallback;
+    Owned<IAllowListHandler> allowListCallback;
     void checkSelfDestruct(void *p,size32_t sz);
 
 public:
@@ -461,13 +461,13 @@ public:
                 printf("CMPConnectThread::stop timed out\n");
         }
     }
-    void installWhiteListCallback(IWhiteListHandler *_whiteListCallback)
+    void installAllowListCallback(IAllowListHandler *_allowListCallback)
     {
-        whiteListCallback.set(_whiteListCallback);
+        allowListCallback.set(_allowListCallback);
     }
-    IWhiteListHandler *queryWhiteListCallback() const
+    IAllowListHandler *queryAllowListCallback() const
     {
-        return whiteListCallback;
+        return allowListCallback;
     }
 };
 
@@ -584,13 +584,13 @@ public:
                 break;
         }
     }
-    virtual void installWhiteListCallback(IWhiteListHandler *whiteListCallback) override
+    virtual void installAllowListCallback(IAllowListHandler *allowListCallback) override
     {
-        connectthread->installWhiteListCallback(whiteListCallback);
+        connectthread->installAllowListCallback(allowListCallback);
     }
-    virtual IWhiteListHandler *queryWhiteListCallback() const override
+    virtual IAllowListHandler *queryAllowListCallback() const override
     {
-        return connectthread->queryWhiteListCallback();
+        return connectthread->queryAllowListCallback();
     }
 };
 
@@ -2129,12 +2129,12 @@ int CMPConnectThread::run()
                     }
                 }
 
-                if (whiteListCallback)
+                if (allowListCallback)
                 {
                     StringBuffer ipStr;
                     peerEp.getIpText(ipStr);
                     StringBuffer responseText; // filled if denied
-                    if (!whiteListCallback->isWhiteListed(ipStr, connectHdr.getRole(), &responseText))
+                    if (!allowListCallback->isAllowListed(ipStr, connectHdr.getRole(), &responseText))
                     {
                         Owned<IException> e = makeStringException(-1, responseText);
                         OWARNLOG(e, nullptr);

system/mp/mpcomm.hpp

@@ -100,8 +100,8 @@ interface IMPServer : extends IInterface
     virtual void stop() = 0;
     virtual INode *queryMyNode() = 0;
     virtual void setOpt(MPServerOpts opt, const char *value) = 0;
-    virtual void installWhiteListCallback(IWhiteListHandler *whiteListCallback) = 0;
-    virtual IWhiteListHandler *queryWhiteListCallback() const = 0;
+    virtual void installAllowListCallback(IAllowListHandler *allowListCallback) = 0;
+    virtual IAllowListHandler *queryAllowListCallback() const = 0;
 };
 
 extern mp_decl void startMPServer(unsigned port, bool paused=false, bool listen=false);

testing/regress/ecl-test

@@ -394,7 +394,7 @@ class RegressMain:
             self.args.setupExtraX = []
             self.args.setupExtraX.append('OriginalTextFilesOsPath='+self.regressionSuiteHpccMainOsDir)
             self.args.setupExtraX.append('OriginalTextFilesEclPath='+self.regressionSuiteHpccMainEclDir)
-            self.args.setupExtraX.append('OriginalTextFilesIp=' + self.config.espIp )
+            self.args.setupExtraX.append('OriginalTextFilesIp=' +self.config.setupExtraParams['OriginalTextFilesIp'] )
 
             self.args.setupExtraX.append('dropzoneIp=' + self.config.dropzoneIp )
             self.args.setupExtraX.append('dropzonePath=' + self.config.dropzonePath )

testing/regress/ecl-test-cluster160.json

@@ -61,7 +61,8 @@
             "disableLocalOptimizations"
         ],
         "setupExtraParams":{
-            "OriginalTextFilesOsPath" : "/opt/HPCCSystems/testing/regress"
+            "OriginalTextFilesOsPath" : "/opt/HPCCSystems/testing/regress",
+            "OriginalTextFilesIp" : "127.0.0.1"
         }
     }
 }

testing/regress/ecl-test.json

@@ -58,7 +58,8 @@
             "failOnLeaks"
         ],
         "setupExtraParams":{
-            "OriginalTextFilesOsPath" : "/opt/HPCCSystems/testing/regress"
+            "OriginalTextFilesOsPath" : "/opt/HPCCSystems/testing/regress",
+            "OriginalTextFilesIp" : "127.0.0.1"
         },
         "FileExclusion":[
             "*_blahblah.ecl"