Merge branch 'candidate-8.4.x' into candidate-8.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitignore

@@ -11,3 +11,4 @@ ln/
 !esp/src/.vscode
 eclcc.log
 *.pyc
+./helm/examples/azure/sa/env-sa

docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -1839,7 +1839,21 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                   <entry>-ssl</entry>
 
                   <entry>Use SSL to secure the connection to the
-                  server.</entry>
+                  server(s)</entry>
+                </row>
+
+                <row>
+                  <entry>--source-ssl</entry>
+
+                  <entry>Use SSL when connecting to the source (default if
+                  --ssl is enabled)</entry>
+                </row>
+
+                <row>
+                  <entry>--source-no-ssl </entry>
+
+                  <entry>Do not use SSL when connecting to source (default if
+                  --ssl is NOT enabled)</entry>
                 </row>
 
                 <row>
@@ -2000,7 +2014,21 @@ ecl queries copy-set roxie1 roxie2 --clone-active-state</programlisting>
                   <entry>-ssl, --ssl</entry>
 
                   <entry>Use SSL to secure the connection to the
-                  server.</entry>
+                  server(s)</entry>
+                </row>
+
+                <row>
+                  <entry>--source-ssl</entry>
+
+                  <entry>Use SSL when connecting to the source (default if
+                  --ssl is enabled)</entry>
+                </row>
+
+                <row>
+                  <entry>--source-no-ssl</entry>
+
+                  <entry>Do not use SSL when connecting to source (default if
+                  --ssl is NOT enabled)</entry>
                 </row>
 
                 <row>

docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -1447,8 +1447,8 @@
                   <entry><emphasis role="bold">Submit drop
                   list</emphasis></entry>
 
-                  <entry>Choose from Submit, Submit Selected, Compile, or
-                  Debug <itemizedlist>
+                  <entry>Choose from Submit, Submit Selected, or Compile
+                  <itemizedlist>
                       <listitem>
                         <para><emphasis role="bold">Submit </emphasis>-
                         submits the ECL code to be compiled and executed on

ecl/eclcmd/queries/ecl-queries.cpp

@@ -344,16 +344,46 @@ public:
             fputs("Target must be specified.\n", stderr);
             return false;
         }
-        if (optTarget.isEmpty())
-        {
-            fputs("Query must be specified.\n", stderr);
-            return false;
-        }
         if (!EclCmdCommon::finalizeOptions(globals))
             return false;
         return true;
     }
 
+    void outputQueryFiles(const char *id, IArrayOf<IConstFileUsedByQuery> &files, IArrayOf<IConstQuerySuperFile> &superfiles)
+    {
+        fputs("------------------\n", stdout);
+        if (!isEmptyString(id))
+            fprintf(stdout, "Query: %s\n", id);
+        if (!files.length())
+            fputs("No files used.\n", stdout);
+        else
+            fputs("Files used:\n", stdout);
+        ForEachItemIn(i, files)
+        {
+            IConstFileUsedByQuery &file = files.item(i);
+            StringBuffer line("  ");
+            line.append(file.getFileName()).append(", ");
+            line.append(file.getFileSize()).append(" bytes, ");
+            line.append(file.getNumberOfParts()).append(" part(s)\n");
+            fputs(line, stdout);
+        }
+        fputs("\n", stdout);
+
+        if (superfiles.length())
+        {
+            fputs("SuperFiles used:\n", stdout);
+            ForEachItemIn(sp, superfiles)
+            {
+                IConstQuerySuperFile &superfile = superfiles.item(sp);
+                fprintf(stdout, "    %s\n", superfile.getName());
+                StringArray &subfiles = superfile.getSubFiles();
+                ForEachItemIn(sb, subfiles)
+                    fprintf(stdout, "    > %s\n", subfiles.item(sb));
+            }
+            fputs("\n", stdout);
+        }
+    }
+
     virtual int processCMD()
     {
         Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
@@ -368,34 +398,18 @@ public:
         if (ret == 0)
         {
             IArrayOf<IConstFileUsedByQuery> &files = resp->getFiles();
-            if (!files.length())
-                fputs("No files used.\n", stdout);
+            if (optQuery.length())
+                outputQueryFiles(optQuery.str(), resp->getFiles(), resp->getSuperFiles());
             else
-                fputs("Files used:\n", stdout);
-            ForEachItemIn(i, files)
             {
-                IConstFileUsedByQuery &file = files.item(i);
-                StringBuffer line("  ");
-                line.append(file.getFileName()).append(", ");
-                line.append(file.getFileSize()).append(" bytes, ");
-                line.append(file.getNumberOfParts()).append(" part(s)\n");
-                fputs(line, stdout);
-            }
-            fputs("\n", stdout);
-
-            IArrayOf<IConstQuerySuperFile> &superfiles = resp->getSuperFiles();
-            if (superfiles.length())
-            {
-                fputs("SuperFiles used:\n", stdout);
-                ForEachItemIn(sp, superfiles)
+                IArrayOf<IConstQueryFilesUsed> &queries = resp->getQueries();
+                if (!queries.length())
+                    fputs("No queries found.\n", stdout);
+                ForEachItemIn(i, queries)
                 {
-                    IConstQuerySuperFile &superfile = superfiles.item(sp);
-                    fprintf(stdout, "  %s\n", superfile.getName());
-                    StringArray &subfiles = superfile.getSubFiles();
-                    ForEachItemIn(sb, subfiles)
-                        fprintf(stdout, "    > %s\n", subfiles.item(sb));
+                    IConstQueryFilesUsed &query = queries.item(i);
+                    outputQueryFiles(query.getQueryId(), query.getFiles(), query.getSuperFiles());
                 }
-                fputs("\n", stdout);
             }
         }
         return ret;
@@ -407,7 +421,7 @@ public:
             "The 'queries files' command displays a list of the files currently in use by\n"
             "the given query.\n"
             "\n"
-            "ecl queries files <target> <query>\n\n"
+            "ecl queries files <target> [<query>]\n\n"
             " Options:\n"
             "   <target>               Name of target cluster the query is published on\n"
             "   <query>                Name of the query to get a list of files in use by\n"

esp/applications/common/ldap/ldap.yaml

@@ -20,4 +20,6 @@ ldap:
   workunitsBasedn: ou=workunits,ou=ecl
   ldapAdminSecretKey: "ldapadmincredskey"
   ldapAdminVaultId: ""
+  hpccAdminSecretKey: ""
+  hpccAdminVaultId: ""
   checkScopeScans: true

esp/bindings/http/platform/httptransport.cpp

@@ -2243,6 +2243,9 @@ int CHttpRequest::readContentToFiles(const char * netAddress, const char * path,
         if (writeError)
             break;
 
+        // if remote file is on Windows we must close it before renaming it
+        fileio->close();
+
         file->rename(fileNameWithPath);
 
         if (!foundAnotherFile)

esp/scm/ws_workunits.ecm

@@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp);
 
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.85"), default_client_version("1.85"), cache_group("ESPWsWUs"),
+    version("1.86"), default_client_version("1.86"), cache_group("ESPWsWUs"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/scm/ws_workunits_queryset_req_resp.ecm

@@ -246,10 +246,18 @@ ESPrequest [nil_remove] WUQueryFilesRequest
     string QueryId;
 };
 
+ESPstruct [exceptions_inline] QueryFilesUsed
+{
+    string QueryId;
+    ESParray<ESPstruct FileUsedByQuery, File> Files;
+    ESParray<ESPstruct QuerySuperFile, SuperFile> SuperFiles;
+};
+
 ESPresponse [exceptions_inline] WUQueryFilesResponse
 {
     ESParray<ESPstruct FileUsedByQuery, File> Files;
     [min_ver("1.85")] ESParray<ESPstruct QuerySuperFile, SuperFile> SuperFiles;
+    [min_ver("1.86")] ESParray<ESPstruct QueryFilesUsed, Query> Queries;
 };
 
 ESPrequest WUQueryDetailsLightWeightRequest

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -2241,6 +2241,46 @@ void WsWuInfo::getWUProcessLogSpecs(const char* processName, const char* logSpec
 }
 #endif
 
+bool WsWuInfo::validateWUProcessLog(const char* file, bool eclAgent)
+{
+    Owned<IStringIterator> logs = cw->getLogs(eclAgent ? "EclAgent" : "Thor");
+    ForEach (*logs)
+    {
+        SCMStringBuffer logName;
+        logs->str(logName);
+        if (logName.length() < 1)
+            continue;
+
+        if (strieq(file, logName.str()))
+            return true;
+    }
+    return false;
+}
+
+bool WsWuInfo::validateWUAssociatedFile(const char* file, WUFileType type)
+{
+    Owned<IConstWUQuery> query = cw->getQuery();
+    if (!query)
+        return false;
+
+    Owned<IConstWUAssociatedFileIterator> iter = &query->getAssociatedFiles();
+    ForEach(*iter)
+    {
+        IConstWUAssociatedFile & cur = iter->query();
+        if (cur.getType() != type)
+            continue;
+
+        SCMStringBuffer name;
+        cur.getName(name);
+        if (name.length() < 1)
+            continue;
+
+        if (strieq(file, name.str()))
+            return true;
+    }
+    return false;
+}
+
 void WsWuInfo::getWorkunitResTxt(MemoryBuffer& buf)
 {
     Owned<IConstWUQuery> query = cw->getQuery();
@@ -4013,8 +4053,11 @@ IFileIOStream* CWsWuFileHelper::createWUFileIOStream(IEspContext& context, const
     return createIOStreamWithFileName(zipFileNameWithPath.str(), IFOread);
 }
 
-void CWsWuFileHelper::validateFilePath(const char *file, bool UNCFileName, const char *fileType, const char *compType, const char *compName)
+void CWsWuFileHelper::validateFilePath(const char* file, WsWuInfo& winfo, CWUFileType wuFileType, bool UNCFileName, const char* fileType, const char* compType, const char* compName)
 {
+    if (validateWUFile(file, winfo, wuFileType))
+        return;
+
     StringBuffer actualPath;
     if (UNCFileName)
         splitUNCFilename(file, nullptr, &actualPath, nullptr, nullptr);
@@ -4026,6 +4069,42 @@ void CWsWuFileHelper::validateFilePath(const char *file, bool UNCFileName, const
         throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid file path %s", actualPath.str());
 }
 
+bool CWsWuFileHelper::validateWUFile(const char* file, WsWuInfo& winfo, CWUFileType wuFileType)
+{
+    bool valid = false;
+    switch (wuFileType)
+    {
+    case CWUFileType_ThorLog:
+    {
+        valid = winfo.validateWUProcessLog(file, false);
+        break;
+    }
+    case CWUFileType_EclAgentLog:
+    {
+        valid = winfo.validateWUProcessLog(file, true);
+        break;
+    }
+    case CWUFileType_CPP:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeCpp);
+        break;
+    }
+    case CWUFileType_LOG:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeLog);
+        break;
+    }
+    case CWUFileType_XML:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeXml);
+        break;
+    }
+    default:
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Unsupported file type %d.", wuFileType);
+    }
+    return valid;
+}
+
 void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, WsWuInfo& winfo, IConstWUFileOption& item,
     StringBuffer& fileName, StringBuffer& fileMimeType)
 {
@@ -4045,9 +4124,9 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     {
         const char *file=item.getName();
 #ifndef _CONTAINERIZED
-        validateFilePath(file, false, "run", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, false, "run", nullptr, nullptr);
 #else
-        validateFilePath(file, false, "query", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, false, "query", nullptr, nullptr);
 #endif
 
         const char *tail=pathTail(file);
@@ -4077,7 +4156,7 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     case CWUFileType_ThorLog:
     {
         const char *file=item.getName();
-        validateFilePath(file, true, "log", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, true, "log", nullptr, nullptr);
 
         fileName.set("thormaster.log");
         fileMimeType.set(HTTP_TYPE_TEXT_PLAIN);
@@ -4098,7 +4177,7 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     case CWUFileType_EclAgentLog:
     {
         const char *file=item.getName();
-        validateFilePath(file, true, "log", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, true, "log", nullptr, nullptr);
 
         fileName.set("eclagent.log");
         fileMimeType.set(HTTP_TYPE_TEXT_PLAIN);

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -221,7 +221,8 @@ public:
         const char* instanceName, const char *ipAddress, const char* logDate, int slaveNum,
         MemoryBuffer& buf, const char* outFile, bool forDownload);
 #endif
-
+    bool validateWUProcessLog(const char* file, bool eclAgent);
+    bool validateWUAssociatedFile(const char* file, WUFileType type);
     void getWorkunitResTxt(MemoryBuffer& buf);
     void getWorkunitArchiveQuery(StringBuffer& str);
     void getWorkunitArchiveQuery(MemoryBuffer& mb);
@@ -654,7 +655,8 @@ public:
         CWUFileDownloadOption &downloadOptions, StringBuffer &contentType);
 
     IFileIOStream* createIOStreamWithFileName(const char *fileNameWithPath, IFOmode mode);
-    void validateFilePath(const char *file, bool UNCFileName, const char *fileType, const char *compType, const char *compName);
+    void validateFilePath(const char *file, WsWuInfo &winfo, CWUFileType wuFileType, bool UNCFileName, const char *fileType, const char *compType, const char *compName);
+    bool validateWUFile(const char *file, WsWuInfo &winfo, CWUFileType wuFileType);
 };
 
 class CWsWuEmailHelper

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -1848,29 +1848,9 @@ bool CWsWorkunitsEx::onWUListQueriesUsingFile(IEspContext &context, IEspWUListQu
     return true;
 }
 
-bool CWsWorkunitsEx::onWUQueryFiles(IEspContext &context, IEspWUQueryFilesRequest &req, IEspWUQueryFilesResponse &resp)
+void addQueryFiles(IPropertyTree *queryTree, StringBuffer &queryid, IArrayOf<IEspFileUsedByQuery> &referencedFiles, IArrayOf<IEspQuerySuperFile> &referencedSuperFiles)
 {
-    const char *target = req.getTarget();
-    validateTargetName(target);
-
-    const char *query = req.getQueryId();
-    if (!query || !*query)
-        throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not specified");
-    Owned<IPropertyTree> registeredQuery = resolveQueryAlias(target, query, true);
-    if (!registeredQuery)
-        throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found");
-    PROGLOG("WUQueryFiles: target %s, query %s", target, query);
-    StringAttr queryid(registeredQuery->queryProp("@id"));
-    registeredQuery.clear();
-
-    Owned<IPropertyTree> tree = filesInUse.getTree();
-    VStringBuffer xpath("%s/Query[@id='%s']", target, queryid.get());
-    IPropertyTree *queryTree = tree->queryPropTree(xpath);
-    if (!queryTree)
-       throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found in file cache (%s)", xpath.str());
-
-    IArrayOf<IEspFileUsedByQuery> referencedFiles;
-    IArrayOf<IEspQuerySuperFile> referencedSuperFiles;
+    queryid.set(queryTree->queryProp("@id"));
     Owned<IPropertyTreeIterator> files = queryTree->getElements("File");
     ForEach(*files)
     {
@@ -1893,8 +1873,74 @@ bool CWsWorkunitsEx::onWUQueryFiles(IEspContext &context, IEspWUQueryFilesReques
             referencedFiles.append(*respFile.getClear());
         }
     }
-    resp.setFiles(referencedFiles);
-    resp.setSuperFiles(referencedSuperFiles);
+}
+
+void addQueriesFiles(IPropertyTreeIterator *queriesTrees, IArrayOf<IEspQueryFilesUsed> &queriesFiles)
+{
+    ForEach(*queriesTrees)
+    {
+        IPropertyTree &queryTree = queriesTrees->get();
+
+        StringBuffer id;
+        IArrayOf<IEspFileUsedByQuery> referencedFiles;
+        IArrayOf<IEspQuerySuperFile> referencedSuperFiles;
+        addQueryFiles(&queryTree, id, referencedFiles, referencedSuperFiles);
+
+        Owned<IEspQueryFilesUsed> queryFilesUsed = createQueryFilesUsed();
+        queryFilesUsed->setQueryId(id);
+        queryFilesUsed->setFiles(referencedFiles);
+        queryFilesUsed->setSuperFiles(referencedSuperFiles);
+        queriesFiles.append(*queryFilesUsed.getClear());
+    }
+}
+
+bool CWsWorkunitsEx::onWUQueryFiles(IEspContext &context, IEspWUQueryFilesRequest &req, IEspWUQueryFilesResponse &resp)
+{
+    const char *target = req.getTarget();
+    validateTargetName(target);
+
+    StringAttr queryid;
+    const char *query = req.getQueryId();
+    if (!query || !*query)
+    {
+        if (context.getClientVersion()<1.86)
+            throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not specified");
+        PROGLOG("WUQueryFiles: target %s, all queries", target);
+    }
+    else
+    {
+        Owned<IPropertyTree> registeredQuery = resolveQueryAlias(target, query, true);
+        if (!registeredQuery)
+            throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found");
+        PROGLOG("WUQueryFiles: target %s, query %s", target, query);
+        queryid.set(registeredQuery->queryProp("@id"));
+    }
+
+    Owned<IPropertyTree> tree = filesInUse.getTree();
+    if (queryid.length())
+    {
+        VStringBuffer xpath("%s/Query[@id='%s']", target, queryid.get());
+        IPropertyTree *queryTree = tree->queryPropTree(xpath);
+        if (!queryTree)
+            throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found in file cache (%s)", xpath.str());
+
+        IArrayOf<IEspFileUsedByQuery> referencedFiles;
+        IArrayOf<IEspQuerySuperFile> referencedSuperFiles;
+        StringBuffer id;
+        addQueryFiles(queryTree, id, referencedFiles, referencedSuperFiles);
+
+        resp.setFiles(referencedFiles);
+        resp.setSuperFiles(referencedSuperFiles);
+    }
+    else
+    {
+        //return entire queryset
+        VStringBuffer xpath("%s/Query", target);
+        Owned<IPropertyTreeIterator> queryTrees = tree->getElements(xpath);
+        IArrayOf<IEspQueryFilesUsed> queriesFiles;
+        addQueriesFiles(queryTrees, queriesFiles);
+        resp.setQueries(queriesFiles);
+    }
     return true;
 }
 

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3046,9 +3046,9 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             {
                 const char* file = req.getName();
 #ifndef _CONTAINERIZED
-                helper.validateFilePath(file, false, "run", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, strieq(File_Cpp,req.getType())? CWUFileType_CPP : CWUFileType_LOG, false, "run", nullptr, nullptr);
 #else
-                helper.validateFilePath(file, false, "query", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, strieq(File_Cpp,req.getType())? CWUFileType_CPP : CWUFileType_LOG, false, "query", nullptr, nullptr);
 #endif
                 winfo.getWorkunitCpp(file, req.getDescription(), req.getIPAddress(),mb, opt > 0, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
@@ -3070,7 +3070,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             else if (strncmp(req.getType(), File_ThorLog, 7) == 0)
             {
                 const char* file = req.getName();
-                helper.validateFilePath(file, true, "log", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, CWUFileType_ThorLog, true, "log", nullptr, nullptr);
                 winfo.getWorkunitThorMasterLog(nullptr, file, mb, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
@@ -3083,7 +3083,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             else if (strieq(File_EclAgentLog,req.getType()))
             {
                 const char* file = req.getName();
-                helper.validateFilePath(file, true, "log", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, CWUFileType_EclAgentLog, true, "log", nullptr, nullptr);
                 winfo.getWorkunitEclAgentLog(nullptr, file, req.getProcess(), mb, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
@@ -3092,9 +3092,9 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             {
                 const char* name  = req.getName();
 #ifndef _CONTAINERIZED
-                helper.validateFilePath(name, false, "run", nullptr, nullptr);
+                helper.validateFilePath(name, winfo, CWUFileType_XML, false, "run", nullptr, nullptr);
 #else
-                helper.validateFilePath(name, false, "query", nullptr, nullptr);
+                helper.validateFilePath(name, winfo, CWUFileType_XML, false, "query", nullptr, nullptr);
 #endif
                 const char* ptr = strrchr(name, '/');
                 if (ptr)

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -30,6 +30,7 @@
 #include "referencedfilelist.hpp"
 #include "ws_wuresult.hpp"
 #include "jsmartsock.ipp"
+#include <atomic>
 
 #define UFO_DIRTY                                0x01
 #define UFO_RELOAD_TARGETS_CHANGED_PMID          0x02
@@ -38,9 +39,11 @@
 
 static const __uint64 defaultWUResultMaxSize = 0x100000*10; //10M
 
-class QueryFilesInUse : public CInterface, implements ISDSSubscription
+class QueryFilesInUse : public CInterface, implements ISDSSubscription, implements IThreaded
 {
     mutable CriticalSection crit;
+    CThreaded threaded;
+    Semaphore sem;
     MapStringTo<IUserDescriptor *> roxieUserMap;
     IArrayOf<IUserDescriptor> roxieUsers;
 
@@ -50,7 +53,7 @@ class QueryFilesInUse : public CInterface, implements ISDSSubscription
     SubscriptionId psChange;
     mutable CriticalSection dirtyCrit; //if there were an atomic_or I would just use atomic
     unsigned dirty;
-    bool aborting;
+    std::atomic_bool aborting;
 private:
     void loadTarget(IPropertyTree *tree, const char *target, unsigned flags);
     void loadTargets(IPropertyTree *tree, unsigned flags);
@@ -83,10 +86,31 @@ private:
 
 public:
     IMPLEMENT_IINTERFACE;
-    QueryFilesInUse() : aborting(false), qsChange(0), pmChange(0), psChange(0), dirty(UFO_DIRTY)
+    QueryFilesInUse() : threaded("QueryFilesInUse"), aborting(false), qsChange(0), pmChange(0), psChange(0), dirty(UFO_DIRTY)
     {
         tree.setown(createPTree("QueryFilesInUse"));
         updateUsers();
+        threaded.init(this);
+    }
+
+    ~QueryFilesInUse()
+    {
+        aborting = true;
+        sem.signal();
+        threaded.join();
+    }
+
+    virtual void threadmain()
+    {
+        while (!aborting)
+        {
+            //prepopulate the cache, lazy mode is very slow
+            //getTree() builds the cache, but only does work if the cache is dirty (changes in dali would have dirtied the cache)
+            Owned<IPropertyTree> thetree = getTree();
+            //wait 1 minute, then check if dirty again
+            if (sem.wait(60000))
+                break;
+        }
     }
 
     virtual void notify(SubscriptionId subid, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
@@ -152,6 +176,8 @@ public:
     }
     IPropertyTree *getTree()
     {
+        //getTree() is thread safe because when load(flags) is called below it creates a new working tree,
+        //  the previous tree might still be in use outside this class, in which case it will not actually freed until there are no remaining references outstanding
         CriticalBlock b(crit);
         unsigned flags;
         {

esp/src/eclwatch/LFDetailsWidget.js

@@ -276,11 +276,11 @@ define([
             });
             this.logicalFile.refresh();
 
-            this.isProtected.on("change", function (evt) {
+            this.isProtected.on("click", function (evt) {
                 context._onSave();
             });
 
-            this.isRestricted.on("change", function (evt) {
+            this.isRestricted.on("click", function (evt) {
                 context._onSave();
             });
         },

esp/src/eclwatch/SFDetailsWidget.js

@@ -36,6 +36,7 @@ define([
     "dijit/form/Button",
     "dijit/form/DropDownButton",
     "dijit/form/ToggleButton",
+    "dijit/Fieldset",
     "dijit/TitlePane"
 ], function (exports, declare, nlsHPCCMod, arrayUtil, dom, domAttr, domClass, domForm, MemoryMod, Observable, all,
     registry,

esp/src/src/ESPResult.ts

@@ -476,7 +476,7 @@ class Result {
                             formatter(cell, row) {
                                 switch (typeof cell) {
                                     case "string":
-                                        return cell.replace("\t", "    ");
+                                        return cell.replace(/\t/g, "    ");
                                 }
                                 return cell;
                             }

esp/src/src/Utility.ts

@@ -64,7 +64,7 @@ export function parseXML(val) {
 export function csvEncode(cell) {
     if (!isNaN(cell)) return cell;
     if (cell === undefined) return "";
-    return '"' + String(cell).replace('"', '""') + '"';
+    return '"' + String(cell).replace(/"/g, '""') + '"';
 }
 
 export function espTime2Seconds(duration?: string) {

helm/examples/azure/sa/create-sa.sh

@@ -31,7 +31,7 @@ fi
 rc=$(az group exists --name ${SA_RESOURCE_GROUP})
 if [ "$rc" != "true" ]
 then
-  az group create --name ${SA_RESOURCE_GROUP} --location ${SA_LOCATION}
+  az group create --name ${SA_RESOURCE_GROUP} --location ${SA_LOCATION} --tags ${TAGS}
 fi
 
 az storage account check-name -n $STORAGE_ACCOUNT_NAME | \
@@ -43,7 +43,8 @@ then
     -n $STORAGE_ACCOUNT_NAME \
     -g $SA_RESOURCE_GROUP \
     -l $SA_LOCATION \
-    --sku $SA_SKU
+    --sku $SA_SKU \
+    --tags ${TAGS}
 fi
 # Export the connection string as an environment variable,
 # this is used when creating the Azure file share
@@ -54,7 +55,7 @@ for shareName in $SHARE_NAMES
 do
   az storage share exists --connection-string "${AZURE_STORAGE_CONNECTION_STRING}" \
     --name  $shareName | grep -q  "\"exists\":[[:space:]]*false"
-  if [ $? -eq 0 ]
+  if [ $? -ne 0 ]
   then
     echo "create share $shareName"
     az storage share create \

helm/examples/azure/sa/env-sa

@@ -2,6 +2,7 @@
 SUBSCRIPTION=
 STORAGE_ACCOUNT_NAME=
 SA_RESOURCE_GROUP=
+TAGS=
 # Set the same location as Kubernetes cluster
 SA_LOCATION=eastus
 SA_KEY_DIR=~/.azure/storage
@@ -14,4 +15,4 @@ SA_SKU=Standard_LRS
 # Kubernetes secret.
 SECRET_NAME=
 SECRET_NAMESPACE="default"
-SHARE_NAMES=
+SHARE_NAMES="dalishare dllshare sashashare datashare lzshare"

helm/hpcc/values.schema.json

@@ -691,6 +691,14 @@
           "type": "string",
           "description": "The optional vault name to be used to look up the Active Directory Administrator account Username/Password, using ldapAdminSecretKey"
         },
+        "hpccAdminSecretKey": {
+          "type": "string",
+          "description": "The optional key name to be used to look up the HPCC Administrator account Username/Password"
+        },
+        "hpccAdminVaultId": {
+          "type": "string",
+          "description": "The optional vault name to be used to look up the HPCC Administrator account Username/Password, using hpccAdminSecretKey"
+        },
         "ldapPort": {
           "type": "integer",
           "description": "The port of the nonsecure Active Directory server"

helm/managed/logging/elastic/Chart.yaml

@@ -5,10 +5,10 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 1.2.0
+version: 1.2.1
 
 # Elastic Stack version
-appVersion: 7.16.1
+appVersion: 7.16.2
 
 # Dependencies can be automatically updated via HELM dependancy update command:
 # > 'helm dependency update' command
@@ -16,12 +16,12 @@ appVersion: 7.16.1
 # > helm install myelastic ./ —-dependency-update
 dependencies:
 - name: filebeat
-  version: 7.16.1
+  version: 7.16.2
   repository: https://helm.elastic.co
 - name: elasticsearch
-  version: 7.16.1
+  version: 7.16.2
   repository: https://helm.elastic.co
 - name: kibana # Optional managed logging processor front-end
-  version: 7.16.1
+  version: 7.16.2
   repository: https://helm.elastic.co
   condition: kibana.enabled

helm/managed/logging/elastic/README.md

@@ -4,7 +4,7 @@
   <thead>
     <tr>
       <td align="left">
-        :zap: <b>Note:</b> Elastic Stack components have been reported to be affected by the high-severity vulnerability (CVE-2021-44228) impacting multiple versions of the Apache Log4j 2 utility
+        :zap: <b>Note:</b> Elastic Stack components prior to 7.16.0 have been reported to be affected by the high-severity vulnerability (CVE-2021-44228) impacting multiple versions of the Apache Log4j 2 utility
       </td>
     </tr>
   </thead>
@@ -18,6 +18,14 @@
         </ul>
       </td>
     </tr>
+    <tr>
+      <td>
+        <ul>
+          <li>elastic4hpcclogs chart version 1.2.1 references Elastic Stack 7.16.2 (Log4j 2.17.0) which reportedly fully mitigates CVE-2021-44228 and should avoid false positives in vulnerability scanners.</li>
+          <li>Learn more about Elastic's 7.16.2 release and their response to the vulnerability: https://discuss.elastic.co/t/apache-log4j2-remote-code-execution-rce-vulnerability-cve-2021-44228-esa-2021-31/291476</li>
+        </ul>
+      </td>
+    </tr>
   </tbody>
 </table>
 

initfiles/componentfiles/configschema/templates/AddCost.json

@@ -0,0 +1,44 @@
+{
+    "name": "Add cost element",
+    "description": "Add cost element to the hardware portion of the environment.xml",
+    "type": "modification",
+    "operations": [
+        {
+            "action" : "delete",
+            "target_path" : "/Environment/Hardware/cost",
+            "data" : {
+                "error_if_not_found" : false
+            }
+        },
+        {
+            "action": "create",
+            "description": "Create the cost element in the Environment/Hardware section",
+            "target_path": "/Environment/Hardware",
+            "data": {
+                "node_type": "cost",
+                "attributes": [
+                    {
+                        "name": "currencyCode",
+                        "value": "USD"
+                    },
+                    {
+                        "name": "perCpu",
+                        "value": "0.113"
+                    },
+                    {
+                        "name": "storageAtRest",
+                        "value": "0.0135"
+                    },
+                    {
+                        "name": "storageReads",
+                        "value": "0.0485"
+                    },
+                    {
+                        "name": "storageWrites",
+                        "value": "0.0038"
+                    }
+                ]
+            }
+        }
+    ]
+}

initfiles/componentfiles/configschema/templates/CMakeLists.txt

@@ -15,3 +15,12 @@
 ################################################################################
 
 ADD_SUBDIRECTORY(schema)
+
+
+FOREACH( iFILES
+        ${CMAKE_CURRENT_SOURCE_DIR}/AddAuthenticateFeature.json
+        ${CMAKE_CURRENT_SOURCE_DIR}/AddKeys.json
+        ${CMAKE_CURRENT_SOURCE_DIR}/AddCost.json
+        )
+    Install ( FILES ${iFILES} DESTINATION componentfiles/configschema/templates COMPONENT Runtime)
+ENDFOREACH ( iFILES )

system/security/LdapSecurity/ldapconnection.cpp

@@ -277,6 +277,9 @@ private:
     bool                 m_sysuser_specified;
     StringBuffer         m_sysuser_dn;
 
+    StringBuffer         m_HPCCAdminUser_username;
+    StringBuffer         m_HPCCAdminUser_password;
+
     int                  m_maxConnections;
     
     StringBuffer         m_sdfieldname;
@@ -360,19 +363,19 @@ public:
         //  - Hardcoded : systemCommonName, systemPassword (legacy environment.xml)
         //------------------------------------------------
 
-        StringBuffer systemUserSecretKey;
-        cfg->getProp(".//@ldapAdminSecretKey", systemUserSecretKey);//vault/secrets LDAP username key
-        if (!systemUserSecretKey.isEmpty())
+        StringBuffer adminUserSecretKey;
+        cfg->getProp(".//@ldapAdminSecretKey", adminUserSecretKey);//vault/secrets LDAP username key
+        if (!adminUserSecretKey.isEmpty())
         {
             StringBuffer vaultId;
             cfg->getProp(".//@ldapAdminVaultId", vaultId);//optional HashiCorp vault ID
 
-            DBGLOG("Retrieving LDAP Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", systemUserSecretKey.str());
+            DBGLOG("Retrieving LDAP Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str());
             Owned<IPropertyTree> secretTree;
-            if (!isEmptyString(vaultId.str()))
-                secretTree.setown(getVaultSecret("authn", vaultId, systemUserSecretKey.str(), nullptr));
+            if (!vaultId.isEmpty())
+                secretTree.setown(getVaultSecret("authn", vaultId, adminUserSecretKey.str(), nullptr));
             else
-                secretTree.setown(getSecret("authn", systemUserSecretKey.str()));
+                secretTree.setown(getSecret("authn", adminUserSecretKey.str()));
             if (!secretTree)
                 throw MakeStringException(-1, "Error retrieving LDAP Admin username/password");
 
@@ -444,6 +447,32 @@ public:
             throw MakeStringException(-1, "getServerInfo error - %s", ldap_err2string(rc));
         }
 
+        //------------------------------------------------
+        //Get optional HPCC Admin account username
+        // Can be specified as
+        //  - Kubernetes secret : lookup key value hpccAdminSecretKey
+        //  - Vault secret : lookup key value hpccAdminVaultId, hpccAdminSecretKey
+        //------------------------------------------------
+        cfg->getProp(".//@hpccAdminSecretKey", adminUserSecretKey.clear());//vault/secrets HPCC Admin username key
+        if (!adminUserSecretKey.isEmpty())
+        {
+            StringBuffer vaultId;
+            cfg->getProp(".//@hpccAdminVaultId", vaultId);//optional HashiCorp vault ID
+
+            DBGLOG("Retrieving optional HPCC Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str());
+            Owned<IPropertyTree> secretTree;
+            if (!vaultId.isEmpty())
+                secretTree.setown(getVaultSecret("authn", vaultId, adminUserSecretKey.str(), nullptr));
+            else
+                secretTree.setown(getSecret("authn", adminUserSecretKey.str()));
+            if (secretTree)
+            {
+                getSecretKeyValue(m_HPCCAdminUser_username, secretTree, "username");
+                getSecretKeyValue(m_HPCCAdminUser_password, secretTree, "password");
+                DBGLOG("Retrieved HPCC Admin username %s", m_HPCCAdminUser_username.str());
+            }
+        }
+
         const char* basedn = cfg->queryProp(".//@commonBasedn");
         if(basedn == NULL || *basedn == '\0')
         {
@@ -750,6 +779,16 @@ public:
         return m_sysuser_specified;
     }
 
+    virtual const char* getHPCCAdminUser_username()
+    {
+        return m_HPCCAdminUser_username.str();
+    }
+
+    virtual const char* getHPCCAdminUser_password()
+    {
+        return m_HPCCAdminUser_password.str();
+    }
+
     virtual int getMaxConnections()
     {
         return m_maxConnections;
@@ -1561,6 +1600,18 @@ public:
     virtual void init(IPermissionProcessor* pp)
     {
         m_pp = pp;
+
+        //Isolate optional HPCC Admin group name
+        StringBuffer adminGroupName;
+        if (!isEmptyString(m_ldapconfig->getAdminGroupDN()))
+        {
+            const char * p = strchr(m_ldapconfig->getAdminGroupDN(), '=');
+            if (p)
+            {
+                adminGroupName.append(++p);
+                adminGroupName.replace(',', (char)0);
+            }
+        }
         static bool createdOU = false;
         CriticalBlock block(lcCrit);
         if (!createdOU)
@@ -1593,8 +1644,58 @@ public:
 
             createLdapBasedn(NULL, m_ldapconfig->getUserBasedn(), PT_ADMINISTRATORS_ONLY);
             createLdapBasedn(NULL, m_ldapconfig->getGroupBasedn(), PT_ADMINISTRATORS_ONLY);
+
+            //Create the HPCC Administrators group and admin user
+            if (!adminGroupName.isEmpty())
+            {
+                //Create HPCC admin group
+                bool groupExisted = organizationalUnitExists(m_ldapconfig->getAdminGroupDN());
+                if (!groupExisted)
+                {
+                    DBGLOG("Adding HPCC Admin group %s", adminGroupName.str());
+                    try { addGroup(adminGroupName.str(), nullptr, "HPCC Administrators"); }
+                    catch(...) {}//group may already exist, so just move on
+
+                    //Create HPCC Admin user
+                    const char * pUser = m_ldapconfig->getHPCCAdminUser_username();
+                    if (pUser)
+                    {
+                        DBGLOG("Creating HPCC Admin user %s", pUser);
+                        Owned<ISecUser> user = new CLdapSecUser(pUser, nullptr);
+                        user->credentials().setPassword(m_ldapconfig->getHPCCAdminUser_password());
+                        try { addUser(*user.get()); }
+                        catch(...) {}//user may already exist, so just move on
+
+                        //Add HPCC Admin user to admin group
+                        DBGLOG("Adding HPCC Admin user %s to HPCC Admin group %s", pUser, adminGroupName.str());
+                        try { changeUserGroup("add", pUser, adminGroupName); }
+                        catch(...) {}//user may already be in group so just move on
+                    }
+                    else
+                        DBGLOG("HPCC Admin user not specified in configuration (hpccAdminSecretKey)");
+                }
+            }
             createdOU = true;
         }
+
+        //Set ECLWatch permissions for HPCC admin group
+        if (!adminGroupName.isEmpty())
+        {
+            //SmcAccess OU attribute is not created until later in
+            // startup, so this action must be delayed until here
+            CPermissionAction action;
+            action.m_action = "update";
+            action.m_basedn = m_ldapconfig->getResourceBasedn(RT_DEFAULT);
+            action.m_rname = "SmcAccess";
+            action.m_rtype = RT_SERVICE;
+            action.m_account_name = adminGroupName;
+            action.m_account_type = GROUP_ACT;
+            action.m_allows = SecAccess_Full;
+            action.m_denies = 0;
+            DBGLOG("Setting permissions for HPCC Admin group %s", adminGroupName.str());
+            try { changePermission(action); }
+            catch(...) {}//nothing to do here, so just move on
+        }
     }
 
     virtual LdapServerType getServerType()