Merge branch 'candidate-7.4.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/package.h

@@ -52,7 +52,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
-    virtual bool validate(StringArray &queriesToVerify, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const = 0;
+    virtual bool validate(const StringArray &queriesToVerify, const StringArray &queriesToIgnore, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const = 0;
     virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const = 0;
     virtual const StringArray &getPartIds() const = 0;
 };

common/workunit/pkgimpl.hpp

@@ -457,8 +457,27 @@ public:
         }
     }
 
-    virtual bool validate(StringArray &queriesToCheck, StringArray &warn, StringArray &err, 
-        StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
+    bool checkIgnoreQuery(const StringArray &queriesToIgnore, const char *queryid) const
+    {
+        ForEachItemIn(i, queriesToIgnore)
+        {
+            const char *match = queriesToIgnore.item(i);
+            if (match && *match)
+            {
+                if (containsWildcard(match))
+                {
+                    if (WildMatch(queryid, match, true))
+                        return true;
+                }
+                else if (streq(match, queryid))
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    virtual bool validate(const StringArray &queriesToCheck, const StringArray &queriesToIgnore, StringArray &warn, StringArray &err,
+        StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const
     {
         bool isValid = true;
         MapStringTo<bool> referencedPackages;
@@ -520,6 +539,8 @@ public:
             const char *queryid = queries->query().queryProp("@id");
             if (queryid && *queryid)
             {
+                if (checkIgnoreQuery(queriesToIgnore, queryid))
+                    continue;
                 Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, true, false);
                 Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(queries->query().queryProp("@wuid"));
 
@@ -569,7 +590,11 @@ public:
                     }
                     VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
                     if (!(flags & RefFileNotOptional))
+                    {
+                        if (ignoreOptionalFiles)
+                            continue;
                         fullname.append("/Optional");
+                    }
                     else if (isCompulsory)
                         fullname.append("/Compulsory");
                     unmatchedFiles.append(fullname);

common/workunit/workunit.cpp

@@ -178,16 +178,9 @@ CWuGraphStats::CWuGraphStats(IPropertyTree *_progress, StatisticCreatorType _cre
     StatsScopeId graphScopeId;
     verifyex(graphScopeId.setScopeText(_rootScope));
 
-    if (wfid)
-    {
-        StatsScopeId rootScopeId(SSTworkflow,wfid);
-        collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
-        collector->beginScope(graphScopeId);
-    }
-    else
-    {
-        collector.setown(createStatisticsGatherer(_creatorType, _creator, graphScopeId));
-    }
+    StatsScopeId rootScopeId(SSTworkflow,wfid);
+    collector.setown(createStatisticsGatherer(_creatorType, _creator, rootScopeId));
+    collector->beginScope(graphScopeId);
 }
 
 void CWuGraphStats::beforeDispose()
@@ -208,7 +201,6 @@ void CWuGraphStats::beforeDispose()
     StringBuffer tag;
     tag.append("sg").append(id);
 
-
     //Replace the particular subgraph statistics added by this creator
     StringBuffer qualified(tag);
     qualified.append("[@creator='").append(creator).append("']");
@@ -474,6 +466,11 @@ static int compareGraphNode(IInterface * const *ll, IInterface * const *rr)
 {
     IPropertyTree *l = (IPropertyTree *) *ll;
     IPropertyTree *r = (IPropertyTree *) *rr;
+    unsigned lwfid = l->getPropInt("@wfid");
+    unsigned rwfid = r->getPropInt("@wfid");
+    if (lwfid != rwfid)
+        return lwfid > rwfid ? +1 : -1;
+
     const char * lname = l->queryName();
     const char * rname = r->queryName();
     return compareScopeName(lname, rname);
@@ -3714,9 +3711,9 @@ public:
             }
         }
     }
-    virtual void setGraphState(const char *graphName, WUGraphState state) const
+    virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
     {
-        Owned<IRemoteConnection> conn = getWritableProgressConnection(graphName);
+        Owned<IRemoteConnection> conn = getWritableProgressConnection(graphName, wfid);
         conn->queryRoot()->setPropInt("@_state", state);
     }
     virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
@@ -3753,7 +3750,7 @@ public:
     }
     virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const override
     {
-        return new CDaliWuGraphStats(getWritableProgressConnection(graphName), creatorType, creator, _wfid, graphName, subgraph);
+        return new CDaliWuGraphStats(getWritableProgressConnection(graphName, _wfid), creatorType, creator, _wfid, graphName, subgraph);
     }
 
 protected:
@@ -3767,12 +3764,29 @@ protected:
         }
         return progressConnection.getLink();
     }
-    IRemoteConnection *getWritableProgressConnection(const char *graphName) const
+    IRemoteConnection *getWritableProgressConnection(const char *graphName, unsigned wfid) const
     {
         CriticalBlock block(crit);
         progressConnection.clear(); // Make sure subsequent reads from this workunit get the changes I am making
         VStringBuffer path("/GraphProgress/%s/%s", queryWuid(), graphName);
-        return querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+        Owned<IRemoteConnection> conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+        IPropertyTree * root = conn->queryRoot();
+        assertex(wfid);
+
+        if (!root->hasProp("@wfid"))
+        {
+            root->setPropInt("@wfid", wfid);
+        }
+        else
+        {
+            //Ideally the following code would check that the wfids are passed consistently.
+            //However there is an obscure problem with out of line functions being called from multiple workflow
+            //ids, and possibly library graphs.
+            //Stats for library graphs should be nested below the library call activity
+            //assertex(root->getPropInt("@wfid", 0) == wfid); // check that wfid is passed consistently
+        }
+
+        return conn.getClear();
     }
     IPropertyTree *getGraphProgressTree() const
     {
@@ -4013,8 +4027,8 @@ public:
             { return c->queryGraphState(graphName); }
     virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const
             { return c->queryNodeState(graphName, nodeId); }
-    virtual void setGraphState(const char *graphName, WUGraphState state) const
-            { c->setGraphState(graphName, state); }
+    virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
+            { c->setGraphState(graphName, wfid, state); }
     virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const
             { c->setNodeState(graphName, nodeId, state); }
     virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const override
@@ -9985,7 +9999,7 @@ WUGraphState CLocalWorkUnit::queryNodeState(const char *graphName, WUGraphIDType
 {
     throwUnexpected();   // Should only be used for persisted workunits
 }
-void CLocalWorkUnit::setGraphState(const char *graphName, WUGraphState state) const
+void CLocalWorkUnit::setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
 {
     throwUnexpected();   // Should only be used for persisted workunits
 }

common/workunit/workunit.hpp

@@ -1280,7 +1280,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
 
     virtual WUGraphState queryGraphState(const char *graphName) const = 0;
     virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const = 0;
-    virtual void setGraphState(const char *graphName, WUGraphState state) const = 0;
+    virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const = 0;
     virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const = 0;
     virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const = 0;
     virtual void clearGraphProgress() const = 0;

common/workunit/workunit.ipp

@@ -272,7 +272,7 @@ public:
     virtual IConstWUGraph * getGraph(const char *name) const;
     virtual IConstWUGraphProgress * getGraphProgress(const char * name) const;
     virtual WUGraphState queryGraphState(const char *graphName) const;
-    virtual void setGraphState(const char *graphName, WUGraphState state) const;
+    virtual void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const;
     virtual void setNodeState(const char *graphName, WUGraphIDType nodeId, WUGraphState state) const;
     virtual WUGraphState queryNodeState(const char *graphName, WUGraphIDType nodeId) const;
     virtual IWUGraphStats *updateStats(const char *graphName, StatisticCreatorType creatorType, const char * creator, unsigned _wfid, unsigned subgraph) const override;

dali/base/daclient.cpp

@@ -92,7 +92,7 @@ bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport,
 {
     assertex(servergrp);
     daliClientIsActive = true;
-    startMPServer(mpport);
+    startMPServer(role, mpport);
     Owned<ICommunicator> comm(createCommunicator(servergrp,true));
     IGroup * covengrp;
     if (!registerClientProcess(comm.get(),covengrp,timeout,role))

dali/base/dacoven.cpp

@@ -321,7 +321,7 @@ public:
         if (comm)
             comm->cancel(srcrank,tag);
     }
-    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) override
+    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) const override
     {
         assertex(comm);
         return comm->queryChannelPeerEndpoint(sender);
@@ -727,14 +727,12 @@ public:
         return comm->send(mbuf,dstrank,tag,timeout);
     }
 
-
     bool sendRecv(CMessageBuffer &mbuff, rank_t sendrank, mptag_t sendtag, unsigned timeout=MP_WAIT_FOREVER)
     {
         assertex(comm);
         return comm->sendRecv(mbuff,sendrank,sendtag,timeout);
     }
 
-
     virtual unsigned probe(rank_t srcrank, mptag_t tag, rank_t *sender=NULL, unsigned timeout=0)
     {
         return comm->probe(srcrank,tag,sender,timeout);
@@ -745,20 +743,17 @@ public:
         return comm->recv(mbuf,srcrank,tag,sender,timeout);
     }
 
-    virtual bool reply   (CMessageBuffer &mbuff, unsigned timeout=MP_WAIT_FOREVER)
+    virtual bool reply(CMessageBuffer &mbuff, unsigned timeout=MP_WAIT_FOREVER)
     {
         assertex(comm);
         return comm->reply(mbuff,timeout);
     }
 
-
     virtual void disconnect(INode *node)
     {
         assertex(comm);
         comm->disconnect(node);
     }
-
-
 };
     
 #define CATCH_MPERR_link_closed \

dali/base/dadiags.cpp

@@ -257,7 +257,10 @@ public:
                     mb.append(success).append(reply);
                 }
                 else if (0 == stricmp(id, "whitelist")) {
-                    mb.append(querySessionManager().getWhiteList(buf).str());
+                    Owned<IMPServer> mpServer = getMPServer();
+                    IWhiteListHandler *whiteListHandler = mpServer->queryWhiteListCallback();
+                    if (whiteListHandler)
+                        mb.append(whiteListHandler->getWhiteList(buf).str());
                 }
                 else
                     mb.append(StringBuffer("UNKNOWN OPTION: ").append(id).str());

dali/base/dasds.cpp

@@ -6480,7 +6480,10 @@ void CCovenSDSManager::saveDelta(const char *path, IPropertyTree &changeTree)
 
         if (startsWith(path, "/Environment") || (streq(path, "/") && changeTree.hasProp("*[@name=\"Environment\"]")))
         {
-            querySessionManager().refreshWhiteList();
+            Owned<IMPServer> mpServer = getMPServer();
+            IWhiteListHandler *whiteListHandler = mpServer->queryWhiteListCallback();
+            if (whiteListHandler)
+                whiteListHandler->refresh();
             PROGLOG("Dali Environment updated, path = %s", path);
             return;
         }

dali/base/dasess.cpp

@@ -17,9 +17,8 @@
 
 #define da_decl DECL_EXPORT
 
-#include <unordered_map>
-#include <unordered_set>
 #include <string>
+#include <unordered_map>
 
 #include "platform.h"
 #include "jlib.hpp"
@@ -74,8 +73,18 @@ static std::unordered_map<std::string, DaliClientRole> daliClientRoleMap = {
     { "XRef", DCR_XRef },
     { "EclMinus", DCR_EclMinus },
     { "Monitoring", DCR_Monitoring },
+    { "DaliStop", DCR_DaliStop },
 };
 
+DaliClientRole queryRole(const char *roleStr)
+{
+    const auto &it = daliClientRoleMap.find(roleStr);
+    if (it == daliClientRoleMap.end())
+        return DCR_Unknown;
+    return it->second;
+}
+
+
 const char *queryRoleName(DaliClientRole role)
 {
     switch (role) {
@@ -106,12 +115,12 @@ const char *queryRoleName(DaliClientRole role)
     case DCR_XRef: return "XRef";
     case DCR_EclMinus: return "EclMinus";
     case DCR_Monitoring: return "Monitoring";
+    case DCR_DaliStop: return "DaliStop";
     }
     return "Unknown";
 }
 
 
-
 interface ISessionManagerServer: implements IConnectionMonitor
 {
     virtual SessionId registerSession(SecurityToken tok,SessionId parentid) = 0;
@@ -125,7 +134,6 @@ interface ISessionManagerServer: implements IConnectionMonitor
     virtual void stopSession(SessionId sessid,bool failed) = 0;
     virtual void setClientAuth(IDaliClientAuthConnection *authconn) = 0;
     virtual void setLDAPconnection(IDaliLdapConnection *_ldapconn) = 0;
-    virtual bool authorizeConnection(const IpAddress &clientIP, DaliClientRole role) = 0;
     virtual void start() = 0;
     virtual void ready() = 0;
     virtual void stop() = 0;
@@ -557,35 +565,8 @@ public:
                 const SocketEndpoint &peerIP = coven.queryComm().queryChannelPeerEndpoint(mb.getSender());
                 Owned<INode> servernode(deserializeINode(mb));  // hopefully me, but not if forwarded
                 int role=0;
-                if (mb.length()-mb.getPos()>=sizeof(role)) { // a capability block present
+                if (mb.length()-mb.getPos()>=sizeof(role)) // a capability block present
                     mb.read(role);
-                    if (!manager.authorizeConnection(peerIP, (DaliClientRole) role))
-                    {
-                        MilliSleep(2000); // Delay makes rapid probing of all possible roles slightly more painful.
-                        SocketEndpoint sender = mb.getSender();
-                        mb.clear();
-                        mb.append((SessionId) 0);
-                        INode *na = queryNullNode();
-                        Owned<IGroup> dummyCoven = createIGroup(1, &na);
-                        dummyCoven->serialize(mb);
-                        const char *roleName = queryRoleName((DaliClientRole)role);
-                        StringBuffer ipStr;
-                        peerIP.getIpText(ipStr);
-                        Owned<IException> e = makeStringExceptionV(-1, "Access denied! [client ip=%s, role=%s]", ipStr.str(), roleName);
-                        EXCLOG(e, nullptr);
-                        serializeException(e, mb);
-                        coven.reply(mb);
-                        MilliSleep(100+getRandom()%1000); // Causes client to 'work' for a short time.
-                        Owned<INode> node = createINode(sender);
-                        coven.disconnect(node);
-                        break;
-                    }
-#ifdef _DEBUG
-                    StringBuffer eps;
-                    PROGLOG("Connection to %s at %s authorized",queryRoleName((DaliClientRole)role),mb.getSender().getUrlStr(eps).str());
-#endif
-                }
-                
                 IGroup *covengrp;
                 id = manager.registerClientProcess(node.get(),covengrp,(DaliClientRole)role);
                 mb.clear().append(id);
@@ -905,8 +886,6 @@ public:
             servernotifys.item(i).doNotify(aborted);
         }
     }
-
-
 };
 
 
@@ -1160,16 +1139,6 @@ public:
         queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DALI_SESSION_REQUEST,SESSIONREPLYTIMEOUT);
     }
 
-    virtual void refreshWhiteList() override
-    {
-        throwUnexpectedX("refreshWhiteList called on client");
-    }
-
-    virtual StringBuffer &getWhiteList(StringBuffer &out) const override
-    {
-        throwUnexpectedX("getWhiteList called on client");
-    }
-
     void onClose(SocketEndpoint &ep)
     {
         CHECKEDCRITICALBLOCK(sessmanagersect,60000);
@@ -1319,249 +1288,6 @@ public:
 };
 
 
-// std::hash specialization for DaliClientRole, used in std::pair for whiteList
-namespace std {
-    template <>
-    struct hash<DaliClientRole> {
-        size_t operator ()(DaliClientRole value) const {
-            return static_cast<size_t>(value);
-        }
-    };
-}
-
-/* NB: Ideally this belongs within common/environment,
- * however, that would introduce a circular dependency.
- */
-class CWhiteListHandler
-{
-    struct PairHasher
-    {
-        template <class T1, class T2>
-        std::size_t operator () (std::pair<T1, T2> const &pair) const
-        {
-            std::size_t h1 = std::hash<T1>()(pair.first);
-            std::size_t h2 = std::hash<T2>()(pair.second);
-            return h1 ^ h2;
-        }
-    };
-    std::unordered_set<std::pair<std::string, DaliClientRole>, PairHasher> whiteList;
-    std::unordered_map<std::string, std::string> machineMap;
-    mutable CriticalSection populatedCrit;
-    bool populated = false;
-    bool enabled = true;
-
-    void populateMachineMap(IPropertyTree &environment)
-    {
-        Owned<IPropertyTreeIterator> machineIter = environment.getElements("Hardware/Computer");
-        ForEach(*machineIter)
-        {
-            const IPropertyTree &machine = machineIter->query();
-            const char *name = machine.queryProp("@name");
-            const char *host = machine.queryProp("@netAddress");
-            machineMap.insert({name, host});
-        }
-    }
-    const char *resolveComputer(const char *compName, const char *defaultValue, StringBuffer &result) const
-    {
-        const auto &it = machineMap.find(compName);
-        if (it == machineMap.end())
-            return defaultValue;
-        IpAddress ip(it->second.c_str());
-        if (ip.isNull())
-            return defaultValue;
-        return ip.getIpText(result);
-    }
-    void addRoles(const IPropertyTree &component, const std::vector<DaliClientRole> &roles)
-    {
-        Owned<IPropertyTreeIterator> instanceIter = component.getElements("Instance");
-        ForEach(*instanceIter)
-        {
-            const char *compName = instanceIter->query().queryProp("@computer");
-            StringBuffer ipSB;
-            const char *ip = resolveComputer(compName, component.queryProp("@netAddress"), ipSB);
-            if (ip)
-            {
-                for (auto &role: roles)
-                    whiteList.insert({ ip, role });
-            }
-        }
-    }
-    void populate()
-    {
-        Owned<IRemoteConnection> conn = querySDS().connect("/Environment", 0, 0, INFINITE);
-        assertex(conn);
-        populateMachineMap(*conn->queryRoot());
-        enum SoftwareComponentType
-        {
-            RoxieCluster,
-            ThorCluster,
-            EclAgentProcess,
-            DfuServerProcess,
-            EclCCServerProcess,
-            EspProcess,
-            SashaServerProcess,
-            EclSchedulerProcess,
-            DaliServerProcess,
-            BackupNodeProcess,
-            EclServerProcess,
-        };
-        std::unordered_map<std::string, SoftwareComponentType> softwareTypeRoleMap = {
-                { "RoxieCluster", RoxieCluster },
-                { "ThorCluster", ThorCluster },
-                { "EclAgentProcess", EclAgentProcess },
-                { "DfuServerProcess", DfuServerProcess },
-                { "EclCCServerProcess", EclCCServerProcess },
-                { "EspProcess", EspProcess },
-                { "SashaServerProcess", SashaServerProcess },
-                { "EclSchedulerProcess", EclSchedulerProcess },
-                { "DaliServerProcess", DaliServerProcess },
-                { "BackupNodeProcess", BackupNodeProcess },
-                { "EclServerProcess", EclServerProcess },
-        };
-
-        Owned<IPropertyTreeIterator> softwareIter = conn->queryRoot()->getElements("Software/*");
-        ForEach(*softwareIter)
-        {
-            const IPropertyTree &component = softwareIter->query();
-            const char *compProcess = component.queryName();
-            const auto &it = softwareTypeRoleMap.find(compProcess);
-            if (it != softwareTypeRoleMap.end())
-            {
-                switch (it->second)
-                {
-                    case RoxieCluster:
-                    {
-                        Owned<IPropertyTreeIterator> serverIter = component.getElements("RoxieServerProcess");
-                        ForEach(*serverIter)
-                        {
-                            const IPropertyTree &server = serverIter->query();
-                            const char *serverCompName = server.queryProp("@computer");
-                            StringBuffer ipSB;
-                            const char *ip = resolveComputer(serverCompName, server.queryProp("@netAddress"), ipSB);
-                            if (ip)
-                                whiteList.insert({ ip, DCR_RoxyMaster });
-                        }
-                        break;
-                    }
-                    case ThorCluster:
-                    {
-                        const char *masterCompName = component.queryProp("ThorMasterProcess/@computer");
-                        StringBuffer ipSB;
-                        const char *ip = resolveComputer(masterCompName, component.queryProp("@netAddress"), ipSB);
-                        if (ip)
-                        {
-                            whiteList.insert({ ip, DCR_ThorMaster });
-                            whiteList.insert({ ip, DCR_DaliAdmin });
-                        }
-                        break;
-                    }
-                    case EclAgentProcess:
-                        addRoles(component, { DCR_EclAgent, DCR_AgentExec });
-                        break;
-                    case DfuServerProcess:
-                        addRoles(component, { DCR_DfuServer });
-                        break;
-                    case EclCCServerProcess:
-                        addRoles(component, { DCR_EclCCServer, DCR_EclCC });
-                        break;
-                    case EclServerProcess:
-                        addRoles(component, { DCR_EclServer, DCR_EclCC });
-                        break;
-                    case EspProcess:
-                        addRoles(component, { DCR_EspServer });
-                        break;
-                    case SashaServerProcess:
-                        addRoles(component, { DCR_SashaServer, DCR_XRef });
-                        break;
-                    case EclSchedulerProcess:
-                        addRoles(component, { DCR_EclScheduler });
-                        break;
-                    case BackupNodeProcess:
-                        addRoles(component, { DCR_BackupGen, DCR_DaliAdmin });
-                        break;
-                    case DaliServerProcess:
-                        addRoles(component, { DCR_DaliServer, DCR_DaliDiag, DCR_SwapNode, DCR_UpdateEnv, DCR_DaliAdmin, DCR_TreeView, DCR_Testing, DCR_DaFsControl, DCR_XRef, DCR_Config, DCR_ScheduleAdmin, DCR_Monitoring });
-                        break;
-                }
-            }
-        }
-        // only ever expecting 1 DaliServerProcess and 1 WhiteList
-        IPropertyTree *whiteListTree = conn->queryRoot()->queryPropTree("Software/DaliServerProcess[1]/WhiteList[1]");
-        if (whiteListTree)
-        {
-            enabled = whiteListTree->getPropBool("@enabled", true); // on by default
-            Owned<IPropertyTreeIterator> whiteListIter = whiteListTree->getElements("Entry");
-            ForEach(*whiteListIter)
-            {
-                const IPropertyTree &entry = whiteListIter->query();
-                StringArray hosts, roles;
-                hosts.appendListUniq(entry.queryProp("@hosts"), ",");
-                roles.appendListUniq(entry.queryProp("@roles"), ",");
-                ForEachItemIn(h, hosts)
-                {
-                    ForEachItemIn(r, roles)
-                    {
-                        const char *roleStr = roles.item(r);
-                        const auto &it = daliClientRoleMap.find(roleStr);
-                        if (it != daliClientRoleMap.end())
-                        {
-                            IpAddress ip(hosts.item(h));
-                            if (!ip.isNull())
-                            {
-                                StringBuffer ipStr;
-                                whiteList.insert({ ip.getIpText(ipStr).str(), it->second });
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        populated = true;
-    }
-    void ensurePopulated() const
-    {
-        // should be called within CS
-        if (populated)
-            return;
-        (const_cast <CWhiteListHandler *> (this))->populate();
-    }
-public:
-    bool isWhiteListed(const char *ip, DaliClientRole role) const
-    {
-        CriticalBlock block(populatedCrit);
-        ensurePopulated();
-        const auto &it = whiteList.find({ip, role});
-        if (it != whiteList.end())
-            return true;
-        else if (enabled)
-            return false;
-        else
-        {
-            WARNLOG("Whitelist mechanism is currently disabled");
-            return true;
-        }
-    }
-    StringBuffer &getWhiteList(StringBuffer &out) const
-    {
-        CriticalBlock block(populatedCrit);
-        ensurePopulated();
-        for (auto &it: whiteList)
-            out.append(it.first.c_str()).append(", ").append(queryRoleName(it.second)).append("\n");
-        return out;
-    }
-    void refresh()
-    {
-        /* NB: clear only, so that next usage will re-populated
-         * Do not want to repopulate now, because refresh() is likely called within a update write transaction
-         */
-        CriticalBlock block(populatedCrit);
-        enabled = true;
-        whiteList.clear();
-        machineMap.clear();
-        populated = false;
-    }
-};
-
 class CCovenSessionManager: public CSessionManagerBase, implements ISessionManagerServer, implements ISubscriptionManager
 {
     CSessionRequestServer   sessionrequestserver;
@@ -1573,7 +1299,6 @@ class CCovenSessionManager: public CSessionManagerBase, implements ISessionManag
     atomic_t ldapwaiting;
     Semaphore workthreadsem;
     bool stopping;
-    CWhiteListHandler whiteListHandler;
 
     void remoteAddProcessSession(rank_t dst,SessionId id,INode *node, DaliClientRole role)
     {
@@ -1915,15 +1640,6 @@ public:
 #endif
     }
 
-
-    bool authorizeConnection(const IpAddress &clientIP, DaliClientRole role)
-    {
-        StringBuffer ipStr;
-        clientIP.getIpText(ipStr);
-        return whiteListHandler.isWhiteListed(ipStr, role);
-    }
-
-
     SessionId startSession(SecurityToken tok, SessionId parentid)
     {
         return registerSession(tok,parentid);
@@ -2086,16 +1802,6 @@ protected:
         }
     }
 
-    virtual void refreshWhiteList() override
-    {
-        whiteListHandler.refresh();
-    }
-
-    virtual StringBuffer &getWhiteList(StringBuffer &out) const override
-    {
-        return whiteListHandler.getWhiteList(out);
-    }
-
     void onClose(SocketEndpoint &ep)
     {
         StringBuffer clientStr;

dali/base/dasess.hpp

@@ -80,6 +80,7 @@ enum DaliClientRole // if changed must update queryRoleName()
     DCR_XRef,
     DCR_EclMinus,
     DCR_Monitoring,
+    DCR_DaliStop,
     DCR_Max
 };
 
@@ -133,8 +134,6 @@ interface ISessionManager: extends IInterface
     virtual bool clearPermissionsCache(IUserDescriptor *udesc)=0;
     virtual bool queryScopeScansEnabled(IUserDescriptor *udesc, int * err, StringBuffer &retMsg)=0;
     virtual bool enableScopeScans(IUserDescriptor *udesc, bool enable, int * err, StringBuffer &retMsg)=0;
-    virtual void refreshWhiteList() = 0;
-    virtual StringBuffer &getWhiteList(StringBuffer &out) const = 0;
 };
 
 // the following are getPermissionsLDAP input flags for audit reporting
@@ -168,5 +167,7 @@ interface IDaliClientAuthConnection;
 extern da_decl IDaliServer *createDaliSessionServer(); // called for coven members
 extern da_decl void setLDAPconnection(IDaliLdapConnection *ldapconn); // called for coven members
 extern da_decl void setClientAuth(IDaliClientAuthConnection *authconn); // called for coven members
+extern da_decl const char *queryRoleName(DaliClientRole role);
+extern da_decl DaliClientRole queryRole(const char *roleStr);
 
 #endif

dali/dalistop/CMakeLists.txt

@@ -45,4 +45,5 @@ install ( TARGETS dalistop RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries ( dalistop 
          jlib
          mp 
+         dalibase 
 )

dali/dalistop/dalistop.cpp

@@ -43,8 +43,9 @@ int main(int argc, char* argv[])
             if (argc>=3)
                 nowait = stricmp(argv[2],"/nowait")==0;
             printf("Stopping Dali Server on %s\n",argv[1]);
-            startMPServer(0);
+
             Owned<IGroup> group = createIGroup(1,&ep); 
+            initClientProcess(group, DCR_DaliStop);
             Owned<ICommunicator> comm = createCommunicator(group);
             CMessageBuffer mb;
             int fn=-1;

dali/server/daserver.cpp

@@ -15,6 +15,9 @@
     limitations under the License.
 ############################################################################## */
 
+#include <unordered_map>
+#include <string>
+
 #include "build-config.h"
 #include "platform.h"
 #include "thirdparty.h"
@@ -136,6 +139,199 @@ void usage(void)
     printf("--daemon|-d <instanceName>\t: run daemon as instance\n");
 }
 
+/* NB: Ideally this belongs within common/environment,
+ * however, that would introduce a circular dependency.
+ */
+static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
+{
+    Owned<IRemoteConnection> conn = querySDS().connect("/Environment", 0, 0, INFINITE);
+    assertex(conn);
+
+    // only ever expecting 1 DaliServerProcess and 1 WhiteList
+    const IPropertyTree *whiteListTree = conn->queryRoot()->queryPropTree("Software/DaliServerProcess[1]/WhiteList[1]");
+    if (whiteListTree)
+    {
+        if (!whiteListTree->getPropBool("@enabled", true)) // on by default
+            return false;
+        // Default for now is to allow clients that send no role (legacy) to connect if their IP is whitelisted.
+        writer.setAllowAnonRoles(whiteListTree->getPropBool("@allowAnonRoles", true));
+    }
+
+    std::unordered_map<std::string, std::string> machineMap;
+
+    auto populateMachineMap = [&machineMap](const IPropertyTree &environment)
+    {
+        Owned<IPropertyTreeIterator> machineIter = environment.getElements("Hardware/Computer");
+        ForEach(*machineIter)
+        {
+            const IPropertyTree &machine = machineIter->query();
+            const char *name = machine.queryProp("@name");
+            const char *host = machine.queryProp("@netAddress");
+            machineMap.insert({name, host});
+        }
+    };
+    auto resolveComputer = [&machineMap](const char *compName, const char *defaultValue, StringBuffer &result) -> const char *
+    {
+        const auto &it = machineMap.find(compName);
+        if (it == machineMap.end())
+            return defaultValue;
+        IpAddress ip(it->second.c_str());
+        if (ip.isNull())
+            return defaultValue;
+        return ip.getIpText(result);
+    };
+    auto addRoles = [&writer, &resolveComputer](const IPropertyTree &component, const std::initializer_list<unsigned __int64> &roles)
+    {
+        Owned<IPropertyTreeIterator> instanceIter = component.getElements("Instance");
+        ForEach(*instanceIter)
+        {
+            const char *compName = instanceIter->query().queryProp("@computer");
+            StringBuffer ipSB;
+            const char *ip = resolveComputer(compName, component.queryProp("@netAddress"), ipSB);
+            if (ip)
+            {
+                for (const auto &role: roles)
+                    writer.add(ip, role);
+            }
+        }
+    };
+
+    populateMachineMap(*conn->queryRoot());
+    enum SoftwareComponentType
+    {
+        RoxieCluster,
+        ThorCluster,
+        EclAgentProcess,
+        DfuServerProcess,
+        EclCCServerProcess,
+        EspProcess,
+        SashaServerProcess,
+        EclSchedulerProcess,
+        DaliServerProcess,
+        BackupNodeProcess,
+        EclServerProcess,
+    };
+    std::unordered_map<std::string, SoftwareComponentType> softwareTypeRoleMap = {
+            { "RoxieCluster", RoxieCluster },
+            { "ThorCluster", ThorCluster },
+            { "EclAgentProcess", EclAgentProcess },
+            { "DfuServerProcess", DfuServerProcess },
+            { "EclCCServerProcess", EclCCServerProcess },
+            { "EspProcess", EspProcess },
+            { "SashaServerProcess", SashaServerProcess },
+            { "EclSchedulerProcess", EclSchedulerProcess },
+            { "DaliServerProcess", DaliServerProcess },
+            { "BackupNodeProcess", BackupNodeProcess },
+            { "EclServerProcess", EclServerProcess },
+    };
+
+    Owned<IPropertyTreeIterator> softwareIter = conn->queryRoot()->getElements("Software/*");
+    ForEach(*softwareIter)
+    {
+        const IPropertyTree &component = softwareIter->query();
+        const char *compProcess = component.queryName();
+        const auto &it = softwareTypeRoleMap.find(compProcess);
+        if (it != softwareTypeRoleMap.end())
+        {
+            switch (it->second)
+            {
+                case RoxieCluster:
+                {
+                    Owned<IPropertyTreeIterator> serverIter = component.getElements("RoxieServerProcess");
+                    ForEach(*serverIter)
+                    {
+                        const IPropertyTree &server = serverIter->query();
+                        const char *serverCompName = server.queryProp("@computer");
+                        StringBuffer ipSB;
+                        const char *ip = resolveComputer(serverCompName, server.queryProp("@netAddress"), ipSB);
+                        if (ip)
+                            writer.add(ip, DCR_RoxyMaster);
+                    }
+                    break;
+                }
+                case ThorCluster:
+                {
+                    const char *masterCompName = component.queryProp("ThorMasterProcess/@computer");
+                    StringBuffer ipSB;
+                    const char *ip = resolveComputer(masterCompName, component.queryProp("@netAddress"), ipSB);
+                    if (ip)
+                    {
+                        writer.add(ip, DCR_ThorMaster);
+                        writer.add(ip, DCR_DaliAdmin);
+                    }
+                    break;
+                }
+                case EclAgentProcess:
+                    addRoles(component, { DCR_EclAgent, DCR_AgentExec });
+                    break;
+                case DfuServerProcess:
+                    addRoles(component, { DCR_DfuServer });
+                    break;
+                case EclCCServerProcess:
+                    addRoles(component, { DCR_EclCCServer, DCR_EclCC });
+                    break;
+                case EclServerProcess:
+                    addRoles(component, { DCR_EclServer, DCR_EclCC });
+                    break;
+                case EspProcess:
+                    addRoles(component, { DCR_EspServer });
+                    break;
+                case SashaServerProcess:
+                    addRoles(component, { DCR_SashaServer, DCR_XRef });
+                    break;
+                case EclSchedulerProcess:
+                    addRoles(component, { DCR_EclScheduler });
+                    break;
+                case BackupNodeProcess:
+                    addRoles(component, { DCR_BackupGen, DCR_DaliAdmin });
+                    break;
+                case DaliServerProcess:
+                    addRoles(component, { DCR_DaliServer, DCR_DaliDiag, DCR_SwapNode, DCR_UpdateEnv, DCR_DaliAdmin, DCR_TreeView, DCR_Testing, DCR_DaFsControl, DCR_XRef, DCR_Config, DCR_ScheduleAdmin, DCR_Monitoring, DCR_DaliStop });
+                    break;
+            }
+        }
+    }
+
+    if (whiteListTree)
+    {
+        Owned<IPropertyTreeIterator> whiteListIter = whiteListTree->getElements("Entry");
+        ForEach(*whiteListIter)
+        {
+            const IPropertyTree &entry = whiteListIter->query();
+            StringArray hosts, roles;
+            hosts.appendListUniq(entry.queryProp("@hosts"), ",");
+            roles.appendListUniq(entry.queryProp("@roles"), ",");
+            ForEachItemIn(h, hosts)
+            {
+                IpAddress ip(hosts.item(h));
+                if (!ip.isNull())
+                {
+                    StringBuffer ipStr;
+                    ip.getIpText(ipStr);
+                    ForEachItemIn(r, roles)
+                    {
+                        const char *roleStr = roles.item(r);
+
+                        char *endPtr;
+                        long numericRole = strtol(roleStr, &endPtr, 10);
+                        if (endPtr != roleStr && (numericRole>0 && numericRole<DCR_Max)) // in case legacy role needs adding
+                            writer.add(ipStr.str(), numericRole);
+                        else
+                            writer.add(ipStr.str(), queryRole(roleStr));
+                    }
+                }
+            }
+        }
+    }
+    return true;
+}
+
+static StringBuffer &formatDaliRole(StringBuffer &out, unsigned __int64 role)
+{
+    return out.append(queryRoleName((DaliClientRole)role));
+}
+
+
 int main(int argc, char* argv[])
 {
     rank_t myrank = 0;
@@ -362,7 +558,11 @@ int main(int argc, char* argv[])
         }
 
         unsigned short myport = epa.item(myrank).port;
-        startMPServer(myport,true);
+        startMPServer(DCR_DaliServer, myport, true);
+        Owned<IMPServer> mpServer = getMPServer();
+        Owned<IWhiteListHandler> whiteListHandler = createWhiteListHandler(populateWhiteListFromEnvironment, formatDaliRole);
+        mpServer->installWhiteListCallback(whiteListHandler);
+
         setMsgLevel(fileMsgHandler, serverConfig->getPropInt("SDS/@msgLevel", 100));
         startLogMsgChildReceiver(); 
         startLogMsgParentReceiver();
@@ -433,7 +633,7 @@ int main(int argc, char* argv[])
             throw;
         }
         PROGLOG("DASERVER[%d] starting - listening to port %d",myrank,queryMyNode()->endpoint().port);
-        startMPServer(myport,false);
+        startMPServer(DCR_DaliServer, myport,false);
         bool ok = true;
         ForEachItemIn(i2,servers)
         {

docs/EN_US/ECLLanguageReference/ECLR_mods/Basics-Constants.xml

@@ -262,7 +262,7 @@ MyReal2 := 1.0e1;  // value of MyReal2 is the REAL value 10.0
       time(&timeinsecs);  
       localtime_r(&timeinsecs,&localt);
       char temp[15];
-      strftime(temp , 15, "%Y%m%d%H%M%S", &localt); // Formats the localtime to YYYYMMDDhhmmss 
+      strftime(temp , 15, "%Y%m%d%H%M%S", &localt); // Formats the localtime to YYYYMMDDhhmmss 
       strncpy(__result, temp, 14);
       ENDC++;
       RETURN fGetDimeTime();

docs/EN_US/ECLLanguageReference/ECLR_mods/SpecStruc-BeginC++.xml

@@ -348,9 +348,10 @@ STREAMED DATASET(out1Rec) extractResult3(doneRec done) := BEGINC++
         }
         virtual void stop() {}
     private:
+        Linked<IEngineRowAllocator> allocator;
         unsigned id;
         unsigned idx;
-        Linked<IEngineRowAllocator> allocator;
+        
     };
     #body
     const unsigned id = *(unsigned *)done;

docs/EN_US/ECLLanguageReference/ECLR_mods/SpecStruc-Interface.xml

@@ -141,7 +141,7 @@ InRec  := {HeaderRec AND NOT [RecID,Address,Zip]};
 
 //this MODULE creates a concrete instance
 BatchHeaderSearch(InRec l) := MODULE(IHeaderFileSearch)
-  EXPORT STRING120 company_val := l.company;
+  EXPORT STRING20 company_val := l.company;
   EXPORT STRING2 state_val := l.state;
   EXPORT STRING25 city_val := l.city;
 END;

docs/EN_US/ECLLanguageReference/ECLR_mods/Templ-OPTION.xml

@@ -647,6 +647,19 @@
               <entry>For Roxie queries. When true, generates a unique GlobalId
               if one is not provided.</entry>
             </row>
+
+            <row>
+              <entry><emphasis>analyzeWorkunit</emphasis></entry>
+
+              <entry></entry>
+
+              <entry>Overrides the setting in ECL Agent to analyze workunits
+              after ECL queries are executed (Thor only). This allows a
+              workunit to be further analyzed to identify and display any
+              potential issues. These possible issues display in ECL Watch's
+              "Warnings &amp; Errors" area. The global setting defaults to
+              TRUE, but can be changed using Configuration Manager.</entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/EditDistance.xml

@@ -10,15 +10,15 @@
       <primary>Str.EditDistance</primary>
     </indexterm><indexterm>
       <primary>EditDistance</primary>
-    </indexterm>(</emphasis> <emphasis>string1, string2</emphasis> <emphasis
-  role="bold">)</emphasis></para>
+    </indexterm>(</emphasis> <emphasis>string1, string2, radius</emphasis>
+  <emphasis role="bold">)</emphasis></para>
 
   <para><emphasis role="bold">STD.Uni.EditDistance<indexterm>
       <primary>STD.Uni.EditDistance</primary>
     </indexterm><indexterm>
       <primary>Uni.EditDistance</primary>
-    </indexterm>(</emphasis> <emphasis>string1, string2, locale</emphasis>
-  <emphasis role="bold">)</emphasis></para>
+    </indexterm>(</emphasis> <emphasis>string1, string2, locale,
+  radius</emphasis> <emphasis role="bold">)</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
@@ -48,6 +48,13 @@
         </row>
 
         <row>
+          <entry><emphasis>radius</emphasis></entry>
+
+          <entry>Optional. The maximum acceptable edit distance, or 0 for no
+          limit. Defaults to 0.</entry>
+        </row>
+
+        <row>
           <entry>Return:<emphasis> </emphasis></entry>
 
           <entry>EditDistance returns an UNSIGNED4 value.</entry>
@@ -59,14 +66,21 @@
   <para>The <emphasis role="bold">EditDistance </emphasis>function returns a
   standard Levenshtein distance algorithm score for the edit distance between
   <emphasis>string1</emphasis> and <emphasis>string2</emphasis>. This score
-  i\reflects the minimum number of operations needed to transform
+  reflects the minimum number of operations needed to transform
   <emphasis>string1</emphasis> into <emphasis>string2</emphasis>.</para>
 
+  <para>If the edit distance is larger than the <emphasis>radius</emphasis> it
+  will return an arbitrary value &gt; <emphasis>radius</emphasis>, but it may
+  not be accurate. This allows the function to terminate faster if the strings
+  are significantly different.</para>
+
   <para>Example:</para>
 
   <programlisting format="linespecific">STD.Str.EditDistance('CAT','CAT');  //returns 0
 STD.Str.EditDistance('CAT','BAT');  //returns 1
 STD.Str.EditDistance('BAT','BAIT'); //returns 1
 STD.Str.EditDistance('CAT','BAIT'); //returns 2
+STD.Str.EditDistance('CARTMAN','BATMAN');   //returns 2
+STD.Str.EditDistance('CARTMAN','BATMAN',1); //returns arbitrary number &gt; 1
 </programlisting>
 </sect1>

docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -348,10 +348,6 @@ gpgcheck=0</programlisting></para>
                   <row>
                     <entry><itemizedlist>
                         <listitem>
-                          <para>Python : pyembed</para>
-                        </listitem>
-
-                        <listitem>
                           <para>JAVA : javaembed</para>
                         </listitem>
 
@@ -390,8 +386,8 @@ gpgcheck=0</programlisting></para>
               </tgroup>
             </informaltable></para>
 
-          <para>Some other technologies, such as Cassandra support are
-          included in the platform package.</para>
+          <para>Some other technologies, such as Python and Cassandra support
+          are included in the platform package.</para>
         </sect3>
       </sect2>
 
@@ -2187,7 +2183,7 @@ sudo /etc/init.d/hpcc-init -c esp start
         </listitem>
 
         <listitem>
-          <para>Python</para>
+          <para>Python (full support is already built-in)</para>
         </listitem>
 
         <listitem>

ecl/ecl-package/ecl-package.cpp

@@ -843,12 +843,21 @@ public:
                 continue;
             if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
                 continue;
+            if (iter.matchFlag(optIgnoreWarnings, ECLOPT_IGNORE_WARNINGS))
+                continue;
+            if (iter.matchFlag(optIgnoreOptionalFiles, ECLOPT_IGNORE_OPTIONAL))
+                continue;
             StringAttr queryIds;
             if (iter.matchOption(queryIds, ECLOPT_QUERYID))
             {
                 optQueryIds.appendList(queryIds.get(), ",");
                 continue;
             }
+            if (iter.matchOption(queryIds, ECLOPT_IGNORE_QUERIES))
+            {
+                optIgnoreQueries.appendList(queryIds.get(), ",");
+                continue;
+            }
             eclCmdOptionMatchIndicator ind = EclCmdCommon::matchCommandLineOption(iter, true);
             if (ind != EclCmdOptionMatch)
                 return ind;
@@ -928,8 +937,11 @@ public:
         request->setPMID(optPMID);
         request->setTarget(optTarget);
         request->setQueriesToVerify(optQueryIds);
+        request->setQueriesToIgnore(optIgnoreQueries);
         request->setCheckDFS(optCheckDFS);
         request->setGlobalScope(optGlobalScope);
+        request->setIgnoreWarnings(optIgnoreWarnings);
+        request->setIgnoreOptionalFiles(optIgnoreOptionalFiles);
 
         bool validateMessages = false;
         Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
@@ -1025,6 +1037,11 @@ public:
                     "   --queryid                   Query to verify against packagemap, multiple queries can be\n"
                     "                               specified using a comma separated list, or by using --queryid\n"
                     "                               more than once. Default is all queries in the target queryset\n"
+                    "   --ignore-queries            Queries to exclude from verification, multiple queries can be\n"
+                    "                               specified using wildcards, a comma separated list, or by using\n"
+                    "                               --ignore-queries more than once.\n"
+                    "   --ignore-optional           Doesn't warn when optional files are not defined in packagemap.\n"
+                    "   --ignore-warnings           Doesn't output general packagemap warnings.\n"
                     "   --global-scope              The specified packagemap can be shared across multiple targets\n",
                     stdout);
 
@@ -1032,12 +1049,15 @@ public:
     }
 private:
     StringArray optQueryIds;
+    StringArray optIgnoreQueries;
     StringAttr optFileName;
     StringAttr optTarget;
     StringAttr optPMID;
     bool optValidateActive;
     bool optCheckDFS;
     bool optGlobalScope;
+    bool optIgnoreWarnings = false;
+    bool optIgnoreOptionalFiles = false;
 };
 
 class EclCmdPackageQueryFiles : public EclCmdCommon

ecl/eclagent/eclagent.ipp

@@ -1115,8 +1115,8 @@ class EclGraph : public CInterface
     } graphCodeContext;
 
 public:
-    EclGraph(IAgentContext & _agent, const char *_graphName, IConstWorkUnit * _wu, bool _isLibrary, CHThorDebugContext * _debugContext, IProbeManager * _probeManager) :
-                            graphName(_graphName), wu(_wu), debugContext(_debugContext), probeManager(_probeManager)
+    EclGraph(IAgentContext & _agent, const char *_graphName, IConstWorkUnit * _wu, bool _isLibrary, CHThorDebugContext * _debugContext, IProbeManager * _probeManager, unsigned _wfid) :
+                            graphName(_graphName), wu(_wu), debugContext(_debugContext), probeManager(_probeManager), wfid(_wfid)
     {
         isLibrary = _isLibrary;
         graphCodeContext.set(_agent.queryCodeContext());
@@ -1145,7 +1145,8 @@ public:
 
     void associateSubGraph(EclSubGraph * subgraph);
 
-    inline bool queryLibrary() { return isLibrary; }
+    inline bool queryLibrary() const { return isLibrary; }
+    inline unsigned queryWfid() const { return wfid; }
 
 protected:
     IAgentContext * agent;
@@ -1157,6 +1158,7 @@ protected:
     bool isLibrary;
     CHThorDebugContext * debugContext;
     IProbeManager * probeManager;
+    unsigned wfid;
     bool aborted;
 };
 

ecl/eclagent/eclgraph.cpp

@@ -1211,7 +1211,7 @@ void EclGraph::createFromXGMML(ILoadedDllEntry * dll, IPropertyTree * xgmml, boo
 void EclGraph::execute(const byte * parentExtract)
 {
     if (agent->queryRemoteWorkunit())
-        wu->setGraphState(queryGraphName(), WUGraphRunning);
+        wu->setGraphState(queryGraphName(), wfid, WUGraphRunning);
 
     {
         Owned<IWorkUnit> wu(agent->updateWorkUnit());
@@ -1242,12 +1242,12 @@ void EclGraph::execute(const byte * parentExtract)
         }
 
         if (agent->queryRemoteWorkunit())
-            wu->setGraphState(queryGraphName(), WUGraphComplete);
+            wu->setGraphState(queryGraphName(), wfid, WUGraphComplete);
     }
     catch (...)
     {
         if (agent->queryRemoteWorkunit())
-            wu->setGraphState(queryGraphName(), WUGraphFailed);
+            wu->setGraphState(queryGraphName(), wfid, WUGraphFailed);
         throw;
     }
 }
@@ -1306,7 +1306,8 @@ void EclGraph::updateLibraryProgress()
     ForEachItemIn(idx, graphs)
     {
         EclSubGraph & cur = graphs.item(idx);
-        Owned<IWUGraphStats> progress = wu->updateStats(queryGraphName(), queryStatisticsComponentType(), queryStatisticsComponentName(), agent->getWorkflowId(), cur.id);
+        unsigned wfid = cur.parent.queryWfid();
+        Owned<IWUGraphStats> progress = wu->updateStats(queryGraphName(), queryStatisticsComponentType(), queryStatisticsComponentName(), wfid, cur.id);
         cur.updateProgress(progress->queryStatsBuilder());
     }
 }
@@ -1447,9 +1448,9 @@ void GraphResults::setResult(unsigned id, IHThorGraphResult * result)
 
 //---------------------------------------------------------------------------
 
-IWUGraphStats *EclGraph::updateStats(StatisticCreatorType creatorType, const char * creator, unsigned wfid, unsigned subgraph)
+IWUGraphStats *EclGraph::updateStats(StatisticCreatorType creatorType, const char * creator, unsigned activeWfid, unsigned subgraph)
 {
-    return wu->updateStats (queryGraphName(), creatorType, creator, wfid, subgraph);
+    return wu->updateStats (queryGraphName(), creatorType, creator, activeWfid, subgraph);
 }
 
 void EclGraph::updateWUStatistic(IWorkUnit *lockedwu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * descr, unsigned __int64 value)
@@ -1501,7 +1502,7 @@ EclGraph * EclAgent::loadGraph(const char * graphName, IConstWorkUnit * wu, ILoa
 
     bool probeEnabled = wuRead->getDebugValueBool("_Probe", false);
 
-    Owned<EclGraph> eclGraph = new EclGraph(*this, graphName, wu, isLibrary, debugContext, probeManager);
+    Owned<EclGraph> eclGraph = new EclGraph(*this, graphName, wu, isLibrary, debugContext, probeManager, wuGraph->getWfid());
     eclGraph->createFromXGMML(dll, xgmml, probeEnabled);
     return eclGraph.getClear();
 }

ecl/eclcmd/eclcmd_common.hpp

@@ -117,6 +117,9 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_PART_NAME "--part-name"
 #define ECLOPT_PROTECT "--protect"
 #define ECLOPT_USE_EXISTING "--use-existing"
+#define ECLOPT_IGNORE_WARNINGS "--ignore-warnings"
+#define ECLOPT_IGNORE_OPTIONAL "--ignore-optional"
+#define ECLOPT_IGNORE_QUERIES "--ignore-queries"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

ecl/hqlcpp/hqlcpp.cpp

@@ -140,7 +140,7 @@ void SubStringInfo::bindToFrom(HqlCppTranslator & translator, BuildCtx & ctx)
 
 //---------------------------------------------------------------------------
 
-WorkflowItem::WorkflowItem(IHqlExpression * _function) : wfid(0), function(_function), workflowOp(no_funcdef)
+WorkflowItem::WorkflowItem(IHqlExpression * _function) : wfid(999999999), function(_function), workflowOp(no_funcdef)
 {
     IHqlExpression * body = function->queryChild(0);
     assertex(body->getOperator() == no_outofline);
@@ -3844,8 +3844,6 @@ void HqlCppTranslator::buildStmt(BuildCtx & _ctx, IHqlExpression * expr)
         doBuildStmtUpdate(ctx, expr);
         return;
     case no_output:
-        if (queryRealChild(expr, 1))
-            throwError1(HQLERR_NotSupportedInsideNoThor, "OUTPUT to file");
         doBuildStmtOutput(ctx, expr);
         return;
     case no_subgraph:

ecl/hqlcpp/hqlcpp.ipp

@@ -2017,6 +2017,8 @@ public:
     inline unsigned curGraphSequence() const { return activeGraph ? graphSeqNumber : 0; }
     UniqueSequenceCounter & querySpillSequence() { return spillSequence; }
     unsigned nextLibrarySequence() { return librarySequence++; }
+    unsigned queryMaxWfid() { return maxWfid; }
+    void setMaxWfid(unsigned wfid) { maxWfid = wfid; }
 
 public:
     void traceExpression(const char * title, IHqlExpression * expr, unsigned level=500);
@@ -2047,6 +2049,7 @@ protected:
     HqlCppDerived       derived;
     unsigned            activitiesThisCpp;
     unsigned            curCppFile;
+    unsigned            maxWfid = 0;
     Linked<ICodegenContextCallback> ctxCallback;
     ClusterType         targetClusterType;
     bool contextAvailable;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5977,9 +5977,11 @@ bool HqlCppTranslator::buildCode(HqlQueryContext & query, const char * embeddedL
                 WorkflowItem & cur = workflow.item(i);
                 if (!cur.isFunction())
                 {
-                    assertex(!graph);
                     HqlExprArray & exprs = cur.queryExprs();
+                    assertex(!graph);
                     assertex(exprs.ordinality() == 1);
+
+                    curWfid = cur.queryWfid();
                     graph.set(&exprs.item(0));
                     assertex(graph->getOperator() == no_thor);
                 }
@@ -11670,6 +11672,9 @@ void HqlCppTranslator::doBuildStmtOutput(BuildCtx & ctx, IHqlExpression * expr)
     if (expr->hasAttribute(groupedAtom) && (dataset->getOperator() != no_null))
         throwError1(HQLERR_NotSupportedInsideNoThor, "Grouped OUTPUT");
 
+    if (queryRealChild(expr, 1))
+        throwError1(HQLERR_NotSupportedInsideNoThor, "OUTPUT to file");
+
     LinkedHqlExpr seq = querySequence(expr);
     LinkedHqlExpr name = queryResultName(expr);
     assertex(seq != NULL);
@@ -18555,6 +18560,7 @@ void HqlCppTranslator::buildWorkflow(WorkflowArray & workflow)
 
         if (!isEmpty)
         {
+            curWfid = wfid;
             if (action.isFunction())
             {
                 OwnedHqlExpr function = action.getFunction();
@@ -18565,7 +18571,6 @@ void HqlCppTranslator::buildWorkflow(WorkflowArray & workflow)
                 OwnedHqlExpr expr = createActionList(action.queryExprs());
 
                 IHqlExpression * persistAttr = expr->queryAttribute(_workflowPersist_Atom);
-                curWfid = wfid;
                 if (persistAttr)
                 {
                     if (!options.freezePersists)
@@ -18578,8 +18583,8 @@ void HqlCppTranslator::buildWorkflow(WorkflowArray & workflow)
                 }
                 else
                     buildWorkflowItem(switchctx, switchStmt, wfid, expr);
-                curWfid = 0;
             }
+            curWfid = 0;
         }
     }
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -7252,7 +7252,7 @@ void WorkflowTransformer::analyseAll(const HqlExprArray & in)
 
 void WorkflowTransformer::transformRoot(const HqlExprArray & in, WorkflowArray & out)
 {
-    wfidCount = 0;
+    wfidCount = translator.queryMaxWfid();
     HqlExprArray transformed;
     WorkflowTransformInfo globalInfo(NULL);
     ForEachItemIn(idx, in)
@@ -7316,6 +7316,7 @@ void WorkflowTransformer::transformRoot(const HqlExprArray & in, WorkflowArray &
 
     appendArray(out, workflow);
     appendArray(out, functions);
+    translator.setMaxWfid(wfidCount);
 }
 
 void extractWorkflow(HqlCppTranslator & translator, HqlExprArray & exprs, WorkflowArray & out)

ecl/regress/issue22259.ecl

@@ -0,0 +1,24 @@
+import Std.Str;
+
+rec := RECORD
+ string f1;
+END;
+ds := DATASET([{'a'},{'b'}], rec);
+
+dofunc(string l) := FUNCTION
+ o1 := OUTPUT(ds, , PIPE('/home/jsmith/tmp/pipecmd'));
+ RETURN WHEN(true,o1,BEFORE);
+END;
+rec mytrans(rec l) := TRANSFORM
+ SELF.f1 := IF(dofunc(l.f1), 'a', 'b');
+END;
+rec myfailtrans(STRING x) := TRANSFORM
+ SELF.f1 := x;
+END;
+
+p2 := PROJECT(ds, mytrans(LEFT));
+c2 := CATCH(NOFOLD(p2), ONFAIL(myFailTrans(FAILMESSAGE)));
+
+SEQUENTIAL(
+ OUTPUT(c2);
+);

esp/esdllib/esdl_transformer2.cpp

@@ -1381,7 +1381,8 @@ void Esdl2Response::processChildNamedResponse(Esdl2TransformerContext &ctx, cons
                 if (ctx.do_output_ns)
                 {
                     ctx.writer->outputBeginNested(out_name, true);
-                    ctx.writer->outputXmlns(out_name, ctx.ns.str());
+                    // This ensures the namespace prefix is just 'xmlns'
+                    ctx.writer->outputXmlns("xmlns", ctx.ns.str());
                     ctx.do_output_ns=false;
                 }
                 else

esp/scm/ws_packageprocess.ecm

@@ -189,8 +189,11 @@ ESPrequest ValidatePackageRequest
     string PMID;
     string QueryIdToVerify;
     ESParray<string> QueriesToVerify;
+    ESParray<string> QueriesToIgnore;
     bool CheckDFS;
     bool GlobalScope(0);
+    bool IgnoreWarnings(0);
+    bool IgnoreOptionalFiles(0);
 };
 
 ESPstruct ValidatePackageInfo

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -2106,8 +2106,7 @@ int EsdlBindingImpl::HandleSoapRequest(CHttpRequest* request,
             Owned<IPropertyTree> tgtcfg;
             Owned<IPropertyTree> tgtctx;
 
-            ns.clear();
-            generateNamespace(*ctx, request, srvdef->queryName(), mthdef->queryName(), ns);
+            // Echo back the reqeust namespace, don't generate it here
             getSchemaLocation(*ctx, request, schemaLocation);
 
             ctx->setESDLBindingID(m_bindingId.get());

esp/services/ws_dfu/ws_dfuService.cpp

@@ -6370,13 +6370,6 @@ bool CWsDfuEx::onDFUFileCreateV2(IEspContext &context, IEspDFUFileCreateV2Reques
         tempFileName.append(".").append(dfuCreateUniqId++); // avoid potential clash if >1 creating file. One will succeed at publish time.
         tempFileName.append(DFUFileCreate_FileNamePostfix);
 
-        //create FileId
-        StringBuffer fileId;
-        fileId.set(groupName).append(DFUFileIdSeparator).append(clusterName).append(DFUFileIdSeparator).append(tempFileName);
-        if (req.getCompressed())
-            fileId.append(DFUFileIdSeparator).append("true");
-        resp.setFileId(fileId.str());
-
         if (requestId.isEmpty())
             requestId.appendf("Create %s on %s", normalizedFileName.str(), clusterName);
 
@@ -6386,26 +6379,36 @@ bool CWsDfuEx::onDFUFileCreateV2(IEspContext &context, IEspDFUFileCreateV2Reques
             fileDesc->queryProperties().setProp("@owner", userId);
         fileDesc->queryProperties().setProp("ECL", recordDefinition);
 
+        const char *fileType = nullptr;
         CDFUFileType kind = req.getType();
         switch (kind)
         {
             case CDFUFileType_Flat:
-                fileDesc->queryProperties().setProp("@kind", "flat");
+                fileType = "flat";
                 break;
             case CDFUFileType_Csv:
-                fileDesc->queryProperties().setProp("@kind", "csv");
+                fileType = "csv";
                 break;
             case CDFUFileType_Xml:
-                fileDesc->queryProperties().setProp("@kind", "xml");
+                fileType = "xml";
                 break;
             case CDFUFileType_Json:
-                fileDesc->queryProperties().setProp("@kind", "json");
+                fileType = "json";
                 break;
             case CDFUFileType_Index:
-                fileDesc->queryProperties().setProp("@kind", "key");
+                fileType = "key";
             default:
                 throw makeStringExceptionV(ECLWATCH_MISSING_FILETYPE, "DFUFileCreateV2: File type not provided");
         }
+        fileDesc->queryProperties().setProp("@kind", fileType);
+
+        //create FileId
+        StringBuffer fileId;
+        fileId.set(groupName).append(DFUFileIdSeparator).append(clusterName).append(DFUFileIdSeparator).append(tempFileName);
+        fileId.append(DFUFileIdSeparator).append(boolToStr(req.getCompressed()));
+        fileId.append(DFUFileIdSeparator).append(fileType);
+        resp.setFileId(fileId.str());
+
 
         MemoryBuffer layoutBin;
         exportRecordDefinitionBinaryType(recordDefinition, layoutBin);
@@ -6468,9 +6471,6 @@ bool CWsDfuEx::onDFUFilePublish(IEspContext &context, IEspDFUFilePublishRequest
         const char *groupName = fileIdItems.item(0);
         const char *clusterName = fileIdItems.item(1);
         const char *tempFileName = fileIdItems.item(2);
-        bool compressed = false;
-        if (fileIdItems.ordinality()>3)
-            compressed = strToBool(fileIdItems.item(3));
         if (isEmptyString(groupName))
              throw makeStringException(ECLWATCH_INVALID_INPUT, "DFUFilePublish: Invalid FileId: empty groupName.");
         if (isEmptyString(clusterName))
@@ -6507,8 +6507,14 @@ bool CWsDfuEx::onDFUFilePublish(IEspContext &context, IEspDFUFilePublishRequest
                 throw makeStringExceptionV(ECLWATCH_FILE_NOT_EXIST, "DFUFilePublish: Failed to find group %s.", groupName);
 
             fileDesc.setown(createFileDescriptor(normalizeTempFileName, clusterTypeEx, groupName, group));
-            if (compressed)
-                fileDesc->queryProperties().setPropBool("@blockCompressed", true);
+            if (fileIdItems.ordinality()>3) // compressed
+            {
+                bool compressed = strToBool(fileIdItems.item(3));
+                if (compressed)
+                    fileDesc->queryProperties().setPropBool("@blockCompressed", true);
+                if (fileIdItems.ordinality()>4) // fileType
+                    fileDesc->queryProperties().setProp("@kind", fileIdItems.item(4));
+            }
         }
 
         StringBuffer newFileName(normalizeTempFileName);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -1165,10 +1165,18 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
         if (queryid && *queryid)
             queriesToVerify.appendUniq(queryid);
     }
-    map->validate(queriesToVerify, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles);
+    StringArray queriesToIgnore;
+    ForEachItemIn(i2, req.getQueriesToIgnore())
+    {
+        queryid = req.getQueriesToIgnore().item(i2);
+        if (queryid && *queryid)
+            queriesToIgnore.appendUniq(queryid);
+    }
+    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
 
     resp.setPMID(map->queryPackageId());
-    resp.setWarnings(warnings);
+    if (!req.getIgnoreWarnings())
+        resp.setWarnings(warnings);
     resp.setErrors(errors);
     resp.updateQueries().setUnmatched(unmatchedQueries);
     resp.updatePackages().setUnmatched(unusedPackages);

esp/src/eclwatch/TargetClustersQueryWidget.js

@@ -129,7 +129,7 @@ define([
                         renderHeaderCell: function (node) {
                             node.innerHTML = Utility.getImageHTML("configuration.png", context.i18n.Configuration);
                         },
-                        width: 10,
+                        width: 8,
                         sortable: false,
                         formatter: function (configuration) {
                             if (configuration === true) {
@@ -138,6 +138,20 @@ define([
                             return "";
                         }
                     },
+                    DaliServer: {
+                        label: this.i18n.Dali,
+                        renderHeaderCell: function (node) {
+                            node.innerHTML = Utility.getImageHTML("server.png", context.i18n.Dali);
+                        },
+                        width: 8,
+                        sortable: false,
+                        formatter: function (dali) {
+                            if (dali === true) {
+                                return Utility.getImageHTML("server.png", context.i18n.Dali);
+                            }
+                            return "";
+                        }
+                    },
                     Name: tree({
                         formatter: function (_name, row) {
                             var img = "";
@@ -150,7 +164,6 @@ define([
                         expand: true,
                         label: this.i18n.Name,
                         collapseOnRefresh: false,
-                        sortable: true,
                         width: 150,
                         shouldExpand: function (clusterProcess) {
                             if (clusterProcess.data.type === "targetClusterProcess") {
@@ -244,7 +257,7 @@ define([
         _onRowDblClick: function (item) {
             var nodeTab = this.ensureLogsPane(item.Name, {
                 params: item,
-                LogDirectory: item.LogDirectory,
+                LogDirectory: item.LogDirectory !== undefined ? item.LogDirectory : item.LogDir, // in the case of dali nested log
                 NetAddress: item.Netaddress,
                 OS: item.OS,
                 newPreflight: true

esp/src/eclwatch/nls/bs/hpcc.js

@@ -132,6 +132,8 @@
     Dali: "Dali",
     DaliIP: "Dali IP adresa",
     DataPatterns: "Uzorci Podataka",
+    DataPatternsNotStarted: "Rezultati analize nisu pronađeni. Za početak nove analize pritisnite dugme Analiza.",
+    DataPatternsStarted: "Analiza je u toku. Kada završi, ovdje će biti prikazan kompletan izvještaj.",
     dataset: ":=dataset*",
     Date: "Datum",
     Day: "Dan",
@@ -407,7 +409,8 @@
     MaxNode: "Maksimalni Nod/čvor",
     MaxRecordLength: "Maksimalna Dužina Rekorda",
     MaxSize: "Maksimalna Veličina",
-	MaxSkew: "Maksimalni Skju (Distorzija)",
+    MaxSkew: "Maksimalni Skju (Distorzija)",
+    MaximizeRestore: "Maksimiziraj/Povrati prethodno stanje",
     Mean: "Srednja Vrijednost",
     MemberOf: "Član Od",
     Members: "Članovi",

esp/src/eclwatch/nls/es/hpcc.js

@@ -132,6 +132,8 @@ define(
     Dali: "Dali",
     DaliIP: "IP de Dali",
     DataPatterns: "Patrones de datos",
+    DataPatternsNotStarted: "Análisis no se encuntra. Para comenzar, oprima el botón Analizar encima.",
+    DataPatternsStarted: "Analizando. Cuando el reporte este completo se mostrará aquí.",
     dataset: ":=dataset*",
     Date: "Fecha",
     Day: "Dia",
@@ -403,6 +405,7 @@ define(
     Mappings: "Mapeados",
     Mask: "Máscara",
     Max: "Máx",
+    MaximizeRestore: "Maximizar / Restaurar",
     MaximumNumberOfSlaves: "Numero de esclavo",
     MaxNode: "Nodo máximo",
     MaxRecordLength: "Máx largo de registros",

esp/src/eclwatch/nls/hr/hpcc.js

@@ -132,6 +132,8 @@
     Dali: "Dali",
     DaliIP: "Dali IP Adresa",
     DataPatterns: "Uzorci podataka",
+    DataPatternsNotStarted: "Rezultati analize nisu pronađeni. Za početak nove analize pritisnite dugme Analiza.",
+    DataPatternsStarted: "Analiza je u toku. Po završetku kompletan izveštaj će biti ovdje prikazan.",
     dataset: ":=dataset*",
     Date: "Datum",
     Day: "Dan",

esp/src/eclwatch/nls/hu/hpcc.js

@@ -132,6 +132,8 @@ define(
     Dali: "Dali",
     DaliIP: "Dali IP címe",
     DataPatterns: "Adat minták",
+    DataPatternsNotStarted: "Az analízis eredménye nem található. Létrehozásához nyomja meg a fent lévő “Analízis” gombot.",
+    DataPatternsStarted: "Analizálás folyamatban. A folyamat végeztével az eredmény itt lesz látható.",
     dataset: ":=dataset*",
     Date: "Dátum",
     Day: "Nap",

esp/src/eclwatch/nls/pt-br/hpcc.js

@@ -132,6 +132,8 @@
     Dali: "Dali",
     DaliIP: "IP do Dali",
     DataPatterns: "Padrões de Dados",
+    DataPatternsNotStarted: "Análise não encontrada. Para começar, pressione o botão Analisar acima.",
+    DataPatternsStarted: "Analisando. Quando o relatório for completo, vai ser exibido aqui.",
     dataset: ":=dataset*",
     Date: "Data",
     Day: "Dia",
@@ -404,6 +406,7 @@
     Mappings: "Mapeamento",
     Mask: "Mask (bloqueador de caracteres)",
     Max: "Max",
+    MaximizeRestore: "Maximizar/Restaurar",
     MaximumNumberOfSlaves: "Numero de esclavo",
     MaxNode: "Nó Max",
     MaxRecordLength: "Maximo tamanho do registro",

esp/src/eclwatch/nls/sr/hpcc.js

@@ -133,6 +133,8 @@
     Dali: "Дали",
     DaliIP: "Дали ИП адресa",
     DataPatterns: "Узорци података",
+    DataPatternsNotStarted: "Резултати анализе нису пронађени. Да бисте започели нову анализу, притисните дугме Анализа.",
+    DataPatternsStarted: "Анализа је у току. Када заврши, овде ће бити приказан цео извештај.",
     dataset: ":=датасет*",
     Date: "Датум",
     Day: "Дан",

esp/src/eclwatch/nls/zh/hpcc.js

@@ -132,6 +132,8 @@
     Dali: "Dali",
     DaliIP: "Dali IP",
     DataPatterns: "数据模式",
+    DataPatternsNotStarted: "分析无法找到。开始请按上面的分析键",
+    DataPatternsStarted: "正在分析。完成后报告会在这里显示。",
     dataset: ":=数据集*",
     Date: "日期",
     Day: "日",

esp/src/src/ESPPreflight.ts

@@ -126,7 +126,9 @@ var ClusterTargetStore = declare([ESPRequest.Store], {
                 hpcc_id: parent.Name + "_" + item.Name,
                 Name: item.Type + " - " +  item.Name,
                 Type: item.Type,
+                DaliServer: item.DaliServer ? true : false,
                 Directory: item.TpMachines ? item.TpMachines.TpMachine[0].Directory : "",
+                LogDir: item.LogDir,
                 LogDirectory: item.LogDirectory,
                 OS: item.TpMachines.TpMachine[0].OS,
                 Platform: item.TpMachines ? context.getOS(item.TpMachines.TpMachine[0].OS) : "",

fs/dafsserver/dafsserver.cpp

@@ -38,6 +38,8 @@
 #include "jset.hpp"
 #include "jhtree.hpp"
 
+#include "dadfs.hpp"
+
 #include "remoteerr.hpp"
 #include <atomic>
 #include <string>
@@ -2522,7 +2524,7 @@ IRemoteActivity *createRemoteActivity(IPropertyTree &actNode, bool authorizedOnl
                 else
                     activity.setown(new CRemoteIndexReadActivity(actNode, fileDesc));
             }
-            else // flat file
+            else
             {
                 if (!isEmptyString(action))
                 {
@@ -2532,7 +2534,19 @@ IRemoteActivity *createRemoteActivity(IPropertyTree &actNode, bool authorizedOnl
                         throw createDafsExceptionV(DAFSERR_cmdstream_protocol_failure, "Unknown action '%s' on flat file '%s'", action, partFileName);
                 }
                 else
-                    activity.setown(new CRemoteDiskReadActivity(actNode, fileDesc));
+                {
+                    const char *kind = queryFileKind(fileDesc);
+                    if (isEmptyString(kind) || (streq("flat", kind)))
+                        activity.setown(new CRemoteDiskReadActivity(actNode, fileDesc));
+                    else if (streq("csv", kind))
+                        activity.setown(createConditionalProjectingActivity<CRemoteCsvReadActivity>(actNode, fileDesc));
+                    else if (streq("xml", kind))
+                        activity.setown(createConditionalProjectingActivity<CRemoteXmlReadActivity>(actNode, fileDesc));
+                    else if (streq("json", kind))
+                        activity.setown(createConditionalProjectingActivity<CRemoteJsonReadActivity>(actNode, fileDesc));
+                    else
+                        throw createDafsExceptionV(DAFSERR_cmdstream_protocol_failure, "Unknown file kind '%s'", kind);
+                }
             }
             break;
         }
@@ -3193,7 +3207,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
     CClientStatsTable clientStatsTable;
     atomic_t globallasttick;
     unsigned targetActiveThreads;
-    Owned<IPropertyTree> keyPairInfo;
+    Linked<IPropertyTree> keyPairInfo;
 
     int getNextHandle()
     {
@@ -4932,7 +4946,7 @@ public:
                     {
                         EXCLOG(e,"CRemoteFileServer");
                         e->Release();
-                        break;
+                        continue;
                     }
                 }
 

initfiles/etc/DIR_NAME/environment.xml.in

@@ -945,9 +945,9 @@
   <path>${INSTALL_DIR}</path>
   <Keys>
    <KeyPair name="mythor" publicKey="@HOME_DIR@/@RUNTIME_USER@/certificate/public.key.pem" privateKey="@HOME_DIR@/@RUNTIME_USER@/certificate/key.pem"/>
-   <Cluster name="mythor" keyPairName="mythor"/>
-   <Cluster name="myroxie" keyPairName="mythor"/>
-   <Cluster name="hthor__myeclagent" keyPairName="mythor"/>
+   <ClusterGroup name="mythor" keyPairName="mythor"/>
+   <ClusterGroup name="myroxie" keyPairName="mythor"/>
+   <ClusterGroup name="hthor__myeclagent" keyPairName="mythor"/>
   </Keys>
  </EnvSettings>
  <Programs>

plugins/cassandra/cassandrawu.cpp

@@ -2623,7 +2623,7 @@ public:
         else
             return WUGraphUnknown;
     }
-    void setGraphState(const char *graphName, WUGraphState state) const
+    void setGraphState(const char *graphName, unsigned wfid, WUGraphState state) const
     {
         setNodeState(graphName, 0, state);
     }
@@ -3324,7 +3324,7 @@ public:
                         }
                     }
                     if (graph.hasProp("@_state"))
-                        wu->setGraphState(graphName, (WUGraphState) graph.getPropInt("@_state"));
+                        wu->setGraphState(graphName, graph.getPropInt("@wfid"), (WUGraphState) graph.getPropInt("@_state"));
                 }
             }
             wu->commit();

roxie/ccd/ccdstate.cpp

@@ -823,9 +823,9 @@ public:
     {
         return BASE::getPartIds();
     }
-    virtual bool validate(StringArray &queryids, StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
+    virtual bool validate(const StringArray &queryids, const StringArray &queriesToIgnore, StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const
     {
-        return BASE::validate(queryids, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles);
+        return BASE::validate(queryids, queriesToIgnore, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles, ignoreOptionalFiles);
     }
     virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const
     {

system/jlib/jsocket.cpp

@@ -24,6 +24,10 @@
     look at loopback
 */
 
+#include <string>
+#include <unordered_set>
+#include <functional>
+
 #include "platform.h"
 #ifdef _VER_C5
 #include <clwclib.h>
@@ -417,7 +421,7 @@ public:
     bool        connectionless() { return (sockmode!=sm_tcp)&&(sockmode!=sm_tcp_server); }
     void        shutdown(unsigned mode=SHUTDOWN_READWRITE);
 
-    ISocket*    accept(bool allowcancel);
+    ISocket*    accept(bool allowcancel, SocketEndpoint *peerEp=nullptr);
     int         wait_read(unsigned timeout);
     int         logPollError(unsigned revents, const char *rwstr);
     int         wait_write(unsigned timeout);
@@ -1010,7 +1014,7 @@ ErrPortInUse:
 
 
 
-ISocket* CSocket::accept(bool allowcancel)
+ISocket* CSocket::accept(bool allowcancel, SocketEndpoint *peerEp)
 {
     if ((accept_cancel_state!=accept_not_cancelled) && allowcancel) {
         accept_cancel_state=accept_cancelled;
@@ -1023,10 +1027,14 @@ ISocket* CSocket::accept(bool allowcancel)
     if (connectionless()) {
         THROWJSOCKEXCEPTION(JSOCKERR_connectionless_socket);
     }
+
+    DEFINE_SOCKADDR(peerSockAddr);      // used if peerIp
+    socklen_t peerSockAddrLen = sizeof(peerSockAddr);
+
     T_SOCKET newsock;
     for (;;) {
         in_accept = true;
-        newsock = (sock!=INVALID_SOCKET)?::accept(sock, NULL, NULL):INVALID_SOCKET;
+        newsock = (sock!=INVALID_SOCKET)?::accept(sock, &peerSockAddr.sa, &peerSockAddrLen):INVALID_SOCKET;
         in_accept = false;
     #ifdef SOCKTRACE
         PROGLOG("SOCKTRACE: accept created socket %x %d (%p)", newsock,newsock,this);
@@ -1061,6 +1069,10 @@ ISocket* CSocket::accept(bool allowcancel)
             return NULL;
         THROWJSOCKEXCEPTION(JSOCKERR_cancel_accept);
     }
+
+    if (peerEp)
+        getSockAddrEndpoint(peerSockAddr, peerSockAddrLen, *peerEp);
+
     CSocket *ret = new CSocket(newsock,sm_tcp,true);
     ret->set_inherit(false);
     return ret;
@@ -6746,7 +6758,7 @@ int wait_multiple(bool isRead,               //IN   true if wait read, false it
 #ifdef _USE_SELECT
             if (FD_ISSET(s, &fds))
 #else
-            if ( (fds[idx].revents) && (!(fds[idx].revents & POLLNVAL)) )
+            if (fds[idx].revents)
 #endif
             {
 #ifdef _DEBUG
@@ -6760,6 +6772,7 @@ int wait_multiple(bool isRead,               //IN   true if wait read, false it
 #ifdef _DEBUG
         DBGLOG("%s",dbgSB.str());
 #endif
+        res = readySocks.ordinality();
     }
     else if (res == SOCKET_ERROR)
     {
@@ -6794,3 +6807,140 @@ int wait_write_multiple(UnsignedArray &socks,       //IN   sockets to be checked
 {
     return wait_multiple(false, socks, timeoutMS, readySocks);
 }
+
+
+class CWhiteListHandler : public CSimpleInterfaceOf<IWhiteListHandler>, implements IWhiteListWriter
+{
+    typedef CSimpleInterfaceOf<IWhiteListHandler> PARENT;
+
+    struct PairHasher
+    {
+        template <class T1, class T2>
+        std::size_t operator () (std::pair<T1, T2> const &pair) const
+        {
+            std::size_t h1 = std::hash<T1>()(pair.first);
+            std::size_t h2 = std::hash<T2>()(pair.second);
+            return h1 ^ h2;
+        }
+    };
+
+    using WhiteListHT = std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher>;
+    WhiteListPopulateFunction populateFunc;
+    WhiteListFormatFunction roleFormatFunc;
+    std::unordered_set<std::pair<std::string, unsigned __int64>, PairHasher> whiteList;
+    std::unordered_set<std::string> IPOnlyWhiteList;
+    bool allowAnonRoles = false;
+    mutable CriticalSection populatedCrit;
+    mutable bool populated = false;
+    mutable bool enabled = true;
+
+    void ensurePopulated() const
+    {
+        // should be called within CS
+        if (populated)
+            return;
+        // NB: want to keep this method const, as used by isXX functions that are const, but if need to refresh it's effectively mutable
+        enabled = populateFunc(* const_cast<IWhiteListWriter *>((const IWhiteListWriter *)this));
+        populated = true;
+    }
+public:
+    IMPLEMENT_IINTERFACE_O_USING(PARENT);
+
+    CWhiteListHandler(WhiteListPopulateFunction _populateFunc, WhiteListFormatFunction _roleFormatFunc) : populateFunc(_populateFunc), roleFormatFunc(_roleFormatFunc)
+    {
+    }
+// IWhiteListHandler impl.
+    virtual bool isWhiteListed(const char *ip, unsigned __int64 role, StringBuffer *responseText) const override
+    {
+        CriticalBlock block(populatedCrit);
+        ensurePopulated();
+        if (0 == role) // unknown, can only check ip
+        {
+            if (allowAnonRoles)
+            {
+                const auto &it = IPOnlyWhiteList.find(ip);
+                if (it != IPOnlyWhiteList.end())
+                    return true;
+            }
+        }
+        else
+        {
+            const auto &it = whiteList.find({ip, role});
+            if (it != whiteList.end())
+                return true;
+        }
+
+        // if !enabled and no responseText supplied, generate response and warn that disabled
+        StringBuffer disabledResponseText;
+        if (!enabled && !responseText)
+            responseText = &disabledResponseText;
+
+        if (responseText)
+        {
+            responseText->append("Access denied! [client ip=");
+            responseText->append(ip);
+            if (role)
+            {
+                responseText->append(", role=");
+                if (roleFormatFunc)
+                    roleFormatFunc(*responseText, role);
+                else
+                    responseText->append(role);
+            }
+            responseText->append("] not whitelisted");
+        }
+
+        if (enabled)
+            return false;
+        else
+        {
+            OWARNLOG("WhiteListing is disabled, ignoring: %s", responseText->str());
+            return true;
+        }
+    }
+    virtual StringBuffer &getWhiteList(StringBuffer &out) const override
+    {
+        CriticalBlock block(populatedCrit);
+        ensurePopulated();
+        for (const auto &it: whiteList)
+        {
+            out.append(it.first.c_str()).append(", ");
+            if (roleFormatFunc)
+                roleFormatFunc(out, it.second);
+            else
+                out.append(it.second);
+            out.newline();
+        }
+        return out;
+    }
+    virtual void refresh() override
+    {
+        /* NB: clear only, so that next usage will re-populated
+         * Do not want to repopulate now, because refresh() is likely called within a update write transaction
+         */
+        CriticalBlock block(populatedCrit);
+        enabled = true;
+        whiteList.clear();
+        IPOnlyWhiteList.clear();
+        populated = false;
+    }
+// IWhiteListWriter impl.
+    virtual void add(const char *ip, unsigned __int64 role) override
+    {
+        // NB: called via populateFunc, which is called whilst populatedCrit is locked.
+        whiteList.insert({ ip, role });
+        if (allowAnonRoles)
+            IPOnlyWhiteList.insert(ip);
+    }
+    virtual void setAllowAnonRoles(bool tf) override
+    {
+        allowAnonRoles = tf;
+        refresh();
+    }
+};
+
+IWhiteListHandler *createWhiteListHandler(WhiteListPopulateFunction populateFunc, WhiteListFormatFunction roleFormatFunc)
+{
+    return new CWhiteListHandler(populateFunc, roleFormatFunc);
+}
+

system/jlib/jsocket.hpp

@@ -276,7 +276,7 @@ public:
     //
     // This method is called by server to accept client connection
     //
-    virtual ISocket* accept(bool allowcancel=false) = 0; // not needed for UDP
+    virtual ISocket* accept(bool allowcancel=false, SocketEndpoint *peerEp = nullptr) = 0; // not needed for UDP
 
     //
     // log poll() errors
@@ -629,5 +629,23 @@ extern jlib_decl int wait_write_multiple(UnsignedArray  &socks,     //IN   socke
 extern jlib_decl void throwJSocketException(int jsockErr);
 extern jlib_decl IJSOCK_Exception* createJSocketException(int jsockErr, const char *_msg);
 
+interface IWhiteListHandler : extends IInterface
+{
+    virtual bool isWhiteListed(const char *ip, unsigned __int64 role, StringBuffer *responseText=nullptr) const = 0;
+    virtual StringBuffer &getWhiteList(StringBuffer &out) const = 0;
+    virtual void refresh() = 0;
+};
+
+interface IWhiteListWriter : extends IInterface
+{
+    virtual void add(const char *ip, unsigned __int64 role) = 0;
+    virtual void setAllowAnonRoles(bool tf) = 0;
+};
+
+typedef std::function<bool(IWhiteListWriter &)> WhiteListPopulateFunction;
+typedef std::function<StringBuffer &(StringBuffer &, unsigned __int64)> WhiteListFormatFunction;
+extern jlib_decl IWhiteListHandler *createWhiteListHandler(WhiteListPopulateFunction populateFunc, WhiteListFormatFunction roleFormatFunc = {}); // format function optional
+
+
 #endif
 

system/mp/mpbuff.hpp

@@ -43,7 +43,7 @@ public:
     inline const SocketEndpoint &getSender() const  { return sender; }
     inline void setReplyTag(mptag_t tag)            { replytag = tag; }  // called prior to send (use cresteReplyTag to make tag)
     inline mptag_t getReplyTag()                    { return replytag; } // called after recv to determine tag to reply to
-    inline mptag_t getTag()                     { return tag; }      
+    inline mptag_t getTag()                         { return tag; }
 
     inline void init()             
     { 

system/mp/mpcomm.cpp

@@ -438,7 +438,9 @@ class CMPConnectThread: public Thread
     CMPServer *parent;
     int mpSoMaxConn;
     unsigned mpTraceLevel;
+    Owned<IWhiteListHandler> whiteListCallback;
     void checkSelfDestruct(void *p,size32_t sz);
+
 public:
     CMPConnectThread(CMPServer *_parent, unsigned port);
     ~CMPConnectThread()
@@ -456,6 +458,14 @@ public:
                 printf("CMPConnectThread::stop timed out\n");
         }
     }
+    void installWhiteListCallback(IWhiteListHandler *_whiteListCallback)
+    {
+        whiteListCallback.set(_whiteListCallback);
+    }
+    IWhiteListHandler *queryWhiteListCallback() const
+    {
+        return whiteListCallback;
+    }
 };
 
 class PingPacketHandler;
@@ -476,6 +486,7 @@ class CMPServer: private CMPChannelHT, implements IMPServer
     CMPNotifyClosedThread       *notifyclosedthread;
     CriticalSection sect;
 protected:
+    unsigned __int64            role;
     unsigned short              port;
 public:
     bool checkclosed;
@@ -491,11 +502,12 @@ public:
 
     IMPLEMENT_IINTERFACE_USING(CMPChannelHT);
 
-    CMPServer(unsigned _port);
+    CMPServer(unsigned __int64 _role, unsigned _port);
     ~CMPServer();
     void start();
     virtual void stop();
-    unsigned short getPort() { return port; }
+    unsigned short getPort() const { return port; }
+    unsigned __int64 getRole() const { return role; }
     void setPort(unsigned short _port) { port = _port; }
     CMPChannel *lookup(const SocketEndpoint &remoteep);
     ISocketSelectHandler &querySelectHandler() { return *selecthandler; };
@@ -569,6 +581,14 @@ public:
                 break;
         }
     }
+    virtual void installWhiteListCallback(IWhiteListHandler *whiteListCallback) override
+    {
+        connectthread->installWhiteListCallback(whiteListCallback);
+    }
+    virtual IWhiteListHandler *queryWhiteListCallback() const override
+    {
+        return connectthread->queryWhiteListCallback();
+    }
 };
 
 //===========================================================================
@@ -708,6 +728,42 @@ void traceSlowReadTms(const char *msg, ISocket *sock, void *dst, size32_t minSiz
     }
 }
 
+/* Legacy header sent id[2] only.
+ * To remain backward compatible (when new MP clients are connecting to old Dali),
+ * we send a regular empty PacketHeader as well that has the 'role' embedded within it,
+ * in unused fields. TAG_SYS_BCAST is used as the message tag, because it is an
+ * unused feature that all Dali's simply receive and delete.
+ */
+struct ConnectHdr
+{
+    ConnectHdr(const SocketEndpoint &hostEp, const SocketEndpoint &remoteEp, unsigned __int64 role)
+    {
+        id[0].set(hostEp);
+        id[1].set(remoteEp);
+
+        hdr.size = sizeof(PacketHeader);
+        hdr.tag = TAG_SYS_BCAST;
+        hdr.flags = 0;
+        hdr.version = MP_PROTOCOL_VERSION;
+        setRole(role);
+    }
+    ConnectHdr()
+    {
+    }
+    SocketEndpointV4 id[2];
+    PacketHeader hdr;
+    inline void setRole(unsigned __int64 role)
+    {
+        hdr.replytag = (mptag_t) (role >> 32);
+        hdr.sequence = (unsigned) (role & 0xffffffff);
+    }
+    inline unsigned __int64 getRole() const
+    {
+        return (((unsigned __int64)hdr.replytag)<<32) | ((unsigned __int64)hdr.sequence);
+    }
+};
+
+
 class CMPPacketReader;
 
 class CMPChannel: public CInterface
@@ -755,19 +811,23 @@ protected: friend class CMPPacketReader;
         // must be called from connectsect
         // also in sendmutex
 
-        ISocket *newsock=NULL;
+        Owned<ISocket> newsock;
         unsigned retrycount = CONNECT_RETRYCOUNT;
         unsigned remaining;
+        Owned<IException> exitException;
 
-        while (!channelsock) {
-            try {
+        while (!channelsock)
+        {
+            try
+            {
                 StringBuffer str;
 #ifdef _TRACE
                 LOG(MCdebugInfo(100), unknownJob, "MP: connecting to %s",remoteep.getUrlStr(str).str());
 #endif
                 if (((int)tm.timeout)<0)
                     remaining = CONNECT_TIMEOUT;
-                else if (tm.timedout(&remaining)) {
+                else if (tm.timedout(&remaining))
+                {
 #ifdef _FULLTRACE
                     PROGLOG("MP: connect timed out 1");
 #endif
@@ -775,34 +835,31 @@ protected: friend class CMPPacketReader;
                 }
                 if (remaining<10000)
                     remaining = 10000; // 10s min granularity for MP
-                newsock = ISocket::connect_timeout(remoteep,remaining);
+                newsock.setown(ISocket::connect_timeout(remoteep,remaining));
                 newsock->set_keep_alive(true);
 #ifdef _FULLTRACE
                 LOG(MCdebugInfo(100), unknownJob, "MP: connect after socket connect, retrycount = %d", retrycount);
 #endif
 
-                SocketEndpointV4 id[2];
                 SocketEndpoint hostep;
                 hostep.setLocalHost(parent->getPort());
-                id[0].set(hostep);
-                id[1].set(remoteep);
+                ConnectHdr connectHdr(hostep, remoteep, parent->getRole());
 
-                unsigned __int64 addrval = DIGIT1*id[0].ip[0] + DIGIT2*id[0].ip[1] + DIGIT3*id[0].ip[2] + DIGIT4*id[0].ip[3] + id[0].port;
+                unsigned __int64 addrval = DIGIT1*connectHdr.id[0].ip[0] + DIGIT2*connectHdr.id[0].ip[1] + DIGIT3*connectHdr.id[0].ip[2] + DIGIT4*connectHdr.id[0].ip[3] + connectHdr.id[0].port;
 #ifdef _TRACE
                 PROGLOG("MP: connect addrval = %" I64F "u", addrval);
 #endif
 
-                newsock->write(&id[0],sizeof(id)); 
+                newsock->write(&connectHdr,sizeof(connectHdr));
 
 #ifdef _FULLTRACE
                 StringBuffer tmp1;
-                id[0].getUrlStr(tmp1);
+                connectHdr.id[0].getUrlStr(tmp1);
                 tmp1.append(' ');
-                id[1].getUrlStr(tmp1);
+                connectHdr.id[1].getUrlStr(tmp1);
                 LOG(MCdebugInfo(100), unknownJob, "MP: connect after socket write %s",tmp1.str());
 #endif
 
-                size32_t reply = 0;
                 size32_t rd = 0;
 
 #ifdef _TRACE
@@ -841,9 +898,11 @@ protected: friend class CMPPacketReader;
 
                     rd = 0;
 
+                    MemoryBuffer replyMb;
+                    void *replyMem = replyMb.ensureCapacity(0x1000); // 4K - max size to allow for serialized exception
                     try
                     {
-                        newsock->readtms(&reply,sizeof(reply),sizeof(reply),rd,CONNECT_TIMEOUT_INTERVAL);
+                        newsock->readtms(replyMem, sizeof(rd), replyMb.capacity(), rd, CONNECT_TIMEOUT_INTERVAL);
                     }
                     catch (IException *e)
                     {
@@ -853,36 +912,35 @@ protected: friend class CMPPacketReader;
                         if ( (e->errorCode() != JSOCKERR_timeout_expired) ||
                              ((e->errorCode() == JSOCKERR_timeout_expired) && (loopCnt == 0)) )
                         {
-                                if (tm.timedout(&remaining))
-                                {
+                            if (tm.timedout(&remaining))
+                            {
 #ifdef _FULLTRACE
-                                    EXCLOG(e,"MP: connect timed out 3");
+                                EXCLOG(e,"MP: connect timed out 3");
 #endif
-                                    e->Release();
-                                    newsock->Release();
-                                    return false;
-                                }
+                                e->Release();
+                                return false;
+                            }
 #ifdef _TRACE
-                                EXCLOG(e, "MP: Failed to connect");
-#endif
-                                if ((retrycount--==0)||(tm.timeout==MP_ASYNC_SEND))
-                                {   // don't bother retrying on async send
-                                    e->Release();
-                                    throw new CMPException(MPERR_connection_failed,remoteep);
-                                }
-
-                                // if other side closes, connect again
-                                if (e->errorCode() == JSOCKERR_graceful_close)
-                                {
-                                    LOG(MCdebugInfo(100), unknownJob, "MP: Retrying (other side closed connection, probably due to clash)");
-                                    e->Release();
-                                    break;
-                                }
+                            EXCLOG(e, "MP: Failed to connect");
+#endif
+                            if ((retrycount--==0)||(tm.timeout==MP_ASYNC_SEND))
+                            {   // don't bother retrying on async send
+                                e->Release();
+                                throw new CMPException(MPERR_connection_failed,remoteep);
+                            }
 
+                            // if other side closes, connect again
+                            if (e->errorCode() == JSOCKERR_graceful_close)
+                            {
+                                LOG(MCdebugInfo(100), unknownJob, "MP: Retrying (other side closed connection, probably due to clash)");
                                 e->Release();
+                                break;
+                            }
+
+                            e->Release();
 
 #ifdef _TRACE
-                                LOG(MCdebugInfo(100), unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).str(),retrycount+1);
+                            LOG(MCdebugInfo(100), unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).str(),retrycount+1);
 #endif
                         }
                         else
@@ -900,15 +958,35 @@ protected: friend class CMPPacketReader;
 #ifdef _FULLTRACE
                     PROGLOG("MP: rd = %d", rd);
 #endif
-                    if (rd != 0)
+                    /* NB: legacy clients that don't handle the exception deserialization here
+                     * will see reply as success, so no clean error,
+                     * but will fail shortly afterwards since server connection is closed
+                     */
+                    if (rd > sizeof(rd)) // legacy clients will only ever send a reply of 0 or 4, if greater, then new client is replying with an exception
+                    {
+                        MemoryBuffer mb;
+                        mb.setBuffer(rd, replyMem, false);
+                        size32_t len;
+                        mb.read(len); // exception length
+                        if (len)
+                        {
+                            exitException.setown(deserializeException(mb));
+                            throw exitException.getLink();
+                        }
                         break;
+                    }
+                    else if (rd != 0)
+                    {
+                        assertex(rd == sizeof(rd));
+                        break;
+                    }
                 }
 
 #ifdef _TRACE
-                LOG(MCdebugInfo(100), unknownJob, "MP: connect after socket read rd=%u, reply=%u, sizeof(id)=%lu", rd, reply, sizeof(id));
+                LOG(MCdebugInfo(100), unknownJob, "MP: connect after socket read rd=%u, reply=%u, sizeof(connectHdr)=%lu", rd, reply, sizeof(connectHdr));
 #endif
 
-                if (reply!=0)
+                if (rd)
                 {
                     unsigned elapsedMs = msTick() - startMs;
                     if (elapsedMs >= TRACESLOW_THRESHOLD)
@@ -922,11 +1000,8 @@ protected: friend class CMPPacketReader;
                         WARNLOG("MP: connect to: %s, took: %d ms", epStr.str(), elapsedMs);
                     }
 
-                    assertex(reply==sizeof(id));    // how can this fail?
                     if (attachSocket(newsock,remoteep,hostep,true,NULL,addrval))
                     {
-                        newsock->Release();
-                        newsock = NULL;
 #ifdef _TRACE
                         LOG(MCdebugInfo(100), unknownJob, "MP: connected to %s",str.str());
 #endif
@@ -939,6 +1014,8 @@ protected: friend class CMPPacketReader;
             }
             catch (IException *e)
             {
+                if (exitException)
+                    throw;
                 if (tm.timedout(&remaining)) {
 #ifdef _FULLTRACE
                     EXCLOG(e,"MP: connect timed out 2");
@@ -961,8 +1038,7 @@ protected: friend class CMPPacketReader;
 #endif
             }
 
-            ::Release(newsock);
-            newsock = NULL;
+            newsock.clear();
 
             {
                 CriticalUnblock unblock(connectsect); // to avoid connecting philosopher problem
@@ -1706,7 +1782,6 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c
             FLLOG(MCoperatorWarning, unknownJob, e,"MP attachsocket(2)");
             e->Release();
         }
-
     }
 
     if (confirm)
@@ -1977,14 +2052,18 @@ int CMPConnectThread::run()
 #ifdef _TRACE
     LOG(MCdebugInfo(100), unknownJob, "MP: Connect Thread Starting - accept loop");
 #endif
-    while (running) {
+    while (running)
+    {
         ISocket *sock=NULL;
-        try {
-            sock=listensock->accept(true);
+        SocketEndpoint peerEp;
+        try
+        {
+            sock=listensock->accept(true, &peerEp);
 #ifdef _FULLTRACE       
             StringBuffer s;
             SocketEndpoint ep1;
-            if (sock) {
+            if (sock)
+            {
                 sock->getPeerEndpoint(ep1);
                 PROGLOG("MP: Connect Thread: socket accepted from %s",ep1.getUrlStr(s).str());
             }
@@ -1995,14 +2074,19 @@ int CMPConnectThread::run()
             LOG(MCdebugInfo, unknownJob, e,"MP accept failed");
             throw; // error handling TBD
         }
-        if (sock) {
-            try {
+        if (sock)
+        {
+            try
+            {
                 sock->set_keep_alive(true);
                 size32_t rd;
                 SocketEndpoint _remoteep;
                 SocketEndpoint hostep;
-                SocketEndpointV4 id[2];
-                traceSlowReadTms("MP: initial accept packet from", sock, &id[0], sizeof(id), sizeof(id), rd, CONFIRM_TIMEOUT, CONFIRM_TIMEOUT_INTERVAL);
+                ConnectHdr connectHdr;
+                bool legacyClient = false;
+
+                // NB: min size is ConnectHdr.id for legacy clients, can thus distinguish old from new
+                traceSlowReadTms("MP: initial accept packet from", sock, &connectHdr, sizeof(connectHdr.id), sizeof(connectHdr), rd, CONFIRM_TIMEOUT, CONFIRM_TIMEOUT_INTERVAL);
                 if (0 == rd)
                 {
                     if (mpTraceLevel > 1)
@@ -2015,45 +2099,84 @@ int CMPConnectThread::run()
                     sock->Release();
                     continue;
                 }
-                else if (rd != sizeof(id))
+                else
                 {
-                    // not sure how to get here as this is not one of the possible outcomes of above: rd == 0 or rd == sizeof(id) or an exception
-                    SocketEndpoint ep;
-                    sock->getPeerEndpoint(ep);
-                    StringBuffer errMsg("MP Connect Thread: invalid number of connection bytes serialized from ");
-                    ep.getUrlStr(errMsg);
-                    FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str());
-                    sock->close();
-                    sock->Release();
-                    continue;
+                    if (rd == sizeof(connectHdr.id)) // legacy client
+                    {
+                        legacyClient = true;
+                        connectHdr.setRole(0); // unknown
+                    }
+                    else if (rd < sizeof(connectHdr.id) || rd > sizeof(connectHdr))
+                    {
+                        // not sure how to get here as this is not one of the possible outcomes of above: rd == 0 or rd == sizeof(id) or an exception
+                        StringBuffer errMsg("MP Connect Thread: invalid number of connection bytes serialized from ");
+                        peerEp.getUrlStr(errMsg);
+                        FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str());
+                        sock->close();
+                        sock->Release();
+                        continue;
+                    }
                 }
-                id[0].get(_remoteep);
-                id[1].get(hostep);
 
-                unsigned __int64 addrval = DIGIT1*id[0].ip[0] + DIGIT2*id[0].ip[1] + DIGIT3*id[0].ip[2] + DIGIT4*id[0].ip[3] + id[0].port;
+                if (whiteListCallback)
+                {
+                    StringBuffer ipStr;
+                    peerEp.getIpText(ipStr);
+                    StringBuffer responseText; // filled if denied
+                    if (!whiteListCallback->isWhiteListed(ipStr, connectHdr.getRole(), &responseText))
+                    {
+                        Owned<IException> e = makeStringException(-1, responseText);
+                        OWARNLOG(e, nullptr);
+
+                        if (legacyClient)
+                        {
+                            /* NB: legacy client can't handle exception response
+                             * Acknowledge legacy connection, then close socket
+                             * The effect will be the client sees an MPERR_link_closed
+                             */
+                            size32_t reply = sizeof(connectHdr.id);
+                            sock->write(&reply, sizeof(reply));
+                        }
+                        else
+                        {
+                            MemoryBuffer mb;
+                            DelayedSizeMarker marker(mb);
+                            serializeException(e, mb);
+                            marker.write();
+                            sock->write(mb.toByteArray(), mb.length());
+                        }
+
+                        sock->close();
+                        sock->Release();
+                        continue;
+                    }
+                }
+
+                connectHdr.id[0].get(_remoteep);
+                connectHdr.id[1].get(hostep);
+
+                unsigned __int64 addrval = DIGIT1*connectHdr.id[0].ip[0] + DIGIT2*connectHdr.id[0].ip[1] + DIGIT3*connectHdr.id[0].ip[2] + DIGIT4*connectHdr.id[0].ip[3] + connectHdr.id[0].port;
 #ifdef _TRACE
                 PROGLOG("MP: Connect Thread: addrval = %" I64F "u", addrval);
 #endif
 
                 if (_remoteep.isNull() || hostep.isNull())
                 {
-                    SocketEndpoint ep;
-                    sock->getPeerEndpoint(ep);
                     StringBuffer errMsg;
                     SocketEndpointV4 zeroTest[2];
                     memset(zeroTest, 0x0, sizeof(zeroTest));
-                    if (memcmp(id, zeroTest, sizeof(id)))
+                    if (memcmp(connectHdr.id, zeroTest, sizeof(connectHdr.id)))
                     {
                         // JCSMORE, I think _remoteep really must/should match a IP of this local host
                         errMsg.append("MP Connect Thread: invalid remote and/or host ep serialized from ");
-                        ep.getUrlStr(errMsg);
+                        peerEp.getUrlStr(errMsg);
                         FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str());
                     }
                     else if (mpTraceLevel > 1)
                     {
                         // all zeros msg received
                         errMsg.append("MP Connect Thread: connect with empty msg received, assumed port monitor check from ");
-                        ep.getUrlStr(errMsg);
+                        peerEp.getUrlStr(errMsg);
                         PROGLOG("%s", errMsg.str());
                     }
                     sock->close();
@@ -2067,14 +2190,16 @@ int CMPConnectThread::run()
                 hostep.getUrlStr(tmp1);
                 PROGLOG("MP: Connect Thread: after read %s",tmp1.str());
 #endif
-                checkSelfDestruct(&id[0],sizeof(id));
+                checkSelfDestruct(&connectHdr.id[0],sizeof(connectHdr.id));
                 Owned<CMPChannel> channel = parent->lookup(_remoteep);
-                if (!channel->attachSocket(sock,_remoteep,hostep,false,&rd,addrval)) {
+                if (!channel->attachSocket(sock,_remoteep,hostep,false,&rd,addrval))
+                {
 #ifdef _FULLTRACE       
                     PROGLOG("MP Connect Thread: lookup failed");
 #endif
                 }
-                else {
+                else
+                {
 #ifdef _TRACE
                     StringBuffer str1;
                     StringBuffer str2;
@@ -2091,7 +2216,8 @@ int CMPConnectThread::run()
                 sock->close();
                 e->Release();
             }
-            try {
+            try
+            {
                 sock->Release();
             }
             catch (IException *e)
@@ -2100,7 +2226,8 @@ int CMPConnectThread::run()
                 e->Release();
             }
         }
-        else {
+        else
+        {
             if (running)
                 LOG(MCdebugInfo(100), unknownJob, "MP Connect Thread accept returned NULL");
         }
@@ -2183,9 +2310,10 @@ CMPChannel *CMPServer::lookup(const SocketEndpoint &endpoint)
 }
 
 
-CMPServer::CMPServer(unsigned _port)
+CMPServer::CMPServer(unsigned __int64 _role, unsigned _port)
 {
     RTsalt=0xff;
+    role = _role;
     port = 0;   // connectthread tells me what port it actually connected on
     checkclosed = false;
     connectthread = new CMPConnectThread(this, _port);
@@ -2987,7 +3115,7 @@ public:
         parent->removeChannel(channel);
     }
 
-    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) override
+    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) const override
     {
         Owned<CMPChannel> channel = parent->lookup(sender);
         assertex(channel);
@@ -3020,7 +3148,7 @@ ICommunicator *CMPServer::createCommunicator(IGroup *group, bool outer)
 IMPServer *startNewMPServer(unsigned port)
 {
     assertex(sizeof(PacketHeader)==32);
-    CMPServer *mpServer = new CMPServer(port);
+    CMPServer *mpServer = new CMPServer(0, port);
     mpServer->start();
     return mpServer;
 }
@@ -3034,7 +3162,7 @@ class CGlobalMPServer : public CMPServer
 public:
     static CriticalSection sect;
 
-    CGlobalMPServer(unsigned _port) : CMPServer(_port)
+    CGlobalMPServer(unsigned __int64 _role, unsigned _port) : CMPServer(_role, _port)
     {
         worldcomm = NULL;
         nestLevel = 0;
@@ -3068,13 +3196,13 @@ MODULE_EXIT()
     ::Release(globalMPServer);
 }
 
-void startMPServer(unsigned port, bool paused)
+void startMPServer(unsigned __int64 role, unsigned port, bool paused)
 {
     assertex(sizeof(PacketHeader)==32);
     CriticalBlock block(CGlobalMPServer::sect);
     if (NULL == globalMPServer)
     {
-        globalMPServer = new CGlobalMPServer(port);
+        globalMPServer = new CGlobalMPServer(role, port);
         initMyNode(globalMPServer->getPort());
     }
     if (0 == globalMPServer->queryNest())
@@ -3091,6 +3219,11 @@ void startMPServer(unsigned port, bool paused)
     globalMPServer->incNest();
 }
 
+void startMPServer(unsigned port, bool paused)
+{
+    startMPServer(0, port, paused);
+}
+
 void stopMPServer()
 {
     CGlobalMPServer *_globalMPServer = NULL;

system/mp/mpcomm.hpp

@@ -57,7 +57,7 @@ interface ICommunicator: extends IInterface
     virtual bool verifyAll(bool duplex=false, unsigned timeout=1000*60*30) = 0;
     virtual void disconnect(INode *node) = 0;
     virtual void barrier() = 0;
-    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) = 0;
+    virtual const SocketEndpoint &queryChannelPeerEndpoint(const SocketEndpoint &sender) const = 0;
 };
 
 interface IInterCommunicator: extends IInterface
@@ -100,9 +100,12 @@ interface IMPServer : extends IInterface
     virtual void stop() = 0;
     virtual INode *queryMyNode() = 0;
     virtual void setOpt(MPServerOpts opt, const char *value) = 0;
+    virtual void installWhiteListCallback(IWhiteListHandler *whiteListCallback) = 0;
+    virtual IWhiteListHandler *queryWhiteListCallback() const = 0;
 };
 
-extern mp_decl void startMPServer(unsigned port,bool paused=false);
+extern mp_decl void startMPServer(unsigned port, bool paused=false);
+extern mp_decl void startMPServer(unsigned __int64 role, unsigned port, bool paused=false);
 extern mp_decl void stopMPServer();
 extern mp_decl IMPServer *getMPServer();
 extern mp_decl IMPServer *startNewMPServer(unsigned port);

system/security/securesocket/securesocket.cpp

@@ -180,7 +180,7 @@ public:
     //
     // This method is called by server to accept client connection
     //
-    virtual ISocket* accept(bool allowcancel=false) // not needed for UDP
+    virtual ISocket* accept(bool allowcancel=false, SocketEndpoint *peerEp=nullptr) // not needed for UDP
     {
         throw MakeStringException(-1, "CSecureSocket::accept: not implemented");
     }

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -247,9 +247,10 @@ public:
                 if (!spill)
                     writer.start(); // writer keeps writing ahead as much as possible, the readahead impl. will block when has too much
             }
-            catch (IException *)
+            catch (IException *e)
             {
                 eofHit = true;
+                writeAheadException.set(e);
                 throw;
             }
         }

thorlcr/graph/thgraphmaster.cpp

@@ -1774,7 +1774,7 @@ bool CJobMaster::go()
     try
     {
         startJob();
-        workunit->setGraphState(queryGraphName(), WUGraphRunning);
+        workunit->setGraphState(queryGraphName(), getWfid(), WUGraphRunning);
         Owned<IThorGraphIterator> iter = queryJobChannel(0).getSubGraphs();
         CICopyArrayOf<CMasterGraph> toRun;
         ForEach(*iter)
@@ -1807,7 +1807,7 @@ bool CJobMaster::go()
     }
     catch (IException *e) { fireException(e); e->Release(); }
     catch (CATCHALL) { Owned<IException> e = MakeThorException(0, "Unknown exception running sub graphs"); fireException(e); }
-    workunit->setGraphState(queryGraphName(), aborted?WUGraphFailed:(allDone?WUGraphComplete:(pausing?WUGraphPaused:WUGraphComplete)));
+    workunit->setGraphState(queryGraphName(), getWfid(), aborted?WUGraphFailed:(allDone?WUGraphComplete:(pausing?WUGraphPaused:WUGraphComplete)));
 
     if (queryPausing())
         saveSpills();

tools/wutool/wutool.cpp

@@ -1246,7 +1246,7 @@ protected:
         ASSERT(wu->queryGraphState("graph1")==WUGraphUnknown);
         ASSERT(wu->queryNodeState("graph1", 1)==WUGraphUnknown);
 
-        wu->setGraphState("graph1",WUGraphRunning);
+        wu->setGraphState("graph1",1,WUGraphRunning);
         ASSERT(wu->queryGraphState("graph1")==WUGraphRunning);
 
         wu->setNodeState("graph1", 1, WUGraphRunning);
@@ -1261,7 +1261,7 @@ protected:
         ret = wu->getRunningGraph(s, subid);
         ASSERT(!ret);
 
-        Owned<IWUGraphStats> progress = wu->updateStats("graph1", SCThthor, queryStatisticsComponentName(), 0, 1);
+        Owned<IWUGraphStats> progress = wu->updateStats("graph1", SCThthor, queryStatisticsComponentName(), 1, 1);
         IStatisticGatherer & stats = progress->queryStatsBuilder();
         {
             StatsSubgraphScope subgraph(stats, 1);