Merge branch 'candidate-7.0.4' into candidate-7.2.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

esp/esdllib/esdl_transformer2.cpp

@@ -1468,9 +1468,24 @@ void Esdl2Transformer::addMethod(Esdl2Base *item)
     }
 }
 
+void Esdl2Transformer::setMethodInfo(const char *name, Esdl2Method *method)
+{
+    if (name && *name && method)
+    {
+        WriteLockBlock block(rwMethodLock);
+        meth_map.setValue(name, method);
+    }
+}
+
+Esdl2Method **Esdl2Transformer::readMethodInfo(const char *method)
+{
+    ReadLockBlock block(rwMethodLock);
+    return meth_map.getValue(method);
+}
+
 IEsdlMethodInfo* Esdl2Transformer::queryMethodInfo(const char* service,const char *method)
 {
-    Esdl2Method **mt = meth_map.getValue(method);
+    Esdl2Method **mt = readMethodInfo(method);
     if (mt && *mt)
         return dynamic_cast<IEsdlMethodInfo *>(*mt);
 
@@ -1480,20 +1495,27 @@ IEsdlMethodInfo* Esdl2Transformer::queryMethodInfo(const char* service,const cha
     IEsdlDefMethod* mth = svc->queryMethodByName(method);
 
     Esdl2Method* m = new Esdl2Method(this,mth);
-    meth_map.setValue(method,m);
+    setMethodInfo(method, m);
     return m;
 }
 
 void Esdl2Transformer::addType(Esdl2Base* type)
 {
+    WriteLockBlock block(rwTypeLock);
     types.append(*type);
     type_map.setValue(type->queryName(), type);
 }
 
+Esdl2Base **Esdl2Transformer::readType(const char *name)
+{
+    ReadLockBlock block(rwTypeLock);
+    return type_map.getValue(name);
+}
+
 Esdl2Base* Esdl2Transformer::queryType(const char* name)
 {
     ESDL_DBG("queryType(%s)", name);
-    Esdl2Base** typ = type_map.getValue(name);
+    Esdl2Base** typ = readType(name);
     if (typ && *typ)
         return *typ;
 

esp/esdllib/esdl_transformer2.ipp

@@ -466,9 +466,11 @@ class Esdl2Transformer : public CInterface, implements IEsdlTransformer
 private:
     Owned<IEsdlDefinition> m_def;
 
+    ReadWriteLock rwTypeLock;
     EsdlBaseArray types;  //elements and arrays
     EsdlBaseMap type_map;
 
+    ReadWriteLock rwMethodLock;
     EsdlMethodArray methods;  //elements and arrays
     EsdlMethodMap meth_map;
 
@@ -482,7 +484,11 @@ public:
 
     virtual ~Esdl2Transformer();
 
+    Esdl2Base **readType(const char *name);
     Esdl2Base* queryType(const char* name);
+
+    void setMethodInfo(const char *name, Esdl2Method *method);
+    Esdl2Method **readMethodInfo(const char *method);
     IEsdlMethodInfo *queryMethodInfo(const char* service,const char *method);
 
     void serialize(StringBuffer &out);

esp/logging/loggingagent/espserverloggingagent/loggingagent.cpp

@@ -303,9 +303,9 @@ bool CESPServerLoggingAgent::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdate
     return true;
 }
 
-void CESPServerLoggingAgent::filterLogContent(IEspUpdateLogRequestWrap* req)
+IEspUpdateLogRequestWrap* CESPServerLoggingAgent::filterLogContent(IEspUpdateLogRequestWrap* req)
 {
-    logContentFilter.filterLogContent(req);
+    return logContentFilter.filterLogContent(req);
 }
 
 bool CESPServerLoggingAgent::sendHTTPRequest(StringBuffer& req, StringBuffer &resp, StringBuffer &status)

esp/logging/loggingagent/espserverloggingagent/loggingagent.hpp

@@ -57,7 +57,7 @@ public:
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
     virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID);
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
-    virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
+    virtual IEspUpdateLogRequestWrap* filterLogContent(IEspUpdateLogRequestWrap* req);
 };
 
 #endif //__ESPSERVERLOGGINGAGENT__HPP__

esp/logging/logginglib/loggingagentbase.cpp

@@ -143,7 +143,7 @@ void CLogContentFilter::filterLogContentTree(StringArray& filters, IPropertyTree
     }
 }
 
-void CLogContentFilter::filterLogContent(IEspUpdateLogRequestWrap* req)
+IEspUpdateLogRequestWrap* CLogContentFilter::filterLogContent(IEspUpdateLogRequestWrap* req)
 {
     const char* logContent = req->getUpdateLogRequest();
     Owned<IPropertyTree> logRequestTree = req->getLogRequestTree();
@@ -294,8 +294,8 @@ void CLogContentFilter::filterLogContent(IEspUpdateLogRequestWrap* req)
     StringBuffer updateLogRequestXML;
     toXML(updateLogRequestTree, updateLogRequestXML);
     ESPLOG(LogMax, "filtered content and option: <%s>", updateLogRequestXML.str());
-    req->clearOriginalContent();
-    req->setUpdateLogRequest(updateLogRequestXML.str());
+
+    return new CUpdateLogRequestWrap(req->getGUID(), req->getOption(), updateLogRequestXML.str());
 }
 
 void CDBLogAgentBase::readDBCfg(IPropertyTree* cfg, StringBuffer& server, StringBuffer& dbUser, StringBuffer& dbPassword)
@@ -550,7 +550,8 @@ void CDBLogAgentBase::getTransactionID(StringAttrMapping* transFields, StringBuf
     //Not implemented
 }
 
-void CDBLogAgentBase::filterLogContent(IEspUpdateLogRequestWrap* req)
+IEspUpdateLogRequestWrap* CDBLogAgentBase::filterLogContent(IEspUpdateLogRequestWrap* req)
 {
     //No filter in CDBSQLLogAgent
+    return req;
 }

esp/logging/logginglib/loggingagentbase.hpp

@@ -241,7 +241,7 @@ interface IEspLogAgent : extends IInterface
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp) = 0;
     virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID) = 0;
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp) = 0;
-    virtual void filterLogContent(IEspUpdateLogRequestWrap* req) = 0;
+    virtual IEspUpdateLogRequestWrap* filterLogContent(IEspUpdateLogRequestWrap* req) = 0;
 };
 
 class CESPLogContentGroupFilters : public CInterface, implements IInterface
@@ -281,7 +281,7 @@ public:
     CLogContentFilter() {};
 
     void readAllLogFilters(IPropertyTree* cfg);
-    void filterLogContent(IEspUpdateLogRequestWrap* req);
+    IEspUpdateLogRequestWrap* filterLogContent(IEspUpdateLogRequestWrap* req);
 };
 
 class LOGGINGCOMMON_API CDBLogAgentBase : public CInterface, implements IEspLogAgent
@@ -316,6 +316,6 @@ public:
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
     virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID);
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
-    virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
+    virtual IEspUpdateLogRequestWrap* filterLogContent(IEspUpdateLogRequestWrap* req);
 };
 #endif  //_LOGGINGAGENT_HPP__

esp/logging/logginglib/logthread.cpp

@@ -158,9 +158,9 @@ bool CLogThread::queueLog(IEspUpdateLogRequest* logRequest)
 bool CLogThread::queueLog(IEspUpdateLogRequestWrap* logRequest)
 {
     unsigned startTime = (getEspLogLevel()>=LogNormal) ? msTick() : 0;
-    logAgent->filterLogContent(logRequest);
+    Owned<IEspUpdateLogRequestWrap> logRequestFiltered = logAgent->filterLogContent(logRequest);
     ESPLOG(LogNormal, "LThread:filterLog: %dms\n", msTick() -  startTime);
-    return enqueue(logRequest);
+    return enqueue(logRequestFiltered);
 }
 
 bool CLogThread::enqueue(IEspUpdateLogRequestWrap* logRequest)

esp/platform/espcfg.cpp

@@ -473,26 +473,29 @@ CEspConfig::CEspConfig(IProperties* inputs, IPropertyTree* envpt, IPropertyTree*
                     else
                     {
                         ptree->getProp("@type", bcfg->type);
-                        ptree->getProp("@plugin", bcfg->plugin);
-                        fixPlugin(bcfg->plugin);
-                        bcfg->isDefault = ptree->getPropBool("@defaultBinding", false);
-
-                        StringBuffer addr;
-                        ptree->getProp("@netAddress", addr);
-                        if (strcmp(addr.str(), ".") == 0)
-                        {
-                            // Here we interpret '.' as binding to all interfaces, so convert it to "0.0.0.0"
-                            bcfg->address.append("0.0.0.0");
-                        }
-                        else
+                        if (!streq(bcfg->type.str(), "EsdlBinding"))
                         {
-                            bcfg->address.append(addr.str());
+                            ptree->getProp("@plugin", bcfg->plugin);
+                            fixPlugin(bcfg->plugin);
+                            bcfg->isDefault = ptree->getPropBool("@defaultBinding", false);
+
+                            StringBuffer addr;
+                            ptree->getProp("@netAddress", addr);
+                            if (strcmp(addr.str(), ".") == 0)
+                            {
+                                // Here we interpret '.' as binding to all interfaces, so convert it to "0.0.0.0"
+                                bcfg->address.append("0.0.0.0");
+                            }
+                            else
+                            {
+                                bcfg->address.append(addr.str());
+                            }
+
+                            ptree->getProp("@service", bcfg->service_name);
+                            ptree->getProp("@protocol", bcfg->protocol_name);
+
+                            m_bindings.push_back(bcfg.getClear());
                         }
-
-                        ptree->getProp("@service", bcfg->service_name);
-                        ptree->getProp("@protocol", bcfg->protocol_name);
-
-                        m_bindings.push_back(bcfg.getClear());
                     }
                 }
 

esp/scm/ws_dfu.ecm

@@ -203,6 +203,7 @@ ESPStruct [nil_remove] DFUFileDetail
     [min_ver("1.38")] string PackageID;
     [min_ver("1.39")] ESPstruct DFUFilePartition Partition;
     [min_ver("1.39")] ESParray<ESPstruct DFUFileBloom> Blooms;
+    [min_ver("1.40")] int ExpireDays;
 };
 
 ESPStruct DFUSpaceItem
@@ -918,7 +919,7 @@ ESPresponse [exceptions_inline, nil_remove] DFUFilePublishResponse
 //  ===========================================================================
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.39"),
+    version("1.40"),
     noforms,
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/esdl_svc_engine/esdl_monitor.cpp

@@ -235,14 +235,6 @@ public:
                 continue;
             }
             cur.getProp("@espbinding", data->name);
-            if (data->name.length() != 0 && existsStaticBinding(data->name.str()))
-            {
-                DBGLOG("ESDL binding %s has esp binding configured, no need to create it dynamically, skip.", data->id.str());
-                continue;
-            }
-            else
-                DBGLOG("ESDL binding %s doesn't have esp binding configured, creating the binding dynamically...", data->id.str());
-
             data->port = cur.getPropInt("@port");
             if (data->port == 0)
             {
@@ -299,19 +291,11 @@ public:
             if (theBinding)
             {
                 DBGLOG("Requesting clearing of binding %s", data->id.str());
-                if (data->name.length() > 0 && existsStaticBinding(data->name.str()))
-                {
-                    DBGLOG("Static esp binding exists for this esdl binding.");
-                    theBinding->reloadBindingFromCentralStore(data->id.str()); //clear the binding by reloading
-                }
-                else
-                {
-                    theBinding->clearBindingState();
-                    if (data->port <= 0)
-                        data->port = theBinding->getPort();
-                    DBGLOG("Removing binding from port %d", data->port);
-                    queryEspServer()->removeBinding(data->port, *theBinding);
-                }
+                theBinding->clearBindingState();
+                if (data->port <= 0)
+                    data->port = theBinding->getPort();
+                DBGLOG("Removing binding from port %d", data->port);
+                queryEspServer()->removeBinding(data->port, *theBinding);
                 removeBindingFromMap(data->id.str());
             }
             else
@@ -346,41 +330,14 @@ public:
                 return;
             }
 
-            bool existsStatic = false;
-            if (data->name.length() > 0)
-                existsStatic = existsStaticBinding(data->name.str());
-            else
+            if (data->name.length() == 0)
                 data->name.set(data->id);
-            if (!existsStatic)
-            {
-                DBGLOG("ESDL binding %s doesn't have esp binding configured, creating the binding dynamically...", data->id.str());
-                if (data->port == 0)
-                {
-                    DBGLOG("Port is not provided for binding, can't create binding.");
-                    return;
-                }
-                addBinding(data);
-            }
-            else
+            if (data->port == 0)
             {
-                DBGLOG("ESDL binding %s has esp binding configured, reloading the esp binding...", data->id.str());
-                //Reload static binding
-                IEspServer* server = queryEspServer();
-                IEspRpcBinding* espBinding = server->queryBinding(data->name.str());
-                if (espBinding != nullptr)
-                {
-                    EsdlBindingImpl* esdlBinding = dynamic_cast<EsdlBindingImpl*>(espBinding);
-                    if (esdlBinding != nullptr)
-                    {
-                        esdlBinding->reloadBindingFromCentralStore(data->id.str());
-                        registerBinding(data->id.str(), esdlBinding);
-                    }
-                    else
-                        DBGLOG("The esp binding failed to be cast to esdl binding.");
-                }
-                else
-                    DBGLOG("Esp binding not found.");
+                DBGLOG("Port is not provided for binding, can't create binding.");
+                return;
             }
+            addBinding(data);
         }
         else
         {
@@ -432,21 +389,6 @@ private:
         DBGLOG("Successfully instantiated new DESDL binding %s and service", data->id.str());
     }
 
-    bool existsStaticBinding(const char* espBinding)
-    {
-        if (!espBinding || !*espBinding)
-            return false;
-        DBGLOG("Checking if there is esp binding %s configured...", espBinding);
-        IPropertyTree* procpt = queryEspServer()->queryProcConfig();
-        if (procpt)
-        {
-            VStringBuffer xpath("EspBinding[@name='%s']", espBinding);
-            if (procpt->queryPropTree(xpath.str()) != nullptr)
-                return true;
-        }
-        return false;
-    }
-
     bool espProcessMatch(const char* espProcess)
     {
         if (!espProcess)
@@ -486,14 +428,12 @@ private:
             //If esp's original config has one configured for this binding, use it
             VStringBuffer xpath("EspBinding[@name='%s']", bindingName);
             IPropertyTree* bindingtree = procpt->queryPropTree(xpath.str());
-            if (bindingtree)
+            if (!bindingtree)
             {
-                bindingtree->getProp("@protocol", protocol);
-                return LINK(procpt);
+                //Otherwise check if there's binding configured on the same port
+                xpath.setf("EspBinding[@type='EsdlBinding'][@port=%s][1]", port);
+                bindingtree = procpt->queryPropTree(xpath.str());
             }
-            //Otherwise check if there's binding configured on the same port
-            xpath.setf("EspBinding[@type='EsdlBinding'][@port=%s][1]", port);
-            bindingtree = procpt->queryPropTree(xpath.str());
             if (!bindingtree)
             {
                 //Otherwise check if there's binding configured with port 0

esp/services/ws_dfu/ws_dfuService.cpp

@@ -2226,6 +2226,9 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
             FileDetails.setBlooms(bloomList);
     }
 
+    if ((version >= 1.40) && df->queryAttributes().hasProp("@expireDays"))
+        FileDetails.setExpireDays(df->queryAttributes().getPropInt("@expireDays"));
+
     //#14280
     IDistributedSuperFile *sf = df->querySuperFile();
     if(sf)

esp/src/eclwatch/package.js

@@ -1,39 +0,0 @@
-var profile = (function () {
-    var copyOnly = function (filename, mid) {
-        var list = {
-            "hpcc/eclwatch.profile": true,
-            "hpcc/eclwatch.json": true,
-            "hpcc/dojoConfig": true,
-            "hpcc/viz/DojoD3": true,
-            "hpcc/viz/DojoD32DChart": true,
-            "hpcc/viz/DojoD3NDChart": true,
-            "hpcc/viz/DojoD3Choropleth": true
-        };
-        return (mid in list) ||
-            (/^hpcc\/resources\//.test(mid) && !/\.css$/.test(filename)) ||
-            /(png|jpg|jpeg|gif|tiff)$/.test(filename);
-    };
-
-    return {
-        destLocation: "eclwatch",
-        resourceTags: {
-            test: function (filename, mid) {
-                return false;
-            },
-
-            copyOnly: function (filename, mid) {
-                return copyOnly(filename, mid);
-            },
-
-            amd: function (filename, mid) {
-                return !copyOnly(filename, mid) && /\.js$/.test(filename);
-            },
-
-            miniExclude: function (filename, mid) {
-                return mid in {
-                    'hpcc/package': 1
-                };
-            }
-        }
-    };
-})();

esp/src/eclwatch/package.json

@@ -1,31 +0,0 @@
-{
-    "name": "eclwatch",
-    "version": "1.0",
-    "main": "stub",
-    "dependencies": {
-        "dojo": "1.9.3",
-        "dijit": "1.9.3",
-        "dojox": "1.9.3",
-        "util": "1.9.3",
-        "d3": "3.4.3",
-        "topojson": "1.4.9",
-        "dgrid": "0.3.13",
-        "xstyle": "0.2.1",
-        "put-selector": "0.3.5"
-    },
-    "description": "'ECL Watch' Web interface for HPCC Platform.",
-    "licenses": [
-        {
-            "type": "Apache License, Version 2.0",
-            "url": "https://raw.github.com/hpcc-systems/HPCC-Platform/master/LICENSE.txt"
-        }
-    ],
-    "bugs": "https://track.hpccsystems.com/issues/?jql=component%20%3D%20EclWatch%20",
-    "keywords": [
-        "JavaScript",
-        "HPCC",
-        "Big Data"
-    ],
-    "homepage": "https://github.com/hpcc-systems/HPCC-Platform",
-    "dojoBuild": "package.js"
-}

initfiles/componentfiles/configxml/@temp/esp_service_DynamicESDL.xsl

@@ -84,7 +84,8 @@
                             </xsl:when>
                             <xsl:when test="($wsLogServiceESPAgentNode)">
                                 <xsl:call-template name="WsLogServiceESPAgent">
-                                    <xsl:with-param name="managerNode" select="$managerNode"/>
+                                    <xsl:with-param name="agentName" select="$agentName"/>
+                                    <xsl:with-param name="agentNode" select="$wsLogServiceESPAgentNode"/>
                                 </xsl:call-template>
                             </xsl:when>
                             <xsl:otherwise>

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -23,100 +23,97 @@ xmlns:set="http://exslt.org/sets">
     <xsl:import href="esp_logging_transid.xsl"/>
 
     <xsl:template name="WsLogServiceESPAgent" type="DefaultLoggingAgent">
-        <xsl:param name="managerNode"/>
-        <xsl:for-each select="$managerNode/ESPLoggingAgent">
-            <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
-            <xsl:variable name="agentNode" select="/Environment/Software/WsLogServiceESPAgent[@name=$agentName]"/>
-            <xsl:if test="not($agentNode)">
-                <xsl:message terminate="yes">An WsLogService ESP Logging Agent <xsl:value-of select="$agentName"/>  for <xsl:value-of select="$managerNode/@name"/> is undefined!</xsl:message>
+        <xsl:param name="agentName"/>
+        <xsl:param name="agentNode"/>
+        <xsl:if test="not($agentNode)">
+            <xsl:message terminate="yes">An WsLogService ESP Logging Agent <xsl:value-of select="$agentName"/> is undefined!</xsl:message>
+        </xsl:if>
+        <xsl:variable name="loggingServer" select="$agentNode/LoggingServer"/>
+        <xsl:if test="not($loggingServer)">
+            <xsl:message terminate="yes">ESP logging server is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+        </xsl:if>
+        <xsl:variable name="loggingServerUrl" select="$loggingServer/@Url"/>
+        <xsl:if test="string($loggingServerUrl) = ''">
+            <xsl:message terminate="yes">Logging Server URL is undefined for <xsl:value-of select="$agentName"/>!</xsl:message>
+        </xsl:if>
+        <xsl:variable name="logDataXPath" select="$agentNode/LogDataXPath"/>
+        <xsl:if test="not($logDataXPath)">
+            <xsl:message terminate="yes">Log Data XPath is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+        </xsl:if>
+
+        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog,GetTransactionID" plugin="wslogserviceespagent">
+            <LoggingServer url="{$loggingServerUrl}" user="{$loggingServer/@User}" password="{$loggingServer/@Password}"/>
+            <xsl:if test="string($agentNode/@FailSafe) != ''">
+                <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+            </xsl:if>
+            <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
+                <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+            </xsl:if>
+            <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
+                <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
             </xsl:if>
-            <xsl:variable name="loggingServer" select="$agentNode/LoggingServer"/>
-            <xsl:if test="not($loggingServer)">
-                <xsl:message terminate="yes">ESP logging server is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+            <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
+                <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
             </xsl:if>
-            <xsl:variable name="loggingServerUrl" select="$loggingServer/@Url"/>
-            <xsl:if test="string($loggingServerUrl) = ''">
-                <xsl:message terminate="yes">Logging Server URL is undefined!</xsl:message>
+            <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
+                <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
             </xsl:if>
-            <xsl:variable name="logDataXPath" select="$agentNode/LogDataXPath"/>
-            <xsl:if test="not($logDataXPath)">
-                <xsl:message terminate="yes">Log Data XPath is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+            <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
+                <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+            </xsl:if>
+            <xsl:if test="string($agentNode/@TransactionSeedType) != ''">
+                <TransactionSeedType><xsl:value-of select="$agentNode/@TransactionSeedType"/></TransactionSeedType>
+            </xsl:if>
+            <xsl:if test="string($agentNode/@AlternativeTransactionSeedType) != ''">
+                <AlternativeTransactionSeedType><xsl:value-of select="$agentNode/@AlternativeTransactionSeedType"/></AlternativeTransactionSeedType>
             </xsl:if>
 
-            <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog,GetTransactionID" plugin="wslogserviceespagent">
-                <LoggingServer url="{$loggingServerUrl}" user="{$loggingServer/@User}" password="{$loggingServer/@Password}"/>
-                <xsl:if test="string($agentNode/@FailSafe) != ''">
-                    <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+            <xsl:call-template name="EspLoggingTransactionID">
+                <xsl:with-param name="agentNode" select="$agentNode"/>
+            </xsl:call-template>
+                
+            <LogDataXPath>
+                <xsl:if test="string($logDataXPath/@IP) != ''">
+                    <IP><xsl:value-of select="$logDataXPath/@IP"/></IP>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
-                    <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+                <xsl:if test="string($logDataXPath/@UserName) != ''">
+                    <UserName><xsl:value-of select="$logDataXPath/@UserName"/></UserName>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
-                    <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
+                <xsl:if test="string($logDataXPath/@ServiceName) != ''">
+                    <ServiceName><xsl:value-of select="$logDataXPath/@ServiceName"/></ServiceName>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
-                    <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
+                <xsl:if test="string($logDataXPath/@RecordCount) != ''">
+                    <RecordCount><xsl:value-of select="$logDataXPath/@RecordCount"/></RecordCount>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
-                    <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
+                <xsl:if test="string($logDataXPath/@DomainName) != ''">
+                    <DomainName><xsl:value-of select="$logDataXPath/@DomainName"/></DomainName>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
-                    <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+                <xsl:if test="string($logDataXPath/@GUID) != ''">
+                    <GUID><xsl:value-of select="$logDataXPath/@GUID"/></GUID>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@TransactionSeedType) != ''">
-                    <TransactionSeedType><xsl:value-of select="$agentNode/@TransactionSeedType"/></TransactionSeedType>
+                <xsl:if test="string($logDataXPath/@BlindLogging) != ''">
+                    <BlindLogging><xsl:value-of select="$logDataXPath/@BlindLogging"/></BlindLogging>
                 </xsl:if>
-                <xsl:if test="string($agentNode/@AlternativeTransactionSeedType) != ''">
-                    <AlternativeTransactionSeedType><xsl:value-of select="$agentNode/@AlternativeTransactionSeedType"/></AlternativeTransactionSeedType>
+                <xsl:if test="string($logDataXPath/@EncryptedLogging) != ''">
+                    <EncryptedLogging><xsl:value-of select="$logDataXPath/@EncryptedLogging"/></EncryptedLogging>
                 </xsl:if>
-
-                <xsl:call-template name="EspLoggingTransactionID">
-                    <xsl:with-param name="agentNode" select="$agentNode"/>
-                </xsl:call-template>
-                
-                <LogDataXPath>
-                    <xsl:if test="string($logDataXPath/@IP) != ''">
-                        <IP><xsl:value-of select="$logDataXPath/@IP"/></IP>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@UserName) != ''">
-                        <UserName><xsl:value-of select="$logDataXPath/@UserName"/></UserName>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@ServiceName) != ''">
-                        <ServiceName><xsl:value-of select="$logDataXPath/@ServiceName"/></ServiceName>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@RecordCount) != ''">
-                        <RecordCount><xsl:value-of select="$logDataXPath/@RecordCount"/></RecordCount>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@DomainName) != ''">
-                        <DomainName><xsl:value-of select="$logDataXPath/@DomainName"/></DomainName>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@GUID) != ''">
-                        <GUID><xsl:value-of select="$logDataXPath/@GUID"/></GUID>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@BlindLogging) != ''">
-                        <BlindLogging><xsl:value-of select="$logDataXPath/@BlindLogging"/></BlindLogging>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@EncryptedLogging) != ''">
-                        <EncryptedLogging><xsl:value-of select="$logDataXPath/@EncryptedLogging"/></EncryptedLogging>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@ForwardLog) != ''">
-                        <ForwardLog><xsl:value-of select="$logDataXPath/@ForwardLog"/></ForwardLog>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@RawLogInformation) != ''">
-                        <RawLogInformation><xsl:value-of select="$logDataXPath/@RawLogInformation"/></RawLogInformation>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@CompressBlobs) != ''">
-                        <CompressBlobs><xsl:value-of select="$logDataXPath/@CompressBlobs"/></CompressBlobs>
-                    </xsl:if>
-                    <xsl:if test="string($logDataXPath/@WorkunitID) != ''">
-                        <WorkunitID><xsl:value-of select="$logDataXPath/@WorkunitID"/></WorkunitID>
-                    </xsl:if>
-                    <xsl:for-each select="$agentNode/LogInfo">
-                        <LogInfo name="{current()/@name}" valueXPath="{current()/@valueXPath}" dataXPath="{current()/@dataXPath}" multipleValue="{current()/@multipleValue}" multipleData="{current()/@multipleData}" encodeValue="{current()/@encodeValue}" encodeData="{current()/@encodeData}"/>
-                    </xsl:for-each>
-                </LogDataXPath>
-            </LogAgent>
-        </xsl:for-each>
+                <xsl:if test="string($logDataXPath/@ForwardLog) != ''">
+                    <ForwardLog><xsl:value-of select="$logDataXPath/@ForwardLog"/></ForwardLog>
+                </xsl:if>
+                <xsl:if test="string($logDataXPath/@RawLogInformation) != ''">
+                    <RawLogInformation><xsl:value-of select="$logDataXPath/@RawLogInformation"/></RawLogInformation>
+                </xsl:if>
+                <xsl:if test="string($logDataXPath/@CompressBlobs) != ''">
+                    <CompressBlobs><xsl:value-of select="$logDataXPath/@CompressBlobs"/></CompressBlobs>
+                </xsl:if>
+                <xsl:if test="string($logDataXPath/@WorkunitID) != ''">
+                    <WorkunitID><xsl:value-of select="$logDataXPath/@WorkunitID"/></WorkunitID>
+                </xsl:if>
+                <xsl:for-each select="$agentNode/LogInfo">
+                    <LogInfo name="{current()/@name}" valueXPath="{current()/@valueXPath}" dataXPath="{current()/@dataXPath}" multipleValue="{current()/@multipleValue}" multipleData="{current()/@multipleData}" encodeValue="{current()/@encodeValue}" encodeData="{current()/@encodeData}"/>
+                </xsl:for-each>
+            </LogDataXPath>
+        </LogAgent>
     </xsl:template>
 
 </xsl:stylesheet>

thorlcr/graph/thgraph.cpp

@@ -2657,6 +2657,7 @@ void CJobBase::init()
     sharedMemoryLimitPercentage = (unsigned)getWorkUnitValueInt("globalMemoryLimitPC", globals->getPropInt("@sharedMemoryLimit", 90));
     sharedMemoryMB = globalMemoryMB*sharedMemoryLimitPercentage/100;
     failOnLeaks = getOptBool("failOnLeaks");
+    maxLfnBlockTimeMins = getOptInt(THOROPT_MAXLFN_BLOCKTIME_MINS, DEFAULT_MAXLFN_BLOCKTIME_MINS);
 
     PROGLOG("Global memory size = %d MB, shared memory = %d%%, memory spill at = %d%%", globalMemoryMB, sharedMemoryLimitPercentage, memorySpillAtPercentage);
     StringBuffer tracing("maxActivityCores = ");

thorlcr/graph/thgraph.hpp

@@ -29,6 +29,7 @@
 #define LONGTIMEOUT (25*60*1000)
 #define MEDIUMTIMEOUT 30000
 #define DEFAULT_MAX_ACTINITWAITTIME_MINS (2*60) // 2hrs
+#define DEFAULT_MAXLFN_BLOCKTIME_MINS 25 // 25 mins
 
 #include "jlib.hpp"
 #include "jarray.hpp"
@@ -826,6 +827,7 @@ protected:
     Owned<IThorAllocator> sharedAllocator;
     bool jobEnded = false;
     bool failOnLeaks = false;
+    unsigned maxLfnBlockTimeMins = DEFAULT_MAXLFN_BLOCKTIME_MINS;
 
     class CThorPluginCtx : public SimplePluginCtx
     {
@@ -847,6 +849,7 @@ public:
     CJobBase(ILoadedDllEntry *querySo, const char *graphName);
     virtual void beforeDispose() override;
 
+    unsigned queryMaxLfnBlockTimeMins() const { return maxLfnBlockTimeMins; }
     virtual void addChannel(IMPServer *mpServer) = 0;
     CJobChannel &queryJobChannel(unsigned c) const;
     CActivityBase &queryChannelActivity(unsigned c, graph_id gid, activity_id id) const;

thorlcr/graph/thgraphslave.cpp

@@ -1671,6 +1671,13 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co
         setRemoteOutputCompressionDefault(remoteCompressedOutput);
 
     actInitWaitTimeMins = getOptInt(THOROPT_ACTINIT_WAITTIME_MINS, DEFAULT_MAX_ACTINITWAITTIME_MINS);
+
+    /* Need to make sure that the activity initialization timeout is at least as long
+     * as the max LFN block time. i.e. so that the query doesn't spuriously abort with an
+     * activity initialization timeout before it hits the configured max LFN block time.
+     */
+    if (queryMaxLfnBlockTimeMins() >= actInitWaitTimeMins)
+        actInitWaitTimeMins = queryMaxLfnBlockTimeMins()+1;
 }
 
 void CJobSlave::addChannel(IMPServer *mpServer)

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -341,7 +341,7 @@ public:
         if (fileMapping)
             return &fileMapping->get();
 
-        Owned<IDistributedFile> file = timedLookup(job, scopedName.str(), false);
+        Owned<IDistributedFile> file = timedLookup(job, scopedName.str(), false, job.queryMaxLfnBlockTimeMins() * 60000);
         if (file && 0 == file->numParts())
         {
             if (file->querySuperFile())
@@ -390,7 +390,7 @@ public:
                 throw MakeStringException(99, "Cannot publish %s, invalid logical name", logicalName);
             if (dlfn.isForeign())
                 throw MakeStringException(99, "Cannot publish to a foreign Dali: %s", logicalName);
-            efile.setown(timedLookup(job, dlfn, true));
+            efile.setown(timedLookup(job, dlfn, true, job.queryMaxLfnBlockTimeMins() * 60000));
             if (efile)
             {
                 if (!extend && !overwriteok)
@@ -420,7 +420,7 @@ public:
                 if (found)
                 {
                     workunit->releaseFile(logicalName);
-                    Owned<IDistributedFile> f = timedLookup(job, dlfn, false);
+                    Owned<IDistributedFile> f = timedLookup(job, dlfn, false, job.queryMaxLfnBlockTimeMins() * 60000);
                     if (f)
                     {
                         unsigned p, parts = f->numParts();
@@ -445,9 +445,9 @@ public:
                     efile->getClusterName(c, clusterName);
                     clusters.append(clusterName);
                 }
-                remove(job, *efile);
+                remove(job, *efile, job.queryMaxLfnBlockTimeMins() * 60000);
                 efile.clear();
-                efile.setown(timedLookup(job, dlfn, true));
+                efile.setown(timedLookup(job, dlfn, true, job.queryMaxLfnBlockTimeMins() * 60000));
                 if (!efile.get())
                 {
                     ForEachItemIn(c, clusters)

thorlcr/thorutil/thormisc.hpp

@@ -96,6 +96,7 @@
 #define THOROPT_FORCE_REMOTE_DISABLED "forceRemoteDisabled"     // disable remote (via dafilesrv) reads (NB: takes precedence over forceRemoteRead) (default = false)
 #define THOROPT_FORCE_REMOTE_READ     "forceRemoteRead"         // force remote (via dafilesrv) read (NB: takes precedence over environment.conf setting) (default = false)
 #define THOROPT_ACTINIT_WAITTIME_MINS "actInitWaitTimeMins"     // max time to wait for slave activity initialization message from master
+#define THOROPT_MAXLFN_BLOCKTIME_MINS "maxLfnBlockTimeMins"     // max time permitted to be blocked on a DFS logical file operation.
 
 
 #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000  // max of row matches before selfjoin emits warning