Merge branch 'candidate-7.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/EN_US/ECLLanguageReference/ECLR_mods/RecordStructure.xml

@@ -301,8 +301,8 @@
             <entry>A previously defined data field, which implicitly provides
             the <emphasis>datatype</emphasis>,
             <emphasis>identifier</emphasis>, and
-            <emphasis>defaultvalue</emphasis> for the new field--inherited from
-            the <emphasis>sourcefield</emphasis>.</entry>
+            <emphasis>defaultvalue</emphasis> for the new field--inherited
+            from the <emphasis>sourcefield</emphasis>.</entry>
           </row>
 
           <row>
@@ -781,7 +781,7 @@ END;</programlisting>
     case field name, and the default value for Repeated is "Row." For example,
     this demonstrates "Container/Repeated":</para>
 
-    <programlisting>DATASET(PeopleNames) People{xpath('people/name'])};
+    <programlisting>DATASET(PeopleNames) People{xpath('people/name')};
           /*matches: &lt;people&gt;
                         &lt;name&gt;Gavin&lt;/name&gt;
                         &lt;name&gt;Ricardo&lt;/name&gt;
@@ -789,7 +789,7 @@ END;</programlisting>
 
     <para>This demonstrates "/Repeated":</para>
 
-    <programlisting>DATASET(Names) Names{xpath('/name'])};
+    <programlisting>DATASET(Names) Names{xpath('/name')};
           /*matches: &lt;name&gt;Gavin&lt;/name&gt;
                      &lt;name&gt;Ricardo&lt;/name&gt; */</programlisting>
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -17966,7 +17966,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
         //virtual const char * getInputIteratorPath()
         IHqlExpression * xpath = expr->queryAttribute(xpathAtom);
         if (xpath)
-            doBuildVarStringFunction(instance->classctx, "getInputIteratorPath", xpath->queryChild(0));
+            doBuildVarStringFunction(instance->startctx, "getInputIteratorPath", xpath->queryChild(0));
 
         IHqlExpression * onFail = expr->queryAttribute(onFailAtom);
         if (onFail)
@@ -18122,7 +18122,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
         //virtual const char * getInputIteratorPath()
         IHqlExpression * xpath = expr->queryAttribute(xpathAtom);
         if (xpath)
-            doBuildVarStringFunction(instance->classctx, "getInputIteratorPath", xpath->queryChild(0));
+            doBuildVarStringFunction(instance->startctx, "getInputIteratorPath", xpath->queryChild(0));
 
         IHqlExpression * onFail = expr->queryAttribute(onFailAtom);
         if (onFail)

ecl/hthor/hthorkey.cpp

@@ -2927,6 +2927,7 @@ public:
     ~CJoinGroup()
     {
         ReleaseRoxieRow(left);
+        join = nullptr; // not required, but clear to highlight any race conditions
     }
 
     MatchSet * getMatchSet()
@@ -2956,6 +2957,9 @@ public:
     inline void noteEnd()
     {
         assertex(!complete());
+        //Another completing group could cause this group to be processed once endMarkersPending is set to 0
+        //So link this object to ensure it is not disposed of while this function is executing
+        Linked<CJoinGroup> saveThis(this);
         if (atomic_dec_and_test(&groupStart->endMarkersPending))
         {
             join->onComplete(groupStart);

esp/logging/loggingmanager/loggingmanager.cpp

@@ -40,7 +40,7 @@ bool CLoggingManager::init(IPropertyTree* cfg, const char* service)
         return false;
     }
 
-    oneTankFile = cfg->getPropBool("FailSafe");
+    oneTankFile = cfg->getPropBool("FailSafe", true);
     if (oneTankFile)
     {
         logFailSafe.setown(createFailSafeLogger(cfg, service, cfg->queryProp("@name")));

esp/platform/espprotocol.cpp

@@ -45,7 +45,7 @@ void ActiveRequests::dec()
     atomic_dec(&gActiveRequests);
 }
 
-CEspApplicationPort::CEspApplicationPort(bool viewcfg, CEspProtocol* prot) : bindingCount(0), defBinding(-1), viewConfig(viewcfg), rootAuth(false), navWidth(165), navResize(false), navScroll(false), protocol(prot)
+CEspApplicationPort::CEspApplicationPort(bool viewcfg, CEspProtocol* prot) : viewConfig(viewcfg), rootAuth(false), navWidth(165), navResize(false), navScroll(false), protocol(prot)
 {
     build_ver = getBuildVersion();
 
@@ -65,12 +65,9 @@ CEspApplicationPort::CEspApplicationPort(bool viewcfg, CEspProtocol* prot) : bin
 void CEspApplicationPort::appendBinding(CEspBindingEntry* entry, bool isdefault)
 {
     WriteLockBlock wblock(rwLock);
-    if (bindingCount + 1 == MAX_ESP_BINDINGS)
-        throw MakeStringException(0,"Error - reached maximum number of bindings allowed.");
-    bindings[bindingCount]=entry;
+    bindings.append(*entry);
     if (isdefault)
-        defBinding=bindingCount;
-    bindingCount++;
+        defBinding = entry;
     EspHttpBinding *httpbind = dynamic_cast<EspHttpBinding *>(entry->queryBinding());
     if (httpbind)
     {
@@ -94,27 +91,18 @@ void CEspApplicationPort::removeBinding(IEspRpcBinding* binding)
     CEspBindingEntry* targetEntry = nullptr;
     {
         WriteLockBlock wblock(rwLock);
-        for (int i = 0; i < bindingCount; i++)
+        ForEachItemInRev(i, bindings)
         {
-            if (!bindings[i])
-                continue;
-            IEspRpcBinding* currentBinding = bindings[i]->queryBinding();
+            IEspRpcBinding* currentBinding = bindings.item(i).queryBinding();
             if (currentBinding && currentBinding == binding)
             {
-                targetEntry = bindings[i];
-                bindings[i] = nullptr;
-                if (i != bindingCount-1)
-                {
-                    bindings[i] = bindings[bindingCount-1];
-                    bindings[bindingCount-1] = nullptr;
-                }
-                bindingCount--;
+                if (defBinding == &bindings.item(i))
+                    defBinding = nullptr;
+                bindings.remove(i);
                 break;
             }
         }
     }
-    if(targetEntry != nullptr)
-        targetEntry->Release();
 }
 
 const StringBuffer &CEspApplicationPort::getAppFrameHtml(time_t &modified, const char *inner, StringBuffer &html, IEspContext* ctx)
@@ -213,9 +201,8 @@ const StringBuffer &CEspApplicationPort::getNavBarContent(IEspContext &context,
         Owned<IPropertyTree> navtree=createPTree("EspNavigationData");
         {
             ReadLockBlock rblock(rwLock);
-            int count = getBindingCount();
-            for (int idx = 0; idx<count; idx++)
-                bindings[idx]->queryBinding()->getNavigationData(context, *navtree.get());
+            ForEachItemIn(idx, bindings)
+                bindings.item(idx).queryBinding()->getNavigationData(context, *navtree.get());
         }
 
         StringBuffer xml;
@@ -259,9 +246,8 @@ const StringBuffer &CEspApplicationPort::getDynNavData(IEspContext &context, IPr
     bVolatile = false;
     {
         ReadLockBlock rblock(rwLock);
-        int count = getBindingCount();
-        for (int idx = 0; idx<count; idx++)
-            bindings[idx]->queryBinding()->getDynNavData(context, params, *navtree.get());
+        ForEachItemIn(idx, bindings)
+            bindings.item(idx).queryBinding()->getDynNavData(context, params, *navtree.get());
     }
 
     if (!bVolatile)
@@ -274,27 +260,14 @@ int CEspApplicationPort::onGetNavEvent(IEspContext &context, IHttpMessage* reque
 {
     int handled=0;
     ReadLockBlock rblock(rwLock);
-    int count = getBindingCount();
-    for (int idx = 0; !handled && idx<count; idx++)
-    {
-        handled = bindings[idx]->queryBinding()->onGetNavEvent(context, request, response);
-    }
+    for (int idx = 0; !handled && idx < bindings.length(); idx++)
+        handled = bindings.item(idx).queryBinding()->onGetNavEvent(context, request, response);
     return handled;
 }
 
 int CEspApplicationPort::onBuildSoapRequest(IEspContext &context, IHttpMessage* ireq, IHttpMessage* iresp)
 {
-    CHttpRequest *request=dynamic_cast<CHttpRequest*>(ireq);
-    CHttpResponse *response=dynamic_cast<CHttpResponse*>(iresp);
-
-    int handled=0;
-    ReadLockBlock rblock(rwLock);
-    int count = getBindingCount();
-    for (int idx = 0; !handled && idx<count; idx++)
-    {
-        //if (bindings[idx]->queryBinding()->isValidServiceName(context, ))
-    }
-    return handled;
+    return 0;
 }
 
 void CEspApplicationPort::buildNavTreeXML(IPropertyTree* navtree, StringBuffer& xmlBuf, bool insideFolder)
@@ -763,7 +736,7 @@ int CEspProtocol::removeBindingMap(int port, IEspRpcBinding* binding)
     {
         CEspApplicationPort* apport = (*apport_it).second;
         apport->removeBinding(binding);
-        left = apport->countBindings();
+        left = apport->getBindingCount();
         if (left == 0)
         {
             delete apport;
@@ -780,5 +753,5 @@ int CEspProtocol::countBindings(int port)
     if (!apport)
         return 0;
     else
-        return apport->countBindings();
+        return apport->getBindingCount();
 }

esp/platform/espprotocol.hpp

@@ -47,7 +47,7 @@ public:
     static long getCount();
 };
 
-class CEspBindingEntry : public CInterface
+class CEspBindingEntry : public CInterface, implements IInterface
 {
 private:
     Owned<ISocket> sock_;
@@ -84,12 +84,10 @@ public:
 
 class CEspProtocol;
 
-#define MAX_ESP_BINDINGS 512
 class CEspApplicationPort
 {
-    CEspBindingEntry* bindings[512];
-    int bindingCount;
-    int defBinding;
+    IArrayOf<CEspBindingEntry> bindings;
+    CEspBindingEntry* defBinding = nullptr;
 
     StringBuffer titleBarHtml;
     StringBuffer appFrameHtml;
@@ -109,8 +107,6 @@ public:
 
     ~CEspApplicationPort()
     {
-        while (bindingCount)
-            bindings[--bindingCount]->Release();
         if (hxsl)
             FreeSharedObject(hxsl);
     }
@@ -125,7 +121,7 @@ public:
     int onGetNavEvent(IEspContext &context, IHttpMessage* request, IHttpMessage* response);
     int onBuildSoapRequest(IEspContext &context, IHttpMessage* request, IHttpMessage* response);
 
-    int getBindingCount(){return bindingCount;}
+    int getBindingCount(){return bindings.length();}
     void appendBinding(CEspBindingEntry* entry, bool isdefault);
     void removeBinding(IEspRpcBinding* binding);
 
@@ -134,11 +130,18 @@ public:
     CEspBindingEntry* queryBindingItem(int item)
     {
         ReadLockBlock rblock(rwLock);
-        return (item<bindingCount) ? bindings[item] : nullptr;
+        return (item<bindings.length()) ? &bindings.item(item) : nullptr;
+    }
+    CEspBindingEntry* getDefaultBinding()
+    {
+        ReadLockBlock rblock(rwLock);
+        if (defBinding)
+            return defBinding;
+        if (bindings.length() > 0)
+            return &bindings.item(0);
+        return nullptr;
     }
-    CEspBindingEntry* getDefaultBinding(){return bindings[(defBinding>=0) ? defBinding : 0];}
     CEspProtocol* queryProtocol() { return protocol; }
-    int countBindings() { return bindingCount; }
 #ifdef _USE_OPENLDAP
     unsigned updatePassword(IEspContext &context, IHttpMessage* request, StringBuffer& message);
     void onUpdatePasswordInput(IEspContext &context, StringBuffer &html);

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -1158,7 +1158,7 @@ void retrieveQuerysetDetails(IEspContext &context, IArrayOf<IEspWUQuerySetDetail
     retrieveQuerysetDetails(context, details, registry, type, value, cluster, queriesOnCluster);
 }
 
-IPropertyTree* getQueriesOnCluster(const char *target, const char *queryset, bool checkAllNodes)
+IPropertyTree* getQueriesOnCluster(const char *target, const char *queryset, StringArray *queryIDs, bool checkAllNodes)
 {
     if (isEmpty(target))
         target = queryset;
@@ -1175,11 +1175,21 @@ IPropertyTree* getQueriesOnCluster(const char *target, const char *queryset, boo
 
     try
     {
+        StringBuffer control;
+        if (!queryIDs || (queryIDs->ordinality() == 0))
+            control.append("<control:queries/>");
+        else
+        {
+            control.append("<control:queries>");
+            ForEachItemIn(i, *queryIDs)
+                control.appendf("<Query id='%s'/>",  queryIDs->item(i));
+            control.append("</control:queries>");
+        }
         Owned<ISocket> sock = ISocket::connect_timeout(eps.item(0), ROXIECONNECTIONTIMEOUT);
         if (checkAllNodes)
-            return sendRoxieControlAllNodes(sock, "<control:queries/>", false, ROXIECONTROLQUERIESTIMEOUT);
+            return sendRoxieControlAllNodes(sock, control, false, ROXIECONTROLQUERIESTIMEOUT);
         else
-            return sendRoxieControlQuery(sock, "<control:queries/>", ROXIECONTROLQUERIESTIMEOUT);
+            return sendRoxieControlQuery(sock, control, ROXIECONTROLQUERIESTIMEOUT);
     }
     catch(IException* e)
     {
@@ -1192,7 +1202,7 @@ IPropertyTree* getQueriesOnCluster(const char *target, const char *queryset, boo
 
 void retrieveQuerysetDetailsByCluster(IEspContext &context, IArrayOf<IEspWUQuerySetDetail> &details, const char *target, const char *queryset, const char *type, const char *value, bool checkAllNodes)
 {
-    Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(target, queryset, checkAllNodes);
+    Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(target, queryset, nullptr, checkAllNodes);
     retrieveQuerysetDetails(context, details, target, type, value, target, queriesOnCluster);
 }
 
@@ -1232,7 +1242,7 @@ bool CWsWorkunitsEx::onWUQuerysetDetails(IEspContext &context, IEspWUQuerySetDet
         const char* cluster = req.getClusterName();
         if (isEmpty(cluster))
             cluster = req.getQuerySetName();
-        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(cluster, req.getQuerySetName(), req.getCheckAllNodes());
+        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(cluster, req.getQuerySetName(), nullptr, req.getCheckAllNodes());
         retrieveQuerysetDetails(context, registry, req.getFilterTypeAsString(), req.getFilter(), respQueries, respAliases, cluster, queriesOnCluster);
 
         resp.setQuerysetQueries(respQueries);
@@ -1347,7 +1357,21 @@ void CWsWorkunitsEx::checkAndSetClusterQueryState(IEspContext &context, const ch
         double version = context.getClientVersion();
         if (isEmpty(cluster))
             cluster = querySetId;
-        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(cluster, querySetId, checkAllNodes);
+
+        StringArray queryIDs;
+        ForEachItemIn(j, queries)
+        {
+            IEspQuerySetQuery& query = queries.item(j);
+            const char* queryId = query.getId();
+            const char* querySetId0 = query.getQuerySetId();
+            if (queryId && querySetId0 && strieq(querySetId0, querySetId))
+                queryIDs.append(queryId);
+        }
+
+        if (queryIDs.ordinality() == 0)
+            return;
+
+        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(cluster, querySetId, &queryIDs, checkAllNodes);
         if (!queriesOnCluster)
         {
             DBGLOG("getQueriesOnCluster() returns NULL for cluster<%s> and querySetId<%s>", cluster, querySetId);
@@ -1539,7 +1563,8 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
         queries.append(*q.getClear());
     }
 
-    checkAndSetClusterQueryState(context, clusterReq, querySetIds, queries, req.getCheckAllNodes());
+    if (queries.ordinality() > 0)
+        checkAndSetClusterQueryState(context, clusterReq, querySetIds, queries, req.getCheckAllNodes());
 
     resp.setQuerysetQueries(queries);
     resp.setNumberOfQueries(numberOfQueries);
@@ -1927,7 +1952,10 @@ bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRe
     }
     if (includeStateOnClusters && (version >= 1.43))
     {
-        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(querySet, querySet, req.getCheckAllNodes());
+        StringArray queryIds;
+        queryIds.append(queryId);
+
+        Owned<IPropertyTree> queriesOnCluster = getQueriesOnCluster(querySet, querySet, &queryIds, req.getCheckAllNodes());
         if (queriesOnCluster)
         {
             IArrayOf<IEspClusterQueryState> clusterStates;

esp/src/eclwatch/ActivityWidget.js

@@ -531,7 +531,7 @@ define([
             },
 
             createDetail: function (id, row, params) {
-                if (params.usage) {
+                if (params && params.usage) {
                     return new DelayLoadWidget({
                         id: id,
                         title: row.details.Name,

esp/src/eclwatch/DFUQueryWidget.js

@@ -55,6 +55,7 @@ define([
     "dijit/form/DropDownButton",
     "dijit/form/Select",
     "dijit/form/CheckBox",
+    "dijit/form/NumberTextBox",
     "dijit/form/RadioButton",
     "dijit/Dialog",
     "dijit/Toolbar",

esp/src/eclwatch/LFDetailsWidget.js

@@ -99,6 +99,7 @@ define([
                 this.replicateSourceLogicalFile = registry.byId(this.id + "ReplicateSourceLogicalFile");
                 this.replicateDropDown = registry.byId(this.id + "ReplicateDropDown");
                 this.desprayIPSelect = registry.byId(this.id + "DesprayTargetIPAddress");
+                this.isProtected = registry.byId(this.id + "isProtected");
                 var context = this;
                 var origOnOpen = this.desprayTooltiopDialog.onOpen;
                 this.desprayTooltiopDialog.onOpen = function () {
@@ -261,6 +262,10 @@ define([
                     Groups: true
                 });
                 this.logicalFile.refresh();
+
+                this.isProtected.on("change", function(evt){
+                    context._onSave();
+                });
             },
 
             initTab: function () {

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -49,7 +49,7 @@
                     <div id="${id}CopyDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Copy}</span>
                         <div data-dojo-type="dijit.TooltipDialog">
-                            <div id="${id}CopyForm" style="width: 460px;" onsubmit="return false;" data-dojo-props="region: 'bottom'" data-dojo-type="dijit.form.Form">
+                            <div id="${id}CopyForm" style="width: 650px;" onsubmit="return false;" data-dojo-props="region: 'bottom'" data-dojo-type="dijit.form.Form">
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
@@ -68,6 +68,7 @@
                                         <input id="${id}CopyTargetRetainSuperfileStructure" title="${i18n.RetainSuperfileStructure}:" name="superCopy" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyPreserveCompression" title="${i18n.PreserveCompression}:" checked="true" name="preserveCompression" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyTargetReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -11,7 +11,7 @@
                     <div id="${id}CopyDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Copy}</span>
                         <div data-dojo-type="dijit.TooltipDialog">
-                            <div id="${id}CopyForm" style="width: 460px;" onsubmit="return false;" data-dojo-type="dijit.form.Form">
+                            <div id="${id}CopyForm" style="width: 650px;" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
@@ -29,6 +29,7 @@
                                         <input id="${id}CopyTargetWrap" title="${i18n.Wrap}:" name="Wrap" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyTargetRetainSuperfileStructure" title="${i18n.RetainSuperfileStructure}:" name="superCopy" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyPreserveCompression" title="${i18n.PreserveCompression}:" checked="true" name="preserveCompression" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox", />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">

esp/xslt/esdl2monitor.xslt

@@ -26,6 +26,8 @@
     <xsl:param name="diffaction" select="'Run'"/>
     <xsl:param name="listCategories" select="false()"/>
     <xsl:variable name="docname" select="/esxdl/@name"/>
+    <xsl:param name="skipResponseTag" select="substring($responseType, string-length($responseType) - 1)='Ex'"/>
+
     <xsl:template match="/">
         <xsl:apply-templates select="esxdl"/>
     </xsl:template>
@@ -367,7 +369,7 @@ END;
 </xsl:choose>
 
   output(executedAction.id, NAMED('MonitorId'));
-  output(executedAction.report, NAMED('Result'));
+  output(executedAction.report<xsl:if test="$skipResponseTag">[1].response</xsl:if>, NAMED('Result'));
 
 <xsl:if test="$listCategories">
   <xsl:if test="Template//*[@diff_monitor]">

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -65,7 +65,7 @@ xmlns:set="http://exslt.org/sets">
                 
             <LogDataXPath>
                 <xsl:for-each select="$agentNode/LogDataItem">
-                    <LogDataItem name="{current()/@name}" XPath="{current()/@xpath}" xsl="{current()/@xsl}" default="{current()/@default}"/>
+                    <LogDataItem name="{current()/@name}" XPath="{current()/@xpath}" xsl="{current()/@xsl}" encode="{current()/@encode}" default="{current()/@default}"/>
                 </xsl:for-each>
                 <xsl:for-each select="$agentNode/LogInfo">
                     <LogInfo name="{current()/@name}" default="{current()/@default}" XPath="{current()/@xpath}" xsl="{current()/@xsl}" multiple="{current()/@multiple}" encode="{current()/@encode}" type="{current()/@type}"/>

plugins/spark/CMakeLists.txt

@@ -59,7 +59,7 @@ if(SPARK)
 
     if(NOT SPARK_HPCC_JAR)
         if(NOT SPARK_HPCC_VERSION)
-            set(SPARK_HPCC_VERSION "7.2.0.1")
+            set(SPARK_HPCC_VERSION "7.2.6")
         endif()
         file(DOWNLOAD
             ${CENTRAL_REPO}/org/hpccsystems/spark-hpcc/${SPARK_HPCC_VERSION}/spark-hpcc-${SPARK_HPCC_VERSION}.jar
@@ -71,7 +71,7 @@ if(SPARK)
 
     if(NOT DFSCLIENT_JAR)    
         if(NOT DFSCLIENT_VERSION)
-            set(DFSCLIENT_VERSION "7.2.0.1")
+            set(DFSCLIENT_VERSION "7.2.6")
         endif()
         file(DOWNLOAD
             ${CENTRAL_REPO}/org/hpccsystems/dfsclient/${DFSCLIENT_VERSION}/dfsclient-${DFSCLIENT_VERSION}-jar-with-dependencies.jar

roxie/ccd/ccdsnmp.cpp

@@ -624,6 +624,24 @@ class CQueryStatsAggregator : public CInterface, implements IQueryStatsAggregato
                 result.setPropInt("percentile97", 0);
         }
 
+        static void getRawStats(IPropertyTree &result, std::deque<QueryStatsRecord> &useStats)
+        {
+            for (auto r : useStats)
+            {
+                Owned<IPropertyTree> queryStatsRecord = createPTree("QueryStatsRecord", ipt_fast);
+                CDateTime dt;
+                StringBuffer s;
+                dt.set(r.startTime);
+                queryStatsRecord->setProp("@startTime", dt.getString(s.clear(), true).str());
+                queryStatsRecord->setPropInt("elapsedTimeMs", r.elapsedTimeMs);
+                queryStatsRecord->setPropInt("memUsed", r.memUsed);
+                queryStatsRecord->setPropInt("slavesReplyLen", r.slavesReplyLen);
+                queryStatsRecord->setPropInt("bytesOut", r.bytesOut);
+                queryStatsRecord->setPropBool("failed", r.failed);
+                result.addPropTree(queryStatsRecord->queryName(), LINK(queryStatsRecord));
+            }
+        }
+
         static bool checkOlder(const void *_left, const void *_right)
         {
             QueryStatsRecord *left = (QueryStatsRecord *) _left;
@@ -673,6 +691,19 @@ class CQueryStatsAggregator : public CInterface, implements IQueryStatsAggregato
             return (difftime(startTime, from) >= 0 && difftime(to, endTime) > 0);
         }
 
+        bool timeOverlap(time_t from, time_t to)
+        {
+            if (from == startTime)
+                return true;
+
+            double diffFrom = difftime(from, startTime);
+            if (diffFrom > 0 && difftime(endTime, from) > 0)
+                return true;
+            if (diffFrom < 0 && difftime(to, startTime) > 0)
+                return true;
+            return false;
+        }
+
         bool matches(time_t queryTime)
         {
             return (difftime(queryTime, startTime) >= 0 && difftime(queryTime, endTime) < 0);
@@ -785,6 +816,16 @@ class CQueryStatsAggregator : public CInterface, implements IQueryStatsAggregato
                 }
             }
         }
+
+        static void getRawStats(IPropertyTree &result, std::deque<QueryStatsAggregateRecord> &useStats, time_t from, time_t to)
+        {
+            for (auto r : useStats)
+            {
+                Owned<IPropertyTree> queryStatsAggregateRecord = createPTree("QueryStatsAggregateRecord", ipt_fast);
+                r.getStats(*queryStatsAggregateRecord, true);
+                result.addPropTree(queryStatsAggregateRecord->queryName(), LINK(queryStatsAggregateRecord));
+            }
+        }
     };
 
     CriticalSection statsLock;  // Protects multithreaded access to recent and aggregated structures
@@ -877,7 +918,7 @@ public:
     ~CQueryStatsAggregator()
     {
     }
-    static IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
+    static IPropertyTree *getAllQueryStats(bool includeQueries, bool rawStats, time_t from, time_t to)
     {
         Owned<IPTree> result = createPTree("QueryStats", ipt_fast);
         if (includeQueries)
@@ -886,10 +927,16 @@ public:
             ForEachItemIn(idx, queryStatsAggregators)
             {
                 CQueryStatsAggregator &thisQuery = queryStatsAggregators.item(idx);
-                result->addPropTree("Query", thisQuery.getStats(from, to));
+                if (!rawStats)
+                    result->addPropTree("Query", thisQuery.getStats(from, to));
+                else
+                    result->addPropTree("Query", thisQuery.getRawStats(from, to));
             }
         }
-        result->addPropTree("Global", globalStatsAggregator.getStats(from, to));
+        if (!rawStats)
+            result->addPropTree("Global", globalStatsAggregator.getStats(from, to));
+        else
+            result->addPropTree("Global", globalStatsAggregator.getRawStats(from, to));
         return result.getClear();
     }
 
@@ -950,6 +997,39 @@ public:
         }
         return result.getClear();
     }
+    virtual IPropertyTree *getRawStats(time_t from, time_t to)
+    {
+        Owned<IPropertyTree> result = createPTree("Query", ipt_fast);
+        result->setProp("@id", queryName);
+
+        std::deque<QueryStatsRecord> useStats;
+        {
+            CriticalBlock b(statsLock);
+            for (auto rec : recent)
+            {
+                if (rec.inRange(from, to))
+                    useStats.push_back(rec);
+            }
+            // lock is released here, and we process the useStats array at our leisure...
+        }
+        QueryStatsRecord::getRawStats(*result, useStats);
+
+        std::deque<QueryStatsAggregateRecord> aggregatedStats;
+        {
+            CriticalBlock b(statsLock);
+            for (auto thisSlot: aggregated)
+            {
+                if (thisSlot.timeOverlap(from, to))
+                    aggregatedStats.push_back(thisSlot);
+                else if (thisSlot.older(from))
+                    break;
+            }
+            // lock is released here, and we process the aggregator at our leisure...
+        }
+        QueryStatsAggregateRecord::getRawStats(*result, aggregatedStats, from, to);
+
+        return result.getClear();
+    }
     static inline IQueryStatsAggregator *queryGlobalStatsAggregator()
     {
         return &globalStatsAggregator;
@@ -970,9 +1050,9 @@ IQueryStatsAggregator *createQueryStatsAggregator(const char *_queryName, unsign
     return new CQueryStatsAggregator(_queryName, _expirySeconds);
 }
 
-IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
+IPropertyTree *getAllQueryStats(bool includeQueries, bool rawStats, time_t from, time_t to)
 {
-     return CQueryStatsAggregator::getAllQueryStats(includeQueries, from, to);
+     return CQueryStatsAggregator::getAllQueryStats(includeQueries, rawStats, from, to);
 }
 
 //=======================================================================================================
@@ -1020,7 +1100,13 @@ protected:
             DBGLOG("%s", stats.str());
         }
         {
-            Owned<IPropertyTree> p = getAllQueryStats(true, start, end);
+            Owned<IPropertyTree> p = getAllQueryStats(true, false, start, end);
+            StringBuffer stats; 
+            toXML(p, stats);
+            DBGLOG("%s", stats.str());
+        }
+        {
+            Owned<IPropertyTree> p = getAllQueryStats(true, true, start, end);
             StringBuffer stats; 
             toXML(p, stats);
             DBGLOG("%s", stats.str());

roxie/ccd/ccdsnmp.hpp

@@ -75,7 +75,7 @@ interface IQueryStatsAggregator : public IInterface
 
 extern IQueryStatsAggregator *queryGlobalQueryStatsAggregator();
 extern IQueryStatsAggregator *createQueryStatsAggregator(const char *queryName, unsigned expirySeconds);
-extern IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to);
+extern IPropertyTree *getAllQueryStats(bool includeQueries, bool rawStats, time_t from, time_t to);
 
 extern RelaxedAtomic<unsigned> queryCount;
 extern RoxieQueryStats unknownQueryStats;

roxie/ccd/ccdstate.cpp

@@ -2478,7 +2478,8 @@ private:
                 else
                 {
                     bool includeAllQueries = control->getPropBool("@all", true);
-                    Owned<const IPropertyTree> stats = getAllQueryStats(includeAllQueries, from, to);
+                    bool rawStats = control->getPropBool("@rawStats", false);
+                    Owned<const IPropertyTree> stats = getAllQueryStats(includeAllQueries, rawStats, from, to);
                     toXML(stats, reply);
                 }
             }

tools/esdlcmd/esdlcmd_monitor.cpp

@@ -993,9 +993,13 @@ public:
         if (!espRespTree)
             throw( MakeStringException(0, "Esdl Response type '%s' definition not found", esp_resp_type.str()));
 
+        bool skipOutputResponseTag = false;
         StringBuffer resp_type(esp_resp_type.get());
         if (resp_type.length()>2 && resp_type.charAt(resp_type.length()-2)=='E' && resp_type.charAt(resp_type.length()-1)=='x')
+        {
+            skipOutputResponseTag = true;
             resp_type.setLength(resp_type.length()-2);
+        }
 
         IPropertyTree *respTree = checkExtractNestedResponseType(depTree, espRespTree, resp_type);
 
@@ -1066,12 +1070,14 @@ public:
 
         xform->setParameter("platform", "'roxie'");
         xform->setParameter("responseType", stringvar.setf("'%s'", resp_type.str()));
+        xform->setParameter("skipResponseTag", "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorRoxie_create_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
 
         xform->setParameter("platform", "'esp'");
         xform->setParameter("responseType", stringvar.setf("'%s'", esp_resp_type.str()));
+        xform->setParameter("skipResponseTag", skipOutputResponseTag ? "true()" : "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorESP_create_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
@@ -1081,12 +1087,14 @@ public:
 
         xform->setParameter("platform", "'roxie'");
         xform->setParameter("responseType", stringvar.setf("'%s'", resp_type.str()));
+        xform->setParameter("skipResponseTag", "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorRoxie_run_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
 
         xform->setParameter("platform", "'esp'");
         xform->setParameter("responseType", stringvar.setf("'%s'", esp_resp_type.str()));
+        xform->setParameter("skipResponseTag", skipOutputResponseTag ? "true()" : "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorESP_run_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
@@ -1096,12 +1104,14 @@ public:
 
         xform->setParameter("platform", "'roxie'");
         xform->setParameter("responseType", stringvar.setf("'%s'", resp_type.str()));
+        xform->setParameter("skipResponseTag", "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorRoxie_demo_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
 
         xform->setParameter("platform", "'esp'");
         xform->setParameter("responseType", stringvar.setf("'%s'", esp_resp_type.str()));
+        xform->setParameter("skipResponseTag", skipOutputResponseTag ? "true()" : "false()");
         xform->transform(ecl.clear());
         filename.setf("MonitorESP_demo_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
@@ -1109,6 +1119,7 @@ public:
 //-------Compare---------
         xform->setParameter("diffmode", "'Compare'");
         xform->setParameter("responseType", stringvar.setf("'%s'", resp_type.str()));
+        xform->setParameter("skipResponseTag", "false()");
         xform->transform(ecl.clear());
         filename.setf("Compare_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);