Merge remote-tracking branch 'origin/candidate-3.8.2' into candidate-3.8.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvresultset.cpp

@@ -3355,7 +3355,7 @@ extern "C" FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSet
         text.append("<XmlSchema name=\"").append(schemaName).append("\">");
         const IResultSetMetaData & meta = cursor->queryResultSet()->getMetaData();
         StringBufferAdaptor adaptor(text);
-        meta.getXmlSchema(adaptor, false);
+        meta.getXmlXPathSchema(adaptor, false);
         text.append("</XmlSchema>").newline();
     }
 

common/workunit/workunit.cpp

@@ -8344,8 +8344,8 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
 {
     StringBuffer wuRoot;
     getXPath(wuRoot, wuid);
-    WorkUnitWaiter waiter(wuRoot.str());
-    LocalIAbortHandler abortHandler(waiter);
+    Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuRoot.str());
+    LocalIAbortHandler abortHandler(*waiter);
     WUState ret = WUStateUnknown;
     Owned<IRemoteConnection> conn = querySDS().connect(wuRoot.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
     if (conn)
@@ -8364,12 +8364,12 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
             case WUStateCompleted:
             case WUStateFailed:
             case WUStateAborted:
-                waiter.unsubscribe();
+                waiter->unsubscribe();
                 return ret;
             case WUStateWait:
                 if(returnOnWaitState)
                 {
-                    waiter.unsubscribe();
+                    waiter->unsubscribe();
                     return ret;
                 }
                 break;
@@ -8384,7 +8384,7 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
                     SessionId agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
                     if((agent>0) && querySessionManager().sessionStopped(agent, 0))
                     {
-                        waiter.unsubscribe();
+                        waiter->unsubscribe();
                         conn->reload();
                         ret = (WUState) getEnum(conn->queryRoot(), "@state", states);
                         bool isEcl = false;
@@ -8418,14 +8418,14 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
             unsigned waited = msTick() - start;
             if (timeout==-1)
             {
-                waiter.wait(20000);  // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
-                if (waiter.aborted)
+                waiter->wait(20000);  // recheck state every 20 seconds even if no timeout, in case eclagent has crashed.
+                if (waiter->aborted)
                 {
                     ret = WUStateUnknown;  // MORE - throw an exception?
                     break;
                 }
             }
-            else if (waited > timeout || !waiter.wait(timeout-waited))
+            else if (waited > timeout || !waiter->wait(timeout-waited))
             {
                 ret = WUStateUnknown;  // MORE - throw an exception?
                 break;
@@ -8433,7 +8433,7 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
             conn->reload();
         }
     }
-    waiter.unsubscribe();
+    waiter->unsubscribe();
     return ret;
 }
 

ecl/agentexec/agentexec.cpp

@@ -137,6 +137,10 @@ int CEclAgentExecutionServer::run()
         ERRLOG("Terminating unexpectedly");
     }
 
+    CSDSServerStatus serverStatus("HThorServer");
+    serverStatus.queryProperties()->setProp("@queue",queueNames.str());
+    serverStatus.queryProperties()->setProp("@cluster", agentName);
+    serverStatus.commitProperties();
     writeSentinelFile(sentinelFile);
 
     try 

esp/eclwatch/ws_XSLT/result_lib.xslt

@@ -235,12 +235,10 @@
         </xsl:if>
     </xsl:template>
     
-    
-    <xsl:template name="grab-snodes">
+    <xsl:template name="grab-snodes-item">
         <xsl:param name="schema"/>
         <xsl:param name="node"/>
         <xsl:param name="height" select="1"/>
-        <xsl:for-each select="$schema/xs:complexType/xs:sequence/xs:element">
             <xsl:variable name="name">
                 <xsl:call-template name="getMatchingName">
                     <xsl:with-param name="schema" select="."/>
@@ -248,6 +246,9 @@
                 </xsl:call-template>
             </xsl:variable>
             <xsl:choose>
+                <xsl:when test="name()='xs:attribute'">
+                    <h name="{concat('@',$name)}" height="{$height}" leaf="1"/>
+                </xsl:when>
                 <xsl:when test="string(../@maxOccurs)!='unbounded'">
                     <xsl:variable name="childNode" select="$node/*[name()=$name]"/>
                     <h name="{$name}" height="{$height}">
@@ -276,6 +277,25 @@
                     </xsl:choose>
                 </xsl:otherwise>
             </xsl:choose>
+    </xsl:template>
+
+    <xsl:template name="grab-snodes">
+        <xsl:param name="schema"/>
+        <xsl:param name="node"/>
+        <xsl:param name="height" select="1"/>
+        <xsl:for-each select="$schema/xs:complexType/xs:attribute">
+            <xsl:call-template name="grab-snodes-item">
+                <xsl:with-param name="schema" select="$schema"/>
+                <xsl:with-param name="node" select="$node"/>
+                <xsl:with-param name="height" select="$height"/>
+            </xsl:call-template>
+        </xsl:for-each>
+        <xsl:for-each select="$schema/xs:complexType/xs:sequence/xs:element">
+            <xsl:call-template name="grab-snodes-item">
+                <xsl:with-param name="schema" select="$schema"/>
+                <xsl:with-param name="node" select="$node"/>
+                <xsl:with-param name="height" select="$height"/>
+            </xsl:call-template>
         </xsl:for-each>
     </xsl:template>
     
@@ -364,7 +384,55 @@
         <!--/xsl:if-->
     </xsl:template>
     
-    
+    <xsl:template name="grab-column">
+            <xsl:param name="schema"/>
+            <xsl:param name="level"/>
+            <xsl:param name="rowSchema"/>
+            <xsl:param name="name"/>
+            <xsl:param name="dname"/>
+            <xsl:param name="matchingData"/>
+            <xsl:param name="matchingData2"/>
+            <xsl:choose>
+                <xsl:when test="$matchingData|$matchingData2">
+                    <xsl:choose>
+                        <xsl:when test="$rowSchema/xs:complexType">
+                            <xsl:for-each select="$matchingData|$matchingData2">
+                                <xsl:call-template name="grab-data">
+                                    <xsl:with-param name="level" select="$level"/>
+                                    <xsl:with-param name="schema" select="$rowSchema"/>
+                                    <xsl:with-param name="data" select="."/>
+                                </xsl:call-template>
+                            </xsl:for-each>
+                        </xsl:when>
+                        <xsl:when test="starts-with($rowSchema/@type, 'setof_')">
+                            <xsl:for-each select="$matchingData|$matchingData2">
+                                <xsl:call-template name="grab-data">
+                                    <xsl:with-param name="level" select="$level"/>
+                                    <xsl:with-param name="schema" select="$rowSchema"/>
+                                    <xsl:with-param name="data" select="."/>
+                                </xsl:call-template>
+                            </xsl:for-each>
+                        </xsl:when>
+                        <xsl:otherwise>
+                            <data>
+                                <xsl:for-each select="$matchingData|$matchingData2">
+                                    <xsl:value-of select="."/>
+                                    <xsl:if test="position()!=last()">, </xsl:if>
+                                </xsl:for-each>
+                            </data>
+                        </xsl:otherwise>
+                    </xsl:choose>
+                </xsl:when>
+                <xsl:otherwise>
+                    <xsl:call-template name="grab-data">
+                        <xsl:with-param name="level" select="$level"/>
+                        <xsl:with-param name="schema" select="$rowSchema"/>
+                        <xsl:with-param name="data" select="/.."/><!--empty node set-->
+                    </xsl:call-template>
+                </xsl:otherwise>
+            </xsl:choose>
+    </xsl:template>
+
     <xsl:template name="grab-row">
         <xsl:param name="schema"/>
         <xsl:param name="data"/>
@@ -407,51 +475,33 @@
                     </xsl:choose>
             </xsl:when>
             <xsl:otherwise>
+                <xsl:for-each select="$schema/xs:complexType/xs:attribute">
+                    <xsl:variable name="name" select="@name"/>
+                    <xsl:variable name="matchingData" select="$data/@*[name()=$name]"/>
+                    <xsl:variable name="dname" select="translate($name, '_', '-')"/>
+                    <xsl:call-template name="grab-column">
+                        <xsl:with-param name="level" select="$level"/>
+                        <xsl:with-param name="schema" select="$schema"/>
+                        <xsl:with-param name="rowSchema" select="."/>
+                        <xsl:with-param name="name" select="$name"/>
+                        <xsl:with-param name="dname" select="$dname"/>
+                        <xsl:with-param name="matchingData" select="$matchingData"/>
+                        <xsl:with-param name="matchingData2" select="$data[not($matchingData)]/@*[name()=$dname]"/>
+                    </xsl:call-template>
+                </xsl:for-each>
                 <xsl:for-each select="$schema/xs:complexType/xs:sequence/xs:element">
-                    <xsl:variable name="rowSchema" select="."/>
                     <xsl:variable name="name" select="@name"/>
+                    <xsl:variable name="matchingData" select="$data/*[name()=$name]"/>
                     <xsl:variable name="dname" select="translate($name, '_', '-')"/>
-                    <xsl:variable name="matchingData" select="$data/*[name()=$name]"/>          
-                    <xsl:variable name="matchingData2" select="$data[not($matchingData)]/*[name()=$dname]"/>            
-                    <xsl:choose>
-                        <xsl:when test="$matchingData|$matchingData2">
-                            <xsl:choose>
-                                <xsl:when test="$rowSchema/xs:complexType">
-                                    <xsl:for-each select="$matchingData|$matchingData2">
-                                        <xsl:call-template name="grab-data">
-                                            <xsl:with-param name="level" select="$level"/>
-                                            <xsl:with-param name="schema" select="$rowSchema"/>
-                                            <xsl:with-param name="data" select="."/>
-                                        </xsl:call-template>
-                                    </xsl:for-each>
-                                </xsl:when>
-                                <xsl:when test="starts-with($rowSchema/@type, 'setof_')">
-                                    <xsl:for-each select="$matchingData|$matchingData2">
-                                        <xsl:call-template name="grab-data">
-                                            <xsl:with-param name="level" select="$level"/>
-                                            <xsl:with-param name="schema" select="$rowSchema"/>
-                                            <xsl:with-param name="data" select="."/>
-                                        </xsl:call-template>
-                                    </xsl:for-each>
-                                </xsl:when>
-                                <xsl:otherwise>
-                                    <data>
-                                        <xsl:for-each select="$matchingData|$matchingData2">
-                                            <xsl:value-of select="."/>
-                                            <xsl:if test="position()!=last()">, </xsl:if> 
-                                        </xsl:for-each>
-                                    </data>
-                                </xsl:otherwise>
-                            </xsl:choose>
-                        </xsl:when>
-                        <xsl:otherwise>
-                            <xsl:call-template name="grab-data">
-                                <xsl:with-param name="level" select="$level"/>
-                                <xsl:with-param name="schema" select="$rowSchema"/>
-                                <xsl:with-param name="data" select="/.."/><!--empty node set-->
-                            </xsl:call-template>
-                        </xsl:otherwise>
-                    </xsl:choose>
+                    <xsl:call-template name="grab-column">
+                        <xsl:with-param name="level" select="$level"/>
+                        <xsl:with-param name="schema" select="$schema"/>
+                        <xsl:with-param name="rowSchema" select="."/>
+                        <xsl:with-param name="name" select="$name"/>
+                        <xsl:with-param name="dname" select="$dname"/>
+                        <xsl:with-param name="matchingData" select="$matchingData"/>
+                        <xsl:with-param name="matchingData2" select="$data[not($matchingData)]/*[name()=$dname]"/>
+                    </xsl:call-template>
                 </xsl:for-each>
             </xsl:otherwise>
         </xsl:choose>

esp/services/ws_smc/ws_smcService.cpp

@@ -389,9 +389,9 @@ bool CWsSMCEx::onActivity(IEspContext &context, IEspActivityRequest &req, IEspAc
                 const char* name = node.queryProp("@name");
                 if (name && *name)
                 {
+                    node.getProp("@queue", qname);
                     if (0 == stricmp("ThorMaster", name))
                     {
-                        node.getProp("@queue", qname);
                         node.getProp("@thorname",instance);
                     }
                     else if (0 == stricmp(name, "ECLAgent"))

roxie/ccd/ccddali.cpp

@@ -130,6 +130,7 @@ private:
     Owned<IUserDescriptor> userdesc;
     InterruptableSemaphore disconnectSem;
     IArrayOf<IDaliPackageWatcher> watchers;
+    CSDSServerStatus *serverStatus;
 
     class CRoxieDaliConnectWatcher : public Thread
     {
@@ -273,7 +274,7 @@ private:
 public:
 
     IMPLEMENT_IINTERFACE;
-    CRoxieDaliHelper() : connectWatcher(this)
+    CRoxieDaliHelper() : connectWatcher(this), serverStatus(NULL)
     {
         if (topology)
         {
@@ -439,6 +440,34 @@ public:
         return wuFactory->openWorkUnit(wuid, false);
     }
 
+    virtual void noteWorkunitRunning(const char *wuid, bool running)
+    {
+        CriticalBlock b(daliConnectionCrit);
+        if (isConnected)
+        {
+            assertex(serverStatus);
+            if (running)
+                serverStatus->queryProperties()->addProp("WorkUnit",wuid);
+            else
+            {
+                VStringBuffer xpath("WorkUnit[.='%s']",wuid);
+                serverStatus->queryProperties()->removeProp(xpath.str());
+            }
+            serverStatus->commitProperties();
+        }
+    }
+
+    virtual void noteQueuesRunning(const char *queueNames)
+    {
+        CriticalBlock b(daliConnectionCrit);
+        if (isConnected)
+        {
+            assertex(serverStatus);
+            serverStatus->queryProperties()->setProp("@queue", queueNames);
+            serverStatus->commitProperties();
+        }
+    }
+
     static IRoxieDaliHelper *connectToDali(unsigned waitToConnect)
     {
         CriticalBlock b(daliHelperCrit);
@@ -529,7 +558,9 @@ public:
                     if (!initClientProcess(serverGroup, DCR_RoxyMaster, 0, NULL, NULL, timeout))
                         throw MakeStringException(ROXIE_DALI_ERROR, "Could not initialize dali client");
                     setPasswordsFromSDS();
-                    CSDSServerStatus serverstatus("Roxieserver");
+                    serverStatus = new CSDSServerStatus("RoxieServer");
+                    serverStatus->queryProperties()->setProp("@cluster", roxieName.str());
+                    serverStatus->commitProperties();
                     initCache();
                     ForEachItemIn(idx, watchers)
                     {
@@ -557,6 +588,8 @@ public:
             CriticalBlock b(daliConnectionCrit);
             if (isConnected)
             {
+                delete serverStatus;
+                serverStatus = NULL;
                 closeDllServer();
                 closeEnvironment();
                 clientShutdownWorkUnit();

roxie/ccd/ccddali.hpp

@@ -53,6 +53,8 @@ interface IRoxieDaliHelper : extends IInterface
     virtual bool connect(unsigned timeout) = 0;
     virtual void disconnect() = 0;
     virtual void waitConnected() = 0;
+    virtual void noteQueuesRunning(const char *queueNames) = 0;
+    virtual void noteWorkunitRunning(const char *wu, bool running) = 0;
 };
 
 

roxie/ccd/ccdserver.cpp

@@ -31044,6 +31044,7 @@ public:
                     {
                         Owned<IJobQueue> queue = createJobQueue(queueNames.str());
                         queue->connect();
+                        daliHelper->noteQueuesRunning(queueNames.str());
                         while (running)
                         {
                             Owned<IJobQueueItem> item = queue->dequeue(5000);
@@ -31276,12 +31277,14 @@ public:
     {
         Owned <IRoxieDaliHelper> daliHelper = connectToDali();
         Owned<IConstWorkUnit> wu = daliHelper->attachWorkunit(wuid.get(), NULL);
+        daliHelper->noteWorkunitRunning(wuid.get(), true);
         if (!wu)
             throw MakeStringException(ROXIE_DALI_ERROR, "Failed to open workunit %s", wuid.get());
         Owned<IQueryFactory> queryFactory = createServerQueryFactoryFromWu(wuid.get());
         Owned<StringContextLogger> logctx = new StringContextLogger(wuid.get());
         doMain(wu, queryFactory, *logctx);
         sendUnloadMessage(queryFactory->queryHash(), wuid.get(), *logctx);
+        daliHelper->noteWorkunitRunning(wuid.get(), false);
     }
 
     void doMain(IConstWorkUnit *wu, IQueryFactory *queryFactory, StringContextLogger &logctx)

thorlcr/thorutil/thmem.cpp

@@ -954,13 +954,20 @@ bool CThorSpillableRowArray::ensure(rowidx_t requiredRows)
             rows = (const void **)newRows.getClear();
             ReleaseThorRow(oldRows);
         }
-
-        // NB: can't release lock, or change maxRows, until know this succeeds
         if (stableSort_earlyAlloc == stableSort)
         {
-            OwnedConstThorRow newStableSortTmp = allocateStableTable(false);
+            // Temporarily release the lock, since MM may callback to spill this.
+            OwnedConstThorRow newStableSortTmp;
+            {
+                CThorSpillableRowArrayUnlock block(*this);
+                newStableSortTmp.setown(allocateStableTable(false));
+            }
+            // NB: If the above alloc fails, 'rows' has expanded, but maxRows has not
+            // this means, that on a subsequent ensure() call, it will only need to [attempt] to resize the stable ptr array.
+            // (see comment if (getRowsCapacity() < requiredRows) check above
             if (!newStableSortTmp)
                 return false;
+
             void **oldStableSortTmp = stableSortTmp;
             stableSortTmp = (void **)newStableSortTmp.getClear();
             ReleaseThorRow(oldStableSortTmp);

thorlcr/thorutil/thmem.hpp

@@ -355,6 +355,14 @@ public:
         inline CThorSpillableRowArrayLock(const CThorSpillableRowArray &_rows) : rows(_rows) { rows.lock(); }
         inline ~CThorSpillableRowArrayLock() { rows.unlock(); }
     };
+    class CThorSpillableRowArrayUnlock
+    {
+        CThorSpillableRowArrayUnlock(CThorSpillableRowArrayLock &); // avoid accidental use
+        const CThorSpillableRowArray & rows;
+    public:
+        inline CThorSpillableRowArrayUnlock(const CThorSpillableRowArray &_rows) : rows(_rows) { rows.unlock(); }
+        inline ~CThorSpillableRowArrayUnlock() { rows.lock(); }
+    };
 
     CThorSpillableRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep);
     ~CThorSpillableRowArray();