Merge branch 'closedown-4.2.x' into candidate-5.0.0

Conflicts:
	roxie/ccd/ccdlistener.cpp
	roxie/ccd/ccdquery.hpp
	roxie/ccd/ccdstate.cpp
	roxie/ccd/ccdstate.hpp
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

roxie/ccd/ccdlistener.cpp

@@ -1653,7 +1653,7 @@ readAnother:
                     else
                     {
                         StringBuffer querySetName;
-                        queryFactory.setown(globalPackageSetManager->getQuery(queryName, &querySetName, logctx));
+                        queryFactory.setown(globalPackageSetManager->getQuery(queryName, &querySetName, NULL, logctx));
                         if (isHTTP)
                             client->setHttpMode(queryName, isRequestArray, httpHelper.queryContentFormat());
                         if (queryFactory)

roxie/ccd/ccdquery.cpp

@@ -1157,7 +1157,7 @@ public:
             }
         }
     }
-    virtual void getQueryInfo(StringBuffer &reply, bool full, const IRoxieContextLogger &logctx) const
+    virtual void getQueryInfo(StringBuffer &reply, bool full, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
         Owned<IPropertyTree> xref = createPTree("Query", 0);
         xref->setProp("@id", id);
@@ -1175,6 +1175,17 @@ public:
                 f->getXrefInfo(*xref, logctx);
             }
         }
+        if (slaveQueries)
+        {
+            ForEachItemIn(idx, *slaveQueries)
+            {
+                if (slaveQueries->item(idx).suspended())
+                {
+                    xref->setPropBool("@suspended", true);
+                    xref->setPropBool("@slaveSuspended", true);
+                }
+            }
+        }
         toXML(xref, reply, 1);
     }
     virtual void resetQueryTimings()

roxie/ccd/ccdquery.hpp

@@ -124,7 +124,7 @@ interface IQueryFactory : extends IInterface
     virtual void getGraphNames(StringArray &ret) const = 0;
 
     virtual IQueryFactory *lookupLibrary(const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const = 0;
-    virtual void getQueryInfo(StringBuffer &result, bool full, const IRoxieContextLogger &logctx) const = 0;
+    virtual void getQueryInfo(StringBuffer &result, bool full, IArrayOf<IQueryFactory> *slaveQueries,const IRoxieContextLogger &logctx) const = 0;
     virtual bool isDynamic() const = 0;
 };
 

roxie/ccd/ccdserver.cpp

@@ -1270,7 +1270,8 @@ public:
             {
                 if (state==STATEstarted || state==STATEstarting)
                 {
-                    CTXLOG("STATE: activity %d reset without stop", activityId);
+                    if (traceStartStop || traceLevel > 2)
+                        CTXLOG("STATE: activity %d reset without stop", activityId);
                     stop(false);
                 }
                 if (ctx->queryTraceActivityTimes())

roxie/ccd/ccdstate.cpp

@@ -997,14 +997,16 @@ public:
         }
     }
 
-    virtual void getAllQueryInfo(StringBuffer &reply, bool full, const IRoxieContextLogger &logctx) const
+    virtual void getAllQueryInfo(StringBuffer &reply, bool full, const IRoxieQuerySetManagerSet *slaves, const IRoxieContextLogger &logctx) const
     {
         HashIterator elems(queries);
         for (elems.first(); elems.isValid(); elems.next())
         {
             IMapping &cur = elems.query();
             IQueryFactory *query = queries.mapToValue(&cur);
-            query->getQueryInfo(reply, full, logctx);
+            IArrayOf<IQueryFactory> slaveQueries;
+            slaves->getQueries(query->queryQueryName(), slaveQueries, logctx);
+            query->getQueryInfo(reply, full, &slaveQueries, logctx);
         }
         HashIterator aliasIterator(aliases);
         for (aliasIterator.first(); aliasIterator.isValid(); aliasIterator.next())
@@ -1112,6 +1114,17 @@ public:
                 managers[channel]->load(querySets, packages, hash, forceRetry); // MORE - this means the hash depends on the number of channels. Is that desirable?
     }
 
+    virtual void getQueries(const char *id, IArrayOf<IQueryFactory> &queries, const IRoxieContextLogger &logctx) const
+    {
+        for (unsigned channel = 0; channel < numChannels; channel++)
+            if (managers[channel])
+            {
+                IQueryFactory *query = managers[channel]->getQuery(id, NULL, logctx);
+                if (query)
+                    queries.append(*query);
+            }
+    }
+
 private:
     unsigned numChannels;
     CRoxieSlaveQuerySetManager **managers;
@@ -1221,6 +1234,12 @@ public:
         return serverManager.getLink();
     }
 
+    IRoxieQuerySetManagerSet* getRoxieSlaveManagers()
+    {
+        CriticalBlock b2(updateCrit);
+        return slaveManagers.getLink();
+    }
+
     void getInfo(StringBuffer &reply, const IRoxieContextLogger &logctx) const
     {
         reply.appendf(" <PackageSet id=\"%s\" querySet=\"%s\"/>\n", queryPackageId(), querySet.get());
@@ -1284,10 +1303,10 @@ public:
             }
         }
     }
-    void getAllQueryInfo(StringBuffer &reply, bool full,  const IRoxieContextLogger &logctx) const
+    void getAllQueryInfo(StringBuffer &reply, bool full, const IRoxieContextLogger &logctx) const
     {
         CriticalBlock b2(updateCrit);
-        serverManager->getAllQueryInfo(reply, full, logctx);
+        serverManager->getAllQueryInfo(reply, full, slaveManagers, logctx);
     }
 protected:
     void reloadQueryManagers(CRoxieSlaveQuerySetManagerSet *newSlaveManagers, IRoxieQuerySetManager *newServerManager, hash64_t newHash)
@@ -1487,7 +1506,7 @@ public:
         throw MakeStringException(ROXIE_LIBRARY_ERROR, "No library available for %s", libraryName);
     }
 
-    IQueryFactory *getQuery(const char *id, StringBuffer *querySet, const IRoxieContextLogger &logctx) const
+    IQueryFactory *getQuery(const char *id, StringBuffer *querySet, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
         ForEachItemIn(idx, allQueryPackages)
         {
@@ -1496,7 +1515,14 @@ public:
             {
                 IQueryFactory *query = sm->getQuery(id, querySet, logctx);
                 if (query)
+                {
+                    if (slaveQueries)
+                    {
+                        Owned<IRoxieQuerySetManagerSet> slaveManagers = allQueryPackages.item(idx).getRoxieSlaveManagers();
+                        slaveManagers->getQueries(id, *slaveQueries, logctx);
+                    }
                     return query;
+                }
             }
         }
         return NULL;
@@ -1524,8 +1550,9 @@ public:
     {
         ForEachItemIn(idx, allQueryPackages)
         {
-            Owned<IRoxieQuerySetManager> sm = allQueryPackages.item(idx).getRoxieServerManager();
-            sm->getAllQueryInfo(reply, full, logctx);
+            Owned<IRoxieQuerySetManager> serverManager = allQueryPackages.item(idx).getRoxieServerManager();
+            Owned<IRoxieQuerySetManagerSet> slaveManagers = allQueryPackages.item(idx).getRoxieSlaveManagers();
+            serverManager->getAllQueryInfo(reply, full, slaveManagers, logctx);
         }
     }
 
@@ -1718,10 +1745,10 @@ public:
         return allQueryPackages->lookupLibrary(libraryName, expectedInterfaceHash, logctx);
     }
 
-    virtual IQueryFactory *getQuery(const char *id, StringBuffer *querySet, const IRoxieContextLogger &logctx) const
+    virtual IQueryFactory *getQuery(const char *id, StringBuffer *querySet, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
         ReadLockBlock b(packageCrit);
-        return allQueryPackages->getQuery(id, querySet, logctx);
+        return allQueryPackages->getQuery(id, querySet, slaveQueries, logctx);
     }
 
     virtual int getActivePackageCount() const
@@ -1830,9 +1857,10 @@ private:
                 const char *id = ids->query().queryProp("@id");
                 if (id)
                 {
-                    Owned<IQueryFactory> query = getQuery(id, NULL, logctx);
+                    IArrayOf<IQueryFactory> slaveQueries;
+                    Owned<IQueryFactory> query = getQuery(id, NULL, &slaveQueries, logctx);
                     if (query)
-                        query->getQueryInfo(reply, full, logctx);
+                        query->getQueryInfo(reply, full, &slaveQueries, logctx);
                     else
                         reply.appendf(" <Query id=\"%s\" error=\"Query not found\"/>\n", id);
                 }
@@ -2051,7 +2079,7 @@ private:
                 if (!id)
                     throw MakeStringException(ROXIE_MISSING_PARAMS, "No query name specified");
 
-                Owned<IQueryFactory> q = getQuery(id, NULL, logctx);
+                Owned<IQueryFactory> q = getQuery(id, NULL, NULL, logctx);
                 if (q)
                 {
                     Owned<IPropertyTree> tempTree = q->cloneQueryXGMML();
@@ -2066,7 +2094,7 @@ private:
                 const char *id = control->queryProp("Query/@id");
                 if (!id)
                     badFormat();
-                Owned<IQueryFactory> f = getQuery(id, NULL, logctx);
+                Owned<IQueryFactory> f = getQuery(id, NULL, NULL, logctx);
                 if (f)
                 {
                     unsigned warnLimit = f->getWarnTimeLimit();
@@ -2270,7 +2298,7 @@ private:
                 const char *id = control->queryProp("Query/@id");
                 if (id)
                 {
-                    Owned<IQueryFactory> f = getQuery(id, NULL, logctx);
+                    Owned<IQueryFactory> f = getQuery(id, NULL, NULL, logctx); // MORE - slave too?
                     if (f)
                     {
                         Owned<const IPropertyTree> stats = f->getQueryStats(from, to);
@@ -2302,7 +2330,7 @@ private:
                 {
                     if (stricmp(action, "listGraphNames") == 0)
                     {
-                        Owned<IQueryFactory> query = getQuery(id, NULL, logctx);
+                        Owned<IQueryFactory> query = getQuery(id, NULL, NULL, logctx); // MORE - slave too?
                         if (query)
                         {
                             reply.appendf("<Query id='%s'>\n", id);
@@ -2440,7 +2468,7 @@ private:
                 if (!id.length())
                     badFormat();
                 {
-                    Owned<IQueryFactory> f = getQuery(id, NULL, logctx);
+                    Owned<IQueryFactory> f = getQuery(id, NULL, NULL, logctx); // MORE - slave too?
                     if (f)
                         id.clear().append(f->queryQueryName());  // use the spelling of the query stored with the query factory
                 }

roxie/ccd/ccdstate.hpp

@@ -92,6 +92,12 @@ interface IFileIOArray : extends IInterface
     virtual StringBuffer &getId(StringBuffer &) const = 0;
 };
 
+interface IRoxieQuerySetManagerSet : extends IInterface
+{
+    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash, bool forceRetry) = 0;
+    virtual void getQueries(const char *id, IArrayOf<IQueryFactory> &queries, const IRoxieContextLogger &logctx) const = 0;
+};
+
 interface IRoxieQuerySetManager : extends IInterface
 {
     virtual bool isActive() const = 0;
@@ -101,7 +107,7 @@ interface IRoxieQuerySetManager : extends IInterface
     virtual void resetQueryTimings(const char *queryName, const IRoxieContextLogger &logctx) = 0;
     virtual void resetAllQueryTimings() = 0;
     virtual void getActivityMetrics(StringBuffer &reply) const = 0;
-    virtual void getAllQueryInfo(StringBuffer &reply, bool full, const IRoxieContextLogger &logctx) const = 0;
+    virtual void getAllQueryInfo(StringBuffer &reply, bool full, const IRoxieQuerySetManagerSet *slaves, const IRoxieContextLogger &logctx) const = 0;
 };
 
 interface IRoxieDebugSessionManager : extends IInterface
@@ -111,17 +117,12 @@ interface IRoxieDebugSessionManager : extends IInterface
     virtual IDebuggerContext *lookupDebuggerContext(const char *id) = 0;
 };
 
-interface IRoxieQuerySetManagerSet : extends IInterface
-{
-    virtual void load(const IPropertyTree *querySets, const IRoxiePackageMap &packages, hash64_t &hash, bool forceRetry) = 0;
-};
-
 interface IRoxieQueryPackageManagerSet : extends IInterface
 {
     virtual void requestReload() = 0;
     virtual void load() = 0;
     virtual void doControlMessage(IPropertyTree *xml, StringBuffer &reply, const IRoxieContextLogger &ctx) = 0;
-    virtual IQueryFactory *getQuery(const char *id, StringBuffer *querySet, const IRoxieContextLogger &logctx) const = 0;
+    virtual IQueryFactory *getQuery(const char *id, StringBuffer *querySet, IArrayOf<IQueryFactory> *slaves, const IRoxieContextLogger &logctx) const = 0;
     virtual IQueryFactory *lookupLibrary(const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const = 0;
     virtual int getActivePackageCount() const = 0;
 };

system/jhtree/jhtree.cpp

@@ -1021,7 +1021,7 @@ public:
 class CNodeCache : public CInterface
 {
 private:
-    mutable SpinLock lock;
+    mutable CriticalSection lock;
     CNodeMRUCache nodeCache;
     CNodeMRUCache leafCache;
     CNodeMRUCache blobCache;
@@ -1059,28 +1059,28 @@ public:
 
     inline size32_t setNodeCacheMem(size32_t newSize)
     {
-        SpinBlock block(lock);
+        CriticalBlock block(lock);
         unsigned oldV = nodeCache.setMemLimit(newSize);
         cacheNodes = (newSize != 0);
         return oldV;
     }
     inline size32_t setLeafCacheMem(size32_t newSize)
     {
-        SpinBlock block(lock);
+        CriticalBlock block(lock);
         unsigned oldV = leafCache.setMemLimit(newSize);
         cacheLeaves = (newSize != 0); 
         return oldV;
     }
     inline size32_t setBlobCacheMem(size32_t newSize)
     {
-        SpinBlock block(lock);
+        CriticalBlock block(lock);
         unsigned oldV = blobCache.setMemLimit(newSize);
         cacheBlobs = (newSize != 0); 
         return oldV;
     }
     void clear()
     {
-        SpinBlock block(lock);
+        CriticalBlock block(lock);
         nodeCache.kill();
         leafCache.kill();
         blobCache.kill();
@@ -2102,7 +2102,7 @@ CJHTreeNode *CNodeCache::getNode(INodeLoader *keyIndex, int iD, offset_t pos, IC
         return NULL;
     { 
         // It's a shame that we don't know the type before we read it. But probably not that big a deal
-        SpinBlock block(lock);
+        CriticalBlock block(lock);
         CKeyIdAndPos key(iD, pos);
         if (preloadNodes)
         {
@@ -2150,7 +2150,7 @@ CJHTreeNode *CNodeCache::getNode(INodeLoader *keyIndex, int iD, offset_t pos, IC
         }
         CJHTreeNode *node;
         {
-            SpinUnblock block(lock);
+            CriticalUnblock block(lock);
             node = keyIndex->loadNode(pos);  // NOTE - don't want cache locked while we load!
         }
         atomic_inc(&cacheAdds);
@@ -2216,7 +2216,7 @@ void CNodeCache::preload(CJHTreeNode *node, int iD, offset_t pos, IContextLogger
 {
     assertex(pos);
     assertex(preloadNodes);
-    SpinBlock block(lock);
+    CriticalBlock block(lock);
     CKeyIdAndPos key(iD, pos);
     CJHTreeNode *cacheNode = preloadCache.query(key);
     if (!cacheNode)
@@ -2230,7 +2230,7 @@ void CNodeCache::preload(CJHTreeNode *node, int iD, offset_t pos, IContextLogger
 
 bool CNodeCache::isPreloaded(int iD, offset_t pos)
 {
-    SpinBlock block(lock);
+    CriticalBlock block(lock);
     CKeyIdAndPos key(iD, pos);
     return NULL != preloadCache.query(key);
 }