Merge branch 'closedown-4.2.x' into closedown-5.0.x

Conflicts:
	common/workunit/workunit.cpp
	common/workunit/workunit.hpp
	ecl/eclagent/eclagent.ipp
	esp/files/templates/QuerySetQueryWidget.html
	esp/services/ws_workunits/ws_workunitsQuerySets.cpp
	esp/services/ws_workunits/ws_workunitsService.cpp
	roxie/ccd/ccdcontext.cpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -1948,6 +1948,7 @@ mapEnums querySortFields[] =
    { WUQSFQuerySet, "../@id" },
    { WUQSFActivited, "@activated" },
    { WUQSFSuspendedByUser, "@suspended" },
+   { WUQSFLibrary, "Library"},
    { WUQSFterm, NULL }
 };
 
@@ -10146,7 +10147,30 @@ extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot()
             return NULL;
 }
 
+extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries)
+{
+    if (!queryTree || !libraries)
+        return;
+    if (queryTree->hasProp("@libCount")) //already added
+        return;
+    unsigned libCount=0;
+    ForEach(*libraries)
+    {
+        IConstWULibrary &library = libraries->query();
+        SCMStringBuffer libname;
+        if (!library.getName(libname).length())
+            continue;
+        queryTree->addProp("Library", libname.str());
+        libCount++;
+    }
+    queryTree->setPropInt("@libCount", libCount);
+}
 
+extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw)
+{
+    Owned<IConstWULibraryIterator> libraries = &cw->getLibraries();
+    checkAddLibrariesToQueryEntry(queryTree, libraries);
+}
 
 extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly)
 {
@@ -10265,6 +10289,8 @@ void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const
     }
 
     IPropertyTree *newEntry = addNamedQuery(queryRegistry, cleanQueryName, wuid.str(), dllName.str(), isLibrary(workunit), userid, snapshot.str());
+    Owned<IConstWULibraryIterator> libraries = &workunit->getLibraries();
+    checkAddLibrariesToQueryEntry(newEntry, libraries);
     newQueryId.append(newEntry->queryProp("@id"));
     workunit->setIsQueryService(true); //will check querysets before delete
     workunit->commit();

common/workunit/workunit.hpp

@@ -1158,6 +1158,7 @@ enum WUQuerySortField
     WUQSFQuerySet = 13,
     WUQSFActivited = 14,
     WUQSFSuspendedByUser = 15,
+    WUQSFLibrary = 16,
     WUQSFterm = 0,
     WUQSFreverse = 256,
     WUQSFnocase = 512,
@@ -1328,6 +1329,9 @@ extern WORKUNIT_API IPropertyTree * resolveQueryAlias(const char *queryset, cons
 extern WORKUNIT_API IPropertyTree * getQueryRegistry(const char * wsEclId, bool readonly);
 extern WORKUNIT_API IPropertyTree * getQueryRegistryRoot();
 
+extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWULibraryIterator *libraries);
+extern WORKUNIT_API void checkAddLibrariesToQueryEntry(IPropertyTree *queryTree, IConstWorkUnit *cw);
+
 extern WORKUNIT_API void setQueryCommentForNamedQuery(IPropertyTree * queryRegistry, const char *id, const char *queryComment);
 
 extern WORKUNIT_API void setQuerySuspendedState(IPropertyTree * queryRegistry, const char * name, bool suspend, const char *userid);

esp/scm/ws_workunits.ecm

@@ -1252,6 +1252,7 @@ ESPrequest [nil_remove] WUListQueriesRequest
 {
     string  QuerySetName;
     string  ClusterName;
+    string  LibraryName;
     int64 MemoryLimitLow;
     int64 MemoryLimitHigh;
     nonNegativeInteger TimeLimitLow;

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -723,7 +723,6 @@ bool CWsWorkunitsEx::isQuerySuspended(const char* query, IConstWUClusterInfo *cl
     }
 }
 
-
 bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWorkunitRequest & req, IEspWUPublishWorkunitResponse & resp)
 {
     StringBuffer wuid = req.getWuid();
@@ -1273,6 +1272,7 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
     addWUQSQueryFilter(filters, filterCount, filterBuf, req.getQueryID(), (WUQuerySortField) (WUQSFId | WUQSFwild));
     addWUQSQueryFilter(filters, filterCount, filterBuf, req.getQueryName(), (WUQuerySortField) (WUQSFname | WUQSFwild));
     addWUQSQueryFilter(filters, filterCount, filterBuf, req.getWUID(), WUQSFwuid);
+    addWUQSQueryFilter(filters, filterCount, filterBuf, req.getLibraryName(), (WUQuerySortField) (WUQSFLibrary | WUQSFnocase));
     if (!req.getMemoryLimitLow_isNull())
         addWUQSQueryFilterInt64(filters, filterCount, filterBuf, req.getMemoryLimitLow(), (WUQuerySortField) (WUQSFmemoryLimit | WUQSFnumeric));
     if (!req.getMemoryLimitHigh_isNull())
@@ -1972,6 +1972,12 @@ public:
                 if (!destQuery->hasProp(atname))
                     destQuery->setProp(atname, aiter->queryValue());
             }
+            Owned<IPropertyTreeIterator> children = query->getElements("*");
+            ForEach(*children)
+            {
+                IPropertyTree &child = children->query();
+                destQuery->addPropTree(child.queryName(), createPTreeFromIPT(&child));
+            }
             if (cloneFilesEnabled && wufiles)
                 wufiles->addFilesFromQuery(workunit, pm, newQueryId);
         }
@@ -2046,7 +2052,6 @@ public:
     StringArray copiedQueryIds;
 };
 
-
 bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRequest &req, IEspWUCopyQuerySetResponse &resp)
 {
     const char *source = req.getSource();

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -48,6 +48,8 @@
 
 #define ESP_WORKUNIT_DIR "workunits/"
 
+#define SDS_LOCK_TIMEOUT (5*60*1000) // 5 mins
+
 class ExecuteExistingQueryInfo
 {
 public:
@@ -362,6 +364,41 @@ bool doAction(IEspContext& context, StringArray& wuids, int action, IProperties*
     return bAllSuccess;
 }
 
+static void checkUpdateQuerysetLibraries()
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!globalLock)
+        return;
+
+    IPropertyTree *root = globalLock->queryRoot();
+    if (!root)
+        return;
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IPropertyTreeIterator> querySets = root->getElements("QuerySet");
+    ForEach(*querySets)
+    {
+        IPropertyTree &querySet = querySets->query();
+        if (querySet.hasProp("@updatedLibraries")) //only need to do this once, then publish and copy will keep up to date
+            continue;
+        Owned<IPropertyTreeIterator> queries = querySet.getElements("Query");
+        ForEach(*queries)
+        {
+            IPropertyTree &query = queries->query();
+            if (query.hasProp("@libCount"))
+                continue;
+            const char *wuid = query.queryProp("@wuid");
+            if (!wuid || !*wuid)
+                continue;
+            Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid, false);
+            if (!cw)
+                continue;
+            checkAddLibrariesToQueryEntry(&query, cw);
+        }
+        querySet.setPropBool("@updatedLibraries", true);
+    }
+}
+
 MapStringTo<int> wuActionTable;
 
 void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *service)
@@ -375,6 +412,7 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
 
     DBGLOG("Initializing %s service [process = %s]", service, process);
 
+    checkUpdateQuerysetLibraries();
     refreshValidClusters();
 
     daliServers.set(cfg->queryProp("Software/EspProcess/@daliServers"));