Merge pull request #5870 from afishbeck/listqueriesbyfile

HPCC-11317 add support for filtering WUListQueries by file used 

Reviewed-By: Kevin Wang <kevin.wang@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -2450,12 +2450,13 @@ public:
     }
 
     IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
-                                                WUQuerySortField *filters,   // NULL or list of fields to folteron (terminated by WUSFterm)
+                                                WUQuerySortField *filters,   // NULL or list of fields to filter on (terminated by WUSFterm)
                                                 const void *filterbuf,  // (appended) string values for filters
                                                 unsigned startoffset,
                                                 unsigned maxnum,
                                                 __int64 *cachehint,
-                                                unsigned *total)
+                                                unsigned *total,
+                                                const MapStringTo<bool> *_subset)
     {
         struct PostFilters
         {
@@ -2475,6 +2476,7 @@ public:
             StringAttr sortOrder;
             PostFilters postFilters;
             StringArray unknownAttributes;
+            const MapStringTo<bool> *subset;
 
             void populateQueryTree(IPropertyTree* queryRegistry, const char* querySetId, IPropertyTree* querySetTree, const char *xPath, IPropertyTree* queryTree)
             {
@@ -2487,6 +2489,12 @@ public:
                     const char* queryId = query.queryProp("@id");
                     if (queryId && *queryId)
                     {
+                        if (subset)
+                        {
+                            VStringBuffer match("%s/%s", querySetId, queryId);
+                            if (!subset->getValue(match))
+                                continue;
+                        }
                         VStringBuffer xPath("Alias[@id='%s']", queryId);
                         IPropertyTree *alias = queryRegistry->queryPropTree(xPath.str());
                         if (alias)
@@ -2538,8 +2546,8 @@ public:
         public:
             IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-            CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes)
-                : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder)
+            CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
+                : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
             {
                 postFilters.activatedFilter = _postFilters.activatedFilter;
                 postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
@@ -2611,7 +2619,7 @@ public:
             }
         }
         IArrayOf<IPropertyTree> results;
-        Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes);
+        Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
         Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total);
         return new CConstQuerySetQueryIterator(results);
     }
@@ -2887,9 +2895,10 @@ public:
                                                 unsigned startoffset,
                                                 unsigned maxnum,
                                                 __int64 *cachehint,
-                                                unsigned *total)
+                                                unsigned *total,
+                                                const MapStringTo<bool> *subset)
     {
-        return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total);
+        return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total, subset);
     }
 
     virtual unsigned numWorkUnits()

common/workunit/workunit.hpp

@@ -1186,7 +1186,7 @@ interface IWorkUnitFactory : extends IInterface
     virtual unsigned numWorkUnitsFiltered(WUSortField * filters, const void * filterbuf) = 0;
     virtual void descheduleAllWorkUnits() = 0;
     virtual bool deleteWorkUnitEx(const char * wuid) = 0;
-    virtual IConstQuerySetQueryIterator * getQuerySetQueriesSorted(WUQuerySortField *sortorder, WUQuerySortField *filters, const void *filterbuf, unsigned startoffset, unsigned maxnum, __int64 *cachehint, unsigned *total) = 0;
+    virtual IConstQuerySetQueryIterator * getQuerySetQueriesSorted(WUQuerySortField *sortorder, WUQuerySortField *filters, const void *filterbuf, unsigned startoffset, unsigned maxnum, __int64 *cachehint, unsigned *total, const MapStringTo<bool> *subset) = 0;
 };
 
 

esp/scm/ws_workunits.ecm

@@ -1257,6 +1257,7 @@ ESPrequest [nil_remove] WUListQueriesRequest
     string Sortby;
     bool Descending(false);
     int64 CacheHint;
+    string FileName;
 };
 
 ESPresponse [exceptions_inline] WUListQueriesResponse

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -335,6 +335,7 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
             continue;
 
         queryTree = targetTree->addPropTree("Query", createPTree("Query"));
+        queryTree->setProp("@target", target); //for reference when searching across targets
         queryTree->setProp("@id", queryid);
         if (pkgid && *pkgid)
             queryTree->setProp("@pkgid", pkgid);
@@ -384,12 +385,25 @@ void QueryFilesInUse::loadTargets(IPropertyTree *t, unsigned flags)
     }
 }
 
+IPropertyTreeIterator *QueryFilesInUse::findAllQueriesUsingFile(const char *lfn)
+{
+    CriticalBlock b(crit);
+
+    if (!lfn || !*lfn)
+        return NULL;
+
+    VStringBuffer xpath("*/Query[File/@lfn='%s']", lfn);
+    return tree->getElements(xpath);
+}
+
 IPropertyTreeIterator *QueryFilesInUse::findQueriesUsingFile(const char *target, const char *lfn)
 {
     CriticalBlock b(crit);
 
-    if (!target || !*target || !lfn || !*lfn)
+    if (!lfn || !*lfn)
         return NULL;
+    if (!target || !*target)
+        return findAllQueriesUsingFile(lfn);
     IPropertyTree *targetTree = tree->getPropTree(target);
     if (!targetTree)
         return NULL;
@@ -1244,8 +1258,27 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
     if (!req.getCacheHint_isNull())
         cacheHint = req.getCacheHint();
 
+    Owned<MapStringTo<bool> > queriesUsingFileMap;
+    const char *lfn = req.getFileName();
+    if (lfn && *lfn)
+    {
+        queriesUsingFileMap.setown(new MapStringTo<bool>());
+        Owned<IPropertyTreeIterator> queriesUsingFile = filesInUse.findQueriesUsingFile(clusterReq, lfn);
+        ForEach (*queriesUsingFile)
+        {
+            IPropertyTree &queryUsingFile = queriesUsingFile->query();
+            const char *queryTarget = queryUsingFile.queryProp("@target");
+            const char *queryId = queryUsingFile.queryProp("@id");
+            if (queryTarget && *queryTarget && queryId && *queryId)
+            {
+                VStringBuffer targetQuery("%s/%s", queryTarget, queryId);
+                queriesUsingFileMap->setValue(targetQuery, true);
+            }
+        }
+    }
+
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
-    Owned<IConstQuerySetQueryIterator> it = factory->getQuerySetQueriesSorted(sortOrder, filters, filterBuf.bufferBase(), pageStartFrom, pageSize, &cacheHint, &numberOfQueries);
+    Owned<IConstQuerySetQueryIterator> it = factory->getQuerySetQueriesSorted(sortOrder, filters, filterBuf.bufferBase(), pageStartFrom, pageSize, &cacheHint, &numberOfQueries, queriesUsingFileMap);
     resp.setCacheHint(cacheHint);
 
     IArrayOf<IEspQuerySetQuery> queries;
@@ -1253,10 +1286,13 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
     ForEach(*it)
     {
         IPropertyTree &query=it->query();
+        const char *queryId = query.queryProp("@id");
+        const char *queryTarget = query.queryProp("@querySetId");
+
         Owned<IEspQuerySetQuery> q = createQuerySetQuery();
-        q->setId(query.queryProp("@id"));
+        q->setId(queryId);
+        q->setQuerySetId(queryTarget);
         q->setName(query.queryProp("@name"));
-        q->setQuerySetId(query.queryProp("@querySetId"));
         q->setDll(query.queryProp("@dll"));
         q->setWuid(query.queryProp("@wuid"));
         q->setActivated(query.getPropBool("@activated", false));
@@ -1304,7 +1340,7 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
         queries.append(*q.getClear());
     }
     resp.setQuerysetQueries(queries);
-    resp.setNumberOfQueries(numberOfQueries);
+    resp.setNumberOfQueries(queries.length());
 
     return true;
 }

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -134,6 +134,7 @@ public:
         CriticalBlock b(crit);
     }
     IPropertyTreeIterator *findQueriesUsingFile(const char *target, const char *lfn);
+    IPropertyTreeIterator *findAllQueriesUsingFile(const char *lfn);
     StringBuffer &toStr(StringBuffer &s)
     {
         CriticalBlock b(crit);