HPCC-11317 Move list queries using file checking into CWorkUnitFactory

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/workunit.cpp

@@ -2450,12 +2450,13 @@ public:
     }
 
     IConstQuerySetQueryIterator* getQuerySetQueriesSorted( WUQuerySortField *sortorder, // list of fields to sort by (terminated by WUSFterm)
-                                                WUQuerySortField *filters,   // NULL or list of fields to folteron (terminated by WUSFterm)
+                                                WUQuerySortField *filters,   // NULL or list of fields to filter on (terminated by WUSFterm)
                                                 const void *filterbuf,  // (appended) string values for filters
                                                 unsigned startoffset,
                                                 unsigned maxnum,
                                                 __int64 *cachehint,
-                                                unsigned *total)
+                                                unsigned *total,
+                                                const MapStringTo<bool> *_subset)
     {
         struct PostFilters
         {
@@ -2475,6 +2476,7 @@ public:
             StringAttr sortOrder;
             PostFilters postFilters;
             StringArray unknownAttributes;
+            const MapStringTo<bool> *subset;
 
             void populateQueryTree(IPropertyTree* queryRegistry, const char* querySetId, IPropertyTree* querySetTree, const char *xPath, IPropertyTree* queryTree)
             {
@@ -2487,6 +2489,12 @@ public:
                     const char* queryId = query.queryProp("@id");
                     if (queryId && *queryId)
                     {
+                        if (subset)
+                        {
+                            VStringBuffer match("%s/%s", querySetId, queryId);
+                            if (!subset->getValue(match))
+                                continue;
+                        }
                         VStringBuffer xPath("Alias[@id='%s']", queryId);
                         IPropertyTree *alias = queryRegistry->queryPropTree(xPath.str());
                         if (alias)
@@ -2538,8 +2546,8 @@ public:
         public:
             IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-            CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes)
-                : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder)
+            CQuerySetQueriesPager(const char* _querySet, const char* _xPath, const char *_sortOrder, PostFilters& _postFilters, StringArray& _unknownAttributes, const MapStringTo<bool> *_subset)
+                : querySet(_querySet), xPath(_xPath), sortOrder(_sortOrder), subset(_subset)
             {
                 postFilters.activatedFilter = _postFilters.activatedFilter;
                 postFilters.suspendedByUserFilter = _postFilters.suspendedByUserFilter;
@@ -2611,7 +2619,7 @@ public:
             }
         }
         IArrayOf<IPropertyTree> results;
-        Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes);
+        Owned<IElementsPager> elementsPager = new CQuerySetQueriesPager(querySet.get(), xPath.str(), so.length()?so.str():NULL, postFilters, unknownAttributes, _subset);
         Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,NULL,"",cachehint,results,total);
         return new CConstQuerySetQueryIterator(results);
     }
@@ -2887,9 +2895,10 @@ public:
                                                 unsigned startoffset,
                                                 unsigned maxnum,
                                                 __int64 *cachehint,
-                                                unsigned *total)
+                                                unsigned *total,
+                                                const MapStringTo<bool> *subset)
     {
-        return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total);
+        return factory->getQuerySetQueriesSorted(sortorder,filters,filterbuf,startoffset,maxnum,cachehint,total, subset);
     }
 
     virtual unsigned numWorkUnits()

common/workunit/workunit.hpp

@@ -1186,7 +1186,7 @@ interface IWorkUnitFactory : extends IInterface
     virtual unsigned numWorkUnitsFiltered(WUSortField * filters, const void * filterbuf) = 0;
     virtual void descheduleAllWorkUnits() = 0;
     virtual bool deleteWorkUnitEx(const char * wuid) = 0;
-    virtual IConstQuerySetQueryIterator * getQuerySetQueriesSorted(WUQuerySortField *sortorder, WUQuerySortField *filters, const void *filterbuf, unsigned startoffset, unsigned maxnum, __int64 *cachehint, unsigned *total) = 0;
+    virtual IConstQuerySetQueryIterator * getQuerySetQueriesSorted(WUQuerySortField *sortorder, WUQuerySortField *filters, const void *filterbuf, unsigned startoffset, unsigned maxnum, __int64 *cachehint, unsigned *total, const MapStringTo<bool> *subset) = 0;
 };
 
 

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -1258,14 +1258,11 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
     if (!req.getCacheHint_isNull())
         cacheHint = req.getCacheHint();
 
-    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
-    Owned<IConstQuerySetQueryIterator> it = factory->getQuerySetQueriesSorted(sortOrder, filters, filterBuf.bufferBase(), pageStartFrom, pageSize, &cacheHint, &numberOfQueries);
-    resp.setCacheHint(cacheHint);
-
-    MapStringTo<bool> queriesUsingFileMap;
+    Owned<MapStringTo<bool> > queriesUsingFileMap;
     const char *lfn = req.getFileName();
     if (lfn && *lfn)
     {
+        queriesUsingFileMap.setown(new MapStringTo<bool>());
         Owned<IPropertyTreeIterator> queriesUsingFile = filesInUse.findQueriesUsingFile(clusterReq, lfn);
         ForEach (*queriesUsingFile)
         {
@@ -1275,10 +1272,15 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
             if (queryTarget && *queryTarget && queryId && *queryId)
             {
                 VStringBuffer targetQuery("%s/%s", queryTarget, queryId);
-                queriesUsingFileMap.setValue(targetQuery, true);
+                queriesUsingFileMap->setValue(targetQuery, true);
             }
         }
     }
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+    Owned<IConstQuerySetQueryIterator> it = factory->getQuerySetQueriesSorted(sortOrder, filters, filterBuf.bufferBase(), pageStartFrom, pageSize, &cacheHint, &numberOfQueries, queriesUsingFileMap);
+    resp.setCacheHint(cacheHint);
+
     IArrayOf<IEspQuerySetQuery> queries;
     double version = context.getClientVersion();
     ForEach(*it)
@@ -1286,14 +1288,6 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
         IPropertyTree &query=it->query();
         const char *queryId = query.queryProp("@id");
         const char *queryTarget = query.queryProp("@querySetId");
-        if (lfn && *lfn)
-        {
-            if (!queryTarget || !*queryTarget || !queryId || !*queryId)
-                continue;
-            VStringBuffer targetQuery("%s/%s", queryTarget, queryId);
-            if (!queriesUsingFileMap.getValue(targetQuery))
-                continue;
-        }
 
         Owned<IEspQuerySetQuery> q = createQuerySetQuery();
         q->setId(queryId);