HPCC-8735 - Avoid seperate call to fetch total # workunits

ws_workunits, called numWorkUnitsFiltered and getWorkUnitsSorted
when requesting workunits. The latter was able to calculate the
total with minor modification and avoid the need for a seperate
pass with numWorkUnitsFiltered

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

common/workunit/workunit.cpp

@@ -2183,7 +2183,8 @@ public:
                                                 const char *queryowner, 
                                                 __int64 *cachehint,
                                                 ISecManager *secmgr, 
-                                                ISecUser *secuser)
+                                                ISecUser *secuser,
+                                                unsigned *total)
     {
         class cScopeChecker: implements ISortedElementsTreeFilter
         {
@@ -2249,7 +2250,7 @@ public:
         }
         IArrayOf<IPropertyTree> results;
         Owned<IRemoteConnection> conn=getElementsPaged( "WorkUnits", query.str(), so.length()?so.str():NULL,startoffset,maxnum,
-            secmgr?&sc:NULL,queryowner,cachehint,namefilterlo.get(),namefilterhi.get(),results);
+            secmgr?&sc:NULL,queryowner,cachehint,namefilterlo.get(),namefilterhi.get(),results,total);
         return new CConstWUArrayIterator(conn, results, secmgr, secuser);
     }
 
@@ -2260,9 +2261,10 @@ public:
                                                 unsigned startoffset,
                                                 unsigned maxnum,
                                                 const char *queryowner, 
-                                                __int64 *cachehint)
+                                                __int64 *cachehint,
+                                                unsigned *total)
     {
-        return getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, NULL, NULL);
+        return getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, NULL, NULL, total);
     }
 
     virtual unsigned numWorkUnits()
@@ -2279,12 +2281,9 @@ public:
                                         ISecManager *secmgr, 
                                         ISecUser *secuser)
     {
-        Owned<IConstWorkUnitIterator> iter =  getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL,secmgr,secuser);
-        // this is rather slow but necessarily so (for security check)
-        unsigned ret = 0;
-        ForEach(*iter)
-            ret++;
-        return ret;
+        unsigned total;
+        Owned<IConstWorkUnitIterator> iter =  getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL,secmgr,secuser,&total);
+        return total;
     }
 
     virtual unsigned numWorkUnitsFiltered(WUSortField *filters,const void *filterbuf)
@@ -2494,9 +2493,10 @@ public:
                                                         unsigned startoffset,
                                                         unsigned maxnum,
                                                         const char *queryowner, 
-                                                        __int64 *cachehint)
+                                                        __int64 *cachehint,
+                                                        unsigned *total)
     {
-        return factory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, secMgr.get(), secUser.get());
+        return factory->getWorkUnitsSorted(sortorder,filters,filterbuf,startoffset,maxnum,queryowner,cachehint, secMgr.get(), secUser.get(), total);
     }
 
     virtual unsigned numWorkUnits()

common/workunit/workunit.hpp

@@ -1100,7 +1100,7 @@ interface IWorkUnitFactory : extends IInterface
     virtual IConstWorkUnitIterator * getWorkUnitsByECL(const char * ecl) = 0;
     virtual IConstWorkUnitIterator * getWorkUnitsByCluster(const char * cluster) = 0;
     virtual IConstWorkUnitIterator * getWorkUnitsByXPath(const char * xpath) = 0;
-    virtual IConstWorkUnitIterator * getWorkUnitsSorted(WUSortField * sortorder, WUSortField * filters, const void * filterbuf, unsigned startoffset, unsigned maxnum, const char * queryowner, __int64 * cachehint) = 0;
+    virtual IConstWorkUnitIterator * getWorkUnitsSorted(WUSortField * sortorder, WUSortField * filters, const void * filterbuf, unsigned startoffset, unsigned maxnum, const char * queryowner, __int64 * cachehint, unsigned *total) = 0;
     virtual unsigned numWorkUnits() = 0;
     virtual unsigned numWorkUnitsFiltered(WUSortField * filters, const void * filterbuf) = 0;
     virtual void descheduleAllWorkUnits() = 0;

dali/base/dautils.cpp

@@ -1829,7 +1829,8 @@ IRemoteConnection *getElementsPaged( const char *basexpath,
                                      __int64 *hint,
                                      const char *namefilterlo,
                                      const char *namefilterhi,
-                                     IArrayOf<IPropertyTree> &results)
+                                     IArrayOf<IPropertyTree> &results,
+                                     unsigned *total)
 {
     if (pagesize==0)
         return NULL;
@@ -1850,7 +1851,10 @@ IRemoteConnection *getElementsPaged( const char *basexpath,
     if (!elem->conn)
         return NULL;
     unsigned n;
+    if (total)
+        *total = elem->totalres.ordinality();
     if (postfilter) {
+        unsigned numFiltered = 0;
         n = 0;
         ForEachItemIn(i,elem->totalres) {
             IPropertyTree &item = elem->totalres.item(i);
@@ -1859,11 +1863,20 @@ IRemoteConnection *getElementsPaged( const char *basexpath,
                     item.Link();
                     results.append(item);
                     if (results.ordinality()>=pagesize)
-                        break;
+                    {
+                        // if total needed, need to iterate through all items
+                        if (NULL == total)
+                            break;
+                        startoffset = (unsigned)-1; // no more results needed
+                    }
                 }
                 n++;
             }
+            else
+                ++numFiltered;
         }
+        if (total)
+            *total -= numFiltered;
     }
     else {
         n = (elem->totalres.ordinality()>startoffset)?(elem->totalres.ordinality()-startoffset):0;

dali/base/dautils.hpp

@@ -274,7 +274,8 @@ extern da_decl IRemoteConnection *getElementsPaged( const char *basexpath,
                                      __int64 *hint,                         // if non null points to in/out cache hint
                                      const char *namefilterlo, // if non null filter less than this value
                                      const char *namefilterhi, // if non null filter greater than this value
-                                     IArrayOf<IPropertyTree> &results);
+                                     IArrayOf<IPropertyTree> &results,
+                                     unsigned *total); // total possible filtered matches, i.e. irrespective of startoffset and pagesize
 
 extern da_decl void clearPagedElementsCache();
 

dali/datest/dfuwutest.cpp

@@ -641,7 +641,7 @@ void testPagedIterate()
     unsigned n=0;
     for (unsigned page=0;page<3;page++) {
         DFUsortfield sortorder[] = {DFUsf_user,DFUsf_state,DFUsf_term};
-        Owned<IConstDFUWorkUnitIterator> iter = factory->getWorkUnitsSorted(sortorder, NULL, NULL, page*10, 10, "nigel", &cachehint);
+        Owned<IConstDFUWorkUnitIterator> iter = factory->getWorkUnitsSorted(sortorder, NULL, NULL, page*10, 10, "nigel", &cachehint, NULL);
         StringBuffer s;
         ForEach(*iter) {
 

dali/dfu/dfuwu.cpp

@@ -2967,7 +2967,8 @@ public:
                                                     unsigned startoffset,
                                                     unsigned maxnum,
                                                     const char *queryowner, 
-                                                    __int64 *cachehint)
+                                                    __int64 *cachehint,
+                                                    unsigned *total)
     {
         StringBuffer query("*");
         StringBuffer so;
@@ -3005,7 +3006,7 @@ public:
         }
         IArrayOf<IPropertyTree> results;
         Owned<IRemoteConnection> conn=getElementsPaged( "DFU/WorkUnits", query.str(), so.length()?so.str():NULL,startoffset,maxnum,
-            NULL,queryowner,cachehint,namefilterlo.get(),namefilterhi.get(),results);
+            NULL,queryowner,cachehint,namefilterlo.get(),namefilterhi.get(),results,total);
         return new CConstDFUWUArrayIterator(this,conn,results);
     }
 
@@ -3022,12 +3023,9 @@ public:
     {
         if (!filters)
             return numWorkUnits();
-        // bit slow
-        Owned<IConstDFUWorkUnitIterator> iter =  getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL);
-        unsigned ret = 0;
-        ForEach(*iter)
-            ret++;
-        return ret;
+        unsigned total;
+        Owned<IConstDFUWorkUnitIterator> iter = getWorkUnitsSorted( NULL,filters,filterbuf,0,0x7fffffff,NULL,NULL,&total);
+        return total;
     }
 
 

dali/dfu/dfuwu.hpp

@@ -440,7 +440,8 @@ interface IDFUWorkUnitFactory : extends IInterface
                                                         unsigned startoffset,
                                                         unsigned maxnum,
                                                         const char *queryowner, 
-                                                        __int64 *cachehint) = 0;    // set to NULL if caching not required
+                                                        __int64 *cachehint,         // set to NULL if caching not required
+                                                        unsigned *total) = 0;       // set to NULL if caching not required
     virtual unsigned numWorkUnits()=0;
     virtual unsigned numWorkUnitsFiltered(DFUsortfield *filters,const void *filterbuf)=0;
     virtual __int64  subscribe(const char *xpath,void *iface) =0;       // internal use

ecl/wutest/wutest.cpp

@@ -146,7 +146,7 @@ void testPagedWuList(IWorkUnitFactory *factory)
     unsigned n=0;
     for (unsigned page=0;page<3;page++) {
         WUSortField sortorder[] = {WUSFuser,WUSFstate,WUSFterm};
-        Owned<IConstWorkUnitIterator> it = factory->getWorkUnitsSorted(sortorder, NULL, NULL, page*10, 10, "nigel", &cachehint);
+        Owned<IConstWorkUnitIterator> it = factory->getWorkUnitsSorted(sortorder, NULL, NULL, page*10, 10, "nigel", &cachehint, NULL);
         ForEach(*it) {
             n++;
             IConstWorkUnit& wu = it->query();

esp/services/ws_fs/ws_fsService.cpp

@@ -952,8 +952,8 @@ bool CFileSprayEx::onGetDFUWorkunits(IEspContext &context, IEspGetDFUWorkunits &
 
         IArrayOf<IEspDFUWorkunit> result;
         Owned<IDFUWorkUnitFactory> factory = getDFUWorkUnitFactory();
-        unsigned numWUs = factory->numWorkUnitsFiltered(filters, filterbuf.bufferBase());
-        Owned<IConstDFUWorkUnitIterator> itr = factory->getWorkUnitsSorted(sortorder, filters, filterbuf.bufferBase(), (int) displayFrom, (int) pagesize+1, req.getOwner(), &cachehint);
+        unsigned numWUs;
+        Owned<IConstDFUWorkUnitIterator> itr = factory->getWorkUnitsSorted(sortorder, filters, filterbuf.bufferBase(), (int) displayFrom, (int) pagesize+1, req.getOwner(), &cachehint, &numWUs);
 
         //unsigned actualCount = 0;
         itr->first();

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2023,8 +2023,8 @@ void doWUQueryWithSort(IEspContext &context, IEspWUQueryRequest & req, IEspWUQue
     filters[filterCount] = WUSFterm;
 
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
-    unsigned numWUs = factory->numWorkUnitsFiltered(filters, filterbuf.bufferBase());
-    Owned<IConstWorkUnitIterator> it = factory->getWorkUnitsSorted(sortorder, filters, filterbuf.bufferBase(), begin, pagesize+1, "", NULL);
+    unsigned numWUs;
+    Owned<IConstWorkUnitIterator> it = factory->getWorkUnitsSorted(sortorder, filters, filterbuf.bufferBase(), begin, pagesize+1, "", NULL, &numWUs);
 
     unsigned actualCount = 0;
     ForEach(*it)