HPCC-11822 Improve archived workunit paging in WsWorkunits

1. Call getSortedDirectoryIterator() to retrieve the data about
archived workunits; 2. For archived workunits, the beforeWU/
afterWU and/or StartDate/EndDate are used to skip the directories
which store WUs not in the date range. The beforeWU/afterWU is
also used to skip the WUs which are not in the time range. 3. For
'online' workunits, the beforeWU/afterWU and/or StartDate/EndDate
are used to filter out the WUs which are not in the time range.
4. Remove PageStartFrom from paging request; 5. Refacter the code
inside WUiterate().

Also fixed 2 bugs: 1. unsorted pages are returned by the WUiterate;
2. a '-' is needed for Before/After in SashaCommand.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

dali/sasha/saarch.cpp

@@ -85,295 +85,507 @@ static void mkDateCompare(bool dfu,const char *dt,StringBuffer &out,char fill)
 
 void WUiterate(ISashaCommand *cmd, const char *mask)
 {
-    bool dfu = cmd->getDFU();
-    const char *beforedt = cmd->queryBefore();
-    const char *afterdt = cmd->queryAfter();
-    const char *owner = cmd->queryOwner();
-    const char *state = cmd->queryState();
-    const char *cluster = cmd->queryCluster();
-    const char *jobname = cmd->queryJobName();
-    const char *outputformat = cmd->queryOutputFormat();
-    const char *priority = cmd->queryPriority();
-    const char *fileread = cmd->queryFileRead();
-    const char *filewritten = cmd->queryFileWritten();
-    const char *eclcontains = cmd->queryEclContains();
-    const char *cmdname = cmd->queryDfuCmdName();
-    unsigned start = cmd->getStart(); 
-    unsigned num = cmd->getLimit();
-    StringBuffer before;
-    StringBuffer after;
-    StringBuffer tmppath;
-    mkDateCompare(dfu,afterdt,after,'0');
-    mkDateCompare(dfu,beforedt,before,'9');
-    bool haswusoutput = cmd->getAction()==SCA_WORKUNIT_SERVICES_GET;
-    bool hasdtoutput = cmd->getAction()==SCA_LISTDT;
-    MemoryBuffer WUSbuf;
-    if (haswusoutput)
-        cmd->setWUSresult(WUSbuf);  // swap in/out (in case ever do multiple)
-
-    StringBuffer baseXPath = dfu ? "DFU/WorkUnits" : "WorkUnits";
-    Owned<IRemoteConnection> conn;
-    bool isWild = !mask || !*mask || isWildString(mask);
-    bool unfiltered = !mask || !*mask || (0==strcmp(mask,"*"));
-    if (cmd->getArchived()) {
-        // used to check not online
-        StringBuffer path;
-        if (dfu)
-            getLdsPath("Archive/DFUWorkUnits",path);
-        else
-            getLdsPath("Archive/WorkUnits",path);
-        Owned<IFile> dir = createIFile(path.str());
-        StringBuffer masktmp;
-        if (unfiltered&&after.length()&&before.length()) {
-            const char *lo = after.str();
-            const char *hi = before.str();
-            while (*lo&&(toupper(*lo)==toupper(*hi))) {
-                masktmp.append((char)toupper(*lo));
-                lo++;
-                hi++;
+    class CWUData : public CInterface
+    {
+    public:
+        CDateTime modifiedTime;
+        Owned<IPropertyTree> wuTree;
+    };
+
+    class CArchivedWUReader
+    {
+        StringBuffer mask, fromDT, toDT, fromDir, toDir, beforeWU, afterWU, baseXPath;
+        StringAttr owner, cluster, jobName, state, priority, eclContains, fileRead, fileWritten, cmdName;
+        bool isWild, descendingReq, countBackward, outputXML;
+        unsigned maxNumWUs = 0;
+        StringArray outputFields;
+        ISashaCommand* cmd;
+        const char *fromDTDefinedByBeforeOrAfterWU = nullptr;
+        const char *toDTDefinedByBeforeOrAfterWU = nullptr;
+
+        void getFileMasks(StringBuffer &dirMask, StringBuffer &fileMask)
+        {
+            //The fromDT and toDT are the filters which defines a search range.
+            //The afterWU (beforeWU) defines paging within the search range. 
+            //For the first page, a user may define the fromDT and/or toDT without afterWU or beforeWU.
+            StringBuffer from, to;
+            if (!isEmptyString(fromDTDefinedByBeforeOrAfterWU) && (fromDT.isEmpty() || (strcmp(fromDTDefinedByBeforeOrAfterWU, fromDT.str()) > 0)))
+                from.set(fromDTDefinedByBeforeOrAfterWU);
+            else if (!fromDT.isEmpty())
+                from.set(fromDT.str());
+            if (!isEmptyString(toDTDefinedByBeforeOrAfterWU) && (toDT.isEmpty() || (strcmp(toDTDefinedByBeforeOrAfterWU, toDT.str()) < 0)))
+                to.set(toDTDefinedByBeforeOrAfterWU);
+            else if (!toDT.isEmpty())
+                to.set(toDT.str());
+
+            if (!isEmptyString(from))
+                splitWUID(from, fromDir);
+            if (!isEmptyString(to))
+                splitWUID(to, toDir);
+
+            bool unfiltered = mask.isEmpty() || streq(mask.str(), "*");
+
+            //The dirMask is used to filter out the directories (ex: W20171010) which store archived WU files.
+            //The fileMask is used to filter out the WU files (ex: W20171010-101010*.xml).
+            StringBuffer masktmp;
+            if (unfiltered && !from.isEmpty() && !to.isEmpty())
+            {
+                const char *lo = from.str();
+                const char *hi = to.str();
+                while (*lo && (toupper(*lo) == toupper(*hi)))
+                {
+                    masktmp.append((char)toupper(*lo));
+                    lo++;
+                    hi++;
+                }
+                masktmp.append("*");
+                mask = masktmp.str();
             }
-            masktmp.append("*");
-            mask = masktmp.str();
+
+            StringBuffer head;
+            if (!mask.isEmpty())
+            {
+                splitWUID(mask.str(), head);
+                if (!head.isEmpty())
+                    dirMask = head.str();
+                fileMask.set(mask.str()).toUpperCase();
+            }
+            else
+                fileMask.append("*");
+            fileMask.append(".xml");
         }
-        StringBuffer head, tmask;
-        const char *hmask = NULL;
-        if (mask&&*mask) {
-            splitWUID(mask,head);
-            if (head.length())
-                hmask = head.str();
-            tmask.clear().append(mask).toUpperCase();
+        bool checkDirs(const char *dir)
+        {
+            if (!fromDir.isEmpty() && (strcmp(dir, fromDir.str()) < 0))
+                return false;
+            if (!toDir.isEmpty() && (strcmp(dir, toDir.str()) > 0))
+                return false;
+            return true;
         }
-        else
-            tmask.append("*");
-        tmask.append(".xml");
-        Owned<IDirectoryIterator> di = dir->directoryFiles(hmask,false,true);
-        StringBuffer name;
-        unsigned index = 0;
-        StringBuffer xb;
-        bool overflowed = false;
-        ForEach(*di) {
-            if (overflowed||(index>start+num))
-                break;
-            if (di->isDir()) {
-                Owned<IDirectoryIterator> di2 = di->query().directoryFiles(tmask.str(),false);
+        bool checkArchivedWUID(const char *wuid)
+        {
+            if (isEmptyString(wuid))
+                return false;
+
+            if (!mask.isEmpty() && !WildMatch(wuid, mask.str(),true))
+                return false;
+
+            if (!isEmptyString(fromDTDefinedByBeforeOrAfterWU) && (stricmp(wuid, fromDTDefinedByBeforeOrAfterWU) <= 0))
+                return false;
+            if (!isEmptyString(toDTDefinedByBeforeOrAfterWU) && (stricmp(wuid, toDTDefinedByBeforeOrAfterWU) >= 0))
+                return false;
+
+            if (!fromDT.isEmpty() && (stricmp(wuid, fromDT.str()) < 0))
+                return false;
+            if (!toDT.isEmpty() && (stricmp(wuid, toDT.str()) > 0))
+                return false;
+
+            return true;
+        }
+        bool checkOnlineWUID(const char *wuid)
+        {
+            if (isEmptyString(wuid))
+                return false;
+
+            if (!isEmptyString(fromDTDefinedByBeforeOrAfterWU) && (stricmp(wuid, fromDTDefinedByBeforeOrAfterWU) <= 0))
+                return false;
+            if (!isEmptyString(toDTDefinedByBeforeOrAfterWU) && (stricmp(wuid, toDTDefinedByBeforeOrAfterWU) >= 0))
+                return false;
+
+            if (!fromDT.isEmpty() && (stricmp(wuid, fromDT.str()) < 0))
+                return false;
+            if (!toDT.isEmpty() && (stricmp(wuid, toDT.str()) > 0))
+                return false;
+
+            return true;
+        }
+        const char *readWUIDFromFileName(StringBuffer &name)
+        {
+            if (name.length() < 5)
+                return nullptr;
+
+            name.setLength(name.length()-4);
+            name.toUpperCase();
+            const char *wuid = name.str();
+            if ((name.length()>6) && strieq(wuid+name.length()-6, "_HINTS"))
+                return nullptr;
+            return name.str();
+        }
+        IRemoteConnection *getSDSConnection(const char *mask)
+        {
+            if (!mask)
+                return querySDS().connect(baseXPath.str(), myProcessSession(), 0, 5*60*1000);
+
+            VStringBuffer xpath("%s/%s", baseXPath.str(), mask);
+            return querySDS().connect(xpath.str(), myProcessSession(), 0, 5*60*1000);
+        }
+        bool isOnline(Owned<IRemoteConnection> &conn, const char *wuid)
+        {
+            return (isWild && conn) ? conn->queryRoot()->hasProp(wuid) : conn != nullptr;
+        }
+        bool checkFilters(IPropertyTree *t, bool onlineWU)
+        {
+            StringBuffer path, val;
+            if (!owner.isEmpty() && (!t->getProp("@submitID", val.clear()) || !WildMatch(val.str(), owner.get(), true)))
+                return false;
+            if (!state.isEmpty() && (!t->getProp((!onlineWU && cmd->getDFU()) ? "Progress/@state" : "@state", val.clear()) || !WildMatch(val.str(), state.get(), true)))
+                return false;
+            if (!cluster.isEmpty() && (!t->getProp("@clusterName", val.clear()) || !WildMatch(val.str(), cluster.get(), true)))
+                return false;
+            if (!jobName.isEmpty() && (!t->getProp("@jobName", val.clear()) || !WildMatch(val.str(), jobName.get(), true)))
+                return false;
+
+            if (onlineWU)
+                return true;
+
+            if (!cmdName.isEmpty() && (!t->getProp("@command", val.clear()) || !WildMatch(val.str(), cmdName.get(), true)))
+                return false;
+            if (!priority.isEmpty() && (!t->getProp("@priorityClass", val.clear()) || !WildMatch(val.str(), priority.get(), true)))
+                return false;
+            if (!fileRead.isEmpty() && !t->hasProp(path.setf("FilesRead/File[@name=~?\"%s\"]", fileRead.get()).str()))
+                return false;
+            if (!fileWritten.isEmpty() && !t->hasProp(path.setf("Files/File[@name=~?\"%s\"]", fileWritten.get()).str()))
+                return false;
+            if (!eclContains.isEmpty() && !t->hasProp(path.setf("Query[Text=~?\"*%s*\"]", eclContains.get()).str()))
+                return false;
+            return true;
+        }
+        bool addOutput(bool outputModifiedTime, bool hasWUSOutput, IPropertyTree *wuTree,
+            CDateTime &modifiedTime, MemoryBuffer &WUSbuf)
+        {
+            if (hasWUSOutput)
+            { //cmd->getAction()==SCA_WORKUNIT_SERVICES_GET
+                if (!serializeWUSrow(*wuTree, WUSbuf, WORKUNIT_SERVICES_BUFFER_MAX, false))
+                    return false; //Log overflowed?
+            }
+            else if (!addOutputFromWUTree(wuTree))
+                return false; //Log overflowed?
+            else if (outputModifiedTime)
+                cmd->addDT(modifiedTime);
+            return true;
+        }
+        bool addOutputFromWUTree(IPropertyTree *wuTree)
+        {
+            if (outputXML)
+            { //cmd->getAction()==SCA_GET
+                StringBuffer xml;
+                toXML(wuTree, xml);
+                if (!cmd->addResult(xml.str()))
+                    return false; //Log overflowed?
+            }
+            else
+            {
+                if (outputFields.length() == 0)
+                    cmd->addId(wuTree->queryName());
+                else
+                {
+                    StringBuffer output = wuTree->queryName();
+                    //Append more into the output.
+                    setWUDataTree(wuTree, output);
+                    cmd->addId(output.str());
+                }
+            }
+            return true;
+        }
+        void setWUDataTree(IPropertyTree *wu, StringBuffer &output)
+        {
+            ForEachItemIn(i, outputFields)
+            {
+                const char* outputField = outputFields.item(i);
+
                 StringBuffer val;
-                ForEach(*di2) {
-                    di2->getName(name.clear());
-                    if (!di2->isDir()&&(name.length()>4)) {
-                        name.setLength(name.length()-4);
-                        name.toUpperCase();
-                        const char *wuid = name.str();
-                        if ((name.length()>6)&&(stricmp(wuid+name.length()-6,"_HINTS")==0))
-                            continue;
-                        if ((!mask||!*mask||WildMatch(wuid,mask,true)) &&
-                            ((before.length()==0)||(stricmp(wuid,before.str())<=0)) &&
-                            ((after.length()==0)||(stricmp(wuid,after.str())>=0))) {
-                            if (isWild) {
-                                if (!conn)
-                                    conn.setown(querySDS().connect(baseXPath.str(), myProcessSession(), 0, 5*60*1000)); // connection to all
-                            }
-                            else {
-                                VStringBuffer xpath("%s/%s", baseXPath.str(), mask);
-                                conn.setown(querySDS().connect(xpath.str(), myProcessSession(), 0, 5*60*1000));
-                            }
-                            if ((isWild && !conn->queryRoot()->hasProp(wuid)) || (!isWild && !conn)) { // check not online
-                                Owned<IPropertyTree> t;
-                                bool hasowner = owner&&*owner;
-                                bool hascluster = cluster&&*cluster;
-                                bool hasstate = state&&*state;
-                                bool hasjobname = jobname&&*jobname;
-                                bool hasoutput = outputformat&&*outputformat;
-                                bool inrange = (index>=start)&&(index<start+num);
-                                bool hascommand = cmdname&&*cmdname;
-                                bool haspriority = priority&&*priority;
-                                bool hasfileread = fileread&&*fileread;
-                                bool hasfilewritten = filewritten&&*filewritten;
-                                bool haseclcontains = eclcontains&&*eclcontains;
-                                if ((cmd->getAction()==SCA_GET)||haswusoutput||hasowner||hasstate||hascluster||hasjobname||hascommand||(hasoutput&&inrange)||haspriority||hasfileread||hasfilewritten||haseclcontains) {
-                                    try {
-                                        t.setown(createPTree(di2->query()));
-                                        if (!t)
-                                            continue;
-                                        if (hasowner&&(!t->getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true)))
-                                            continue;
-                                        if (hasstate&&(!t->getProp(dfu?"Progress/@state":"@state",val.clear())||!WildMatch(val.str(),state,true)))
-                                            continue;
-                                        if (hascluster&&(!t->getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true)))
-                                            continue;
-                                        if (hasjobname&&(!t->getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true)))
-                                            continue;
-                                        if (hascommand&&(!t->getProp("@command",val.clear())||!WildMatch(val.str(),cmdname,true)))
-                                            continue;
-                                        if (haspriority&&(!t->getProp("@priorityClass",val.clear())||!WildMatch(val.str(),priority,true)))
-                                            continue;
-                                        if (hasfileread&&!t->hasProp(tmppath.clear().appendf("FilesRead/File[@name=~?\"%s\"]",fileread).str()))
-                                            continue;
-                                        if (hasfilewritten&&!t->hasProp(tmppath.clear().appendf("Files/File[@name=~?\"%s\"]",filewritten).str()))
-                                            continue;
-                                        if (haseclcontains&&!t->hasProp(tmppath.clear().appendf("Query[Text=~?\"*%s*\"]",eclcontains).str()))
-                                            continue;
-                                    }
-                                    catch (IException *e) {
-                                        StringBuffer msg;
-                                        msg.appendf("WUiterate: Workunit %s failed to load", wuid);
-                                        EXCLOG(e,msg.str());
-                                        e->Release();
-                                        continue;
-                                    }
-                                }
-                                index++;
-                                if (!inrange)
-                                    continue;
-                                if (hasoutput) {
-                                    char *saveptr;
-                                    char *parse = strdup(outputformat);
-                                    char *tok = strtok_r(parse, "|,",&saveptr);
-                                    while (tok) {
-                                        val.clear();
-                                        bool found = true;
-                                        if (stricmp(tok,"owner")==0)
-                                            t->getProp("@submitID",val);
-                                        else if (stricmp(tok,"cluster")==0)
-                                            t->getProp("@clusterName",val);
-                                        else if (stricmp(tok,"jobname")==0)
-                                            t->getProp("@jobName",val);
-                                        else if (stricmp(tok,"state")==0)
-                                            t->getProp(dfu?"Progress/@state":"@state",val);
-                                        else if (stricmp(tok,"command")==0)
-                                            t->getProp("@command",val);
-                                        else if (stricmp(tok,"wuid")==0)
-                                            t->getName(val);
-                                        else
-                                            found = false;
-                                        if (found) {
-                                            // remove commas TBD
-                                            name.append(',').append(val);
-                                        }
-                                        tok = strtok_r(NULL, "|,",&saveptr);
-                                    }
-                                    free(parse);
-                                }
-                                if (haswusoutput) {
-                                    if (!serializeWUSrow(*t,WUSbuf,false)) {
-                                        overflowed = true;
-                                        break;
-                                    }
-                                }
-                                else {
-                                    cmd->addId(name.str());
-                                    if (hasdtoutput) {
-                                        CDateTime dt;
-                                        di2->getModifiedTime(dt);
-                                        cmd->addDT(dt);
-                                    }
-                                }
-                                if (cmd->getAction()==SCA_GET) {
-                                    StringBuffer xml;
-                                    toXML(t,xml);
-                                    if (!cmd->addResult(xml.str()))
-                                        break;
-                                }
-                            }
-                        }
-                    }
-                    if (index>start+num)
+                bool found = true;
+                if (strieq(outputField, "owner"))
+                    wu->getProp("@submitID", val);
+                else if (strieq(outputField, "cluster"))
+                    wu->getProp("@clusterName", val);
+                else if (strieq(outputField, "jobname"))
+                    wu->getProp("@jobName",val);
+                else if (strieq(outputField,"state"))
+                    wu->getProp("@state", val);
+                else
+                    found = false;
+                if (found)
+                    output.append(',').append(val);
+            }
+        }
+        bool checkOnlineWU(IPropertyTree *wuTree)
+        {
+            const char *wuid = wuTree->queryName();
+            if (!checkOnlineWUID(wuid))
+                return false;
+
+            try
+            {
+                if (!checkFilters(wuTree, true))
+                    return false;
+            }
+            catch (IException *e)
+            {
+                VStringBuffer msg("WUiterate: Workunit %s failed to load", wuid);
+                EXCLOG(e,msg.str());
+                e->Release();
+                return false;
+            }
+
+            return true;
+        }
+
+        void getOnlineWUsAscending(IRemoteConnection *conn)
+        {
+            IArrayOf<IPropertyTree> wus;
+            unsigned wuCount = 0;
+            Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements(isWild ? (mask.isEmpty() ? "*" : mask.str()) : nullptr, iptiter_sort);
+            ForEach(*iter)
+            {
+                IPropertyTree *pt = &iter->query();
+                if (!checkOnlineWU(pt))
+                    continue;
+
+                if (countBackward)
+                    wus.append(*LINK(pt));
+                else if (!addOutputFromWUTree(pt))
+                    break;
+
+                wuCount++;
+                if (wuCount == maxNumWUs)
+                    break;
+            }
+            if (countBackward)
+            {
+                ForEachItemInRev(i, wus)
+                {
+                    if (!addOutputFromWUTree(&wus.item(i)))
                         break;
                 }
             }
         }
-    }
-    if (cmd->getOnline()) {
-        if (haswusoutput)
-            throw MakeStringException(-1,"SCA_WORKUNIT_SERVICES_GET not implemented for online workunits!");
-        StringBuffer xpath(baseXPath);
-        if (!conn)
+
+        void getOnlineWUsDescending(IRemoteConnection *conn)
+        {
+            IArrayOf<IPropertyTree> wus;
+            Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements(isWild ? (mask.isEmpty() ? "*" : mask.str()) : nullptr);
+            ForEach(*iter)
+            {
+                IPropertyTree *pt = &iter->query();
+                if (!checkOnlineWU(pt))
+                    continue;
+
+                wus.append(*LINK(pt));
+            }
+            if (!wus.empty())
+            {
+                wus.sort(comparePropTrees);
+
+                if (countBackward)
+                {
+                    unsigned i = wus.ordinality();
+                    if (i > maxNumWUs)
+                        i = maxNumWUs;
+                    for (; i--;)
+                    {
+                        if (!addOutputFromWUTree(&wus.item(i)))
+                            break;
+                    }
+                }
+                else
+                {
+                    unsigned wuCount = 0;
+                    ForEachItemIn(i, wus)
+                    {
+                        if (!addOutputFromWUTree(&wus.item(i)))
+                            break;
+                        wuCount++;
+                        if (wuCount == maxNumWUs)
+                            break;
+                    }
+                }
+            }
+        }
+        static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
         {
-            if (!isWild)
-                xpath.append("/").append(mask);
-            conn.setown(querySDS().connect(xpath.str(), myProcessSession(), 0, 5*60*1000));
+            IPropertyTree *l = (IPropertyTree *) *ll;
+            IPropertyTree *r = (IPropertyTree *) *rr;
+            return stricmp(r->queryName(), l->queryName());
         }
-        if (conn)
+
+    public:
+
+        CArchivedWUReader(ISashaCommand *_cmd, const char *_mask)
         {
-            Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements(isWild ? "*" : NULL);
-            unsigned index = 0;
-            StringBuffer val;
-            ForEach(*iter) {
-                IPropertyTree &pt=iter->query();
-                const char *wuid = pt.queryName();
-                if (index>start+num)
-                    break;
-        //      PROGLOG("match before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
-                if ((!mask||!*mask||!isWild||WildMatch(wuid,mask,true)) &&
-                    ((before.length()==0)||(stricmp(wuid,before)<0)) &&
-                    ((after.length()==0)||(stricmp(wuid,after)>=0))) {
-        //          PROGLOG("matched before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
-                    bool hasowner = owner&&*owner;
-                    bool hascluster = cluster&&*cluster;
-                    bool hasstate = state&&*state;
-                    bool hasjobname = jobname&&*jobname;
-                    bool hasoutput = outputformat&&*outputformat;
-                    bool inrange = (index>=start)&&(index<start+num);
-                    if (hasowner||hasstate||hascluster||hasjobname||(hasoutput&&inrange)) {
-                        try {
-                            if (hasowner&&(!pt.getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true)))
-                                continue;
-                            if (hasstate&&(!pt.getProp("@state",val.clear())||!WildMatch(val.str(),state,true)))
-                                continue;
-                            if (hascluster&&(!pt.getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true)))
-                                continue;
-                            if (hasjobname&&(!pt.getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true)))
+            cmd = _cmd;
+            mask.set(_mask);
+            isWild = mask.isEmpty() || isWildString(mask.str());
+
+            bool dfu = cmd->getDFU();
+            baseXPath.set(dfu ? "DFU/WorkUnits" : "WorkUnits");
+            owner.set(cmd->queryOwner());
+            cluster.set(cmd->queryCluster());
+            jobName.set(cmd->queryJobName());
+            state.set(cmd->queryState());
+            priority.set(cmd->queryPriority());
+            eclContains.set(cmd->queryEclContains());
+            fileRead.set(cmd->queryFileRead());
+            fileWritten.set(cmd->queryFileWritten());
+            cmdName.set(cmd->queryDfuCmdName());
+
+            //In WUiterate(), the queryAfter() is used as fromDT and the queryBefore() is used as toDT.
+            mkDateCompare(dfu, cmd->queryAfter(), fromDT, '0');
+            mkDateCompare(dfu, cmd->queryBefore(), toDT, '9');
+
+            beforeWU.set(cmd->queryBeforeWU());
+            afterWU.set(cmd->queryAfterWU());
+            descendingReq = cmd->querySortDescending();
+            if (descendingReq)
+            {
+                fromDTDefinedByBeforeOrAfterWU = beforeWU.str();
+                toDTDefinedByBeforeOrAfterWU = afterWU.str();
+            }
+            else
+            {
+                fromDTDefinedByBeforeOrAfterWU = afterWU.str();
+                toDTDefinedByBeforeOrAfterWU = beforeWU.str();
+            }
+
+            //The countBackward below is used to support paging. For example, a client retrieves
+            //archived WUs with the order new->old (Descending = true) and the client is on page 50.
+            //If a client wants to go back to the previous page by specifying the beforeWU,
+            //the countBackward is set to true (old->new). We call the getSortedDirectoryIterator()
+            //with the order old->new, find the WU before the 'beforeWU', count a page of WUs, and
+            //reorder them to new->old.
+            countBackward = !beforeWU.isEmpty() && afterWU.isEmpty();
+
+            maxNumWUs = cmd->getLimit();
+            outputXML = cmd->getAction()==SCA_GET;
+            const char *outputFormat = cmd->queryOutputFormat();
+            outputFields.appendList(outputFormat, "|,");
+        }
+        void getArchivedWUs()
+        {
+            bool outputModifiedTime = cmd->getAction()==SCA_LISTDT;
+            StringBuffer dirMask, fileMask, path, name;
+            getFileMasks(dirMask, fileMask);
+
+            if (cmd->getDFU())
+                getLdsPath("Archive/DFUWorkUnits", path);
+            else
+                getLdsPath("Archive/WorkUnits", path);
+
+            MemoryBuffer WUSbuf;
+            bool hasWUSOutput = cmd->getAction()==SCA_WORKUNIT_SERVICES_GET;
+            if (hasWUSOutput)
+                cmd->setWUSresult(WUSbuf);  // swap in/out (in case ever do multiple)
+
+            CIArrayOf<CWUData> wus;
+            unsigned wuCount = 0;
+            bool overflowed = false;
+            Owned<IRemoteConnection> conn;
+            Owned<IDirectoryIterator> dirIterator = getSortedDirectoryIterator(path.str(), SD_bynameNC,
+                countBackward ? !descendingReq : descendingReq, dirMask.isEmpty() ? nullptr : dirMask.str(), false, true);
+            ForEach(*dirIterator)
+            {
+                dirIterator->getName(name.clear());
+                if (!dirIterator->isDir() || !checkDirs(name.str()))
+                    continue;
+
+                Owned<IDirectoryIterator> fileIterator = getSortedDirectoryIterator(&dirIterator->query(),
+                    SD_bynameNC, countBackward ? !descendingReq : descendingReq, fileMask.str(), false);
+                ForEach(*fileIterator)
+                {
+                    fileIterator->getName(name.clear());
+                    if (!checkArchivedWUID(readWUIDFromFileName(name)))
+                        continue;
+
+                    const char *wuid = name.str();
+                    if (!isWild)
+                        conn.setown(getSDSConnection(mask.str()));
+                    else if (!conn)
+                        conn.setown(getSDSConnection(nullptr));
+                    if (isOnline(conn, wuid))
+                        continue;
+
+                    Owned<IPropertyTree> wuTree;
+                    if (outputXML || hasWUSOutput || (outputFields.length() > 0) || !owner.isEmpty()
+                        || !state.isEmpty() || !cluster.isEmpty() || !jobName.isEmpty()
+                        || !cmdName.isEmpty() || !priority.isEmpty() || !fileRead.isEmpty()
+                        || !fileWritten.isEmpty() || !eclContains.isEmpty())
+                    {
+                        try
+                        {
+                            wuTree.setown(createPTree(fileIterator->query()));
+                            if (!wuTree || isEmptyString(wuTree->queryName()) || !checkFilters(wuTree, false))
                                 continue;
                         }
-                        catch (IException *e) {
-                            StringBuffer msg;
-                            msg.appendf("WUiterate: Workunit %s failed", wuid);
+                        catch (IException *e)
+                        {
+                            VStringBuffer msg("WUiterate: Workunit %s failed to load", wuid);
                             EXCLOG(e,msg.str());
                             e->Release();
                             continue;
                         }
                     }
-                    index++;
-                    if (!inrange)
-                        continue;
-                    StringBuffer name(wuid);
-                    if (hasoutput) {
-                        char *saveptr;
-                        char *parse = strdup(outputformat);
-                        char *tok = strtok_r(parse, "|,",&saveptr);
-                        while (tok) {
-                            val.clear();
-                            bool found = true;
-                            if (stricmp(tok,"owner")==0)
-                                pt.getProp("@submitID",val);
-                            else if (stricmp(tok,"cluster")==0)
-                                pt.getProp("@clusterName",val);
-                            else if (stricmp(tok,"jobname")==0)
-                                pt.getProp("@jobName",val);
-                            else if (stricmp(tok,"state")==0)
-                                pt.getProp("@state",val);
-                            else
-                                found = false;
-                            if (found)
-                                name.append(',').append(val);
-                            tok = strtok_r(NULL, "|,",&saveptr);
-                        }
-                        free(parse);
+
+                    if (countBackward)
+                    {   //The output should contain WUs in an order specified by the descendingReq.
+                        //Since the getSortedDirectoryIterator() returns WUs in an order of the
+                        //!descendingReq, we store them into the wus for now.
+                        Owned<CWUData> wu = new CWUData();
+                        wu->wuTree.setown(wuTree.getClear());
+                        if (outputModifiedTime)
+                            fileIterator->getModifiedTime(wu->modifiedTime);
+                        wus.append(*wu.getClear());
                     }
-                    cmd->addId(name.str());
-                    if (cmd->getAction()==SCA_GET) {
-                        StringBuffer xml;
-                        toXML(&pt,xml);
-                        if (!cmd->addResult(xml.str()))
+                    else
+                    {
+                        CDateTime dt;
+                        if (outputModifiedTime)
+                            fileIterator->getModifiedTime(dt);
+
+                        if (!addOutput(outputModifiedTime, hasWUSOutput, wuTree, dt, WUSbuf))
+                        {
+                            overflowed = true;
                             break;
+                        }
                     }
+                    wuCount++;
+                    if (wuCount == maxNumWUs)
+                        break;
                 }
-                if (index>start+num)
+                if (overflowed || (wuCount == maxNumWUs))
                     break;
             }
+            if (countBackward)
+            { //now, we output the wus in reverse order.
+                ForEachItemInRev(i, wus)
+                {
+                    CWUData &wuData = wus.item(i);
+                    if (!addOutput(outputModifiedTime, hasWUSOutput, wuData.wuTree, wuData.modifiedTime, WUSbuf))  //Log an error?
+                        break;
+                }
+            }
+
+            if (hasWUSOutput)
+                cmd->setWUSresult(WUSbuf);
         }
-    }
-    if (haswusoutput)
-        cmd->setWUSresult(WUSbuf);
+        void getOnlineWUs()
+        {
+            if (cmd->getAction()==SCA_WORKUNIT_SERVICES_GET)
+                throw MakeStringException(-1,"SCA_WORKUNIT_SERVICES_GET not implemented for online workunits!");
+
+            Owned<IRemoteConnection> conn = getSDSConnection(isWild ? nullptr : mask.str());
+            if (!conn)
+                return;
+    
+            if (countBackward ? !descendingReq : descendingReq)
+                getOnlineWUsDescending(conn);
+            else
+                getOnlineWUsAscending(conn);
+        }
+    } reader(cmd, mask);
+    if (cmd->getArchived())
+        reader.getArchivedWUs();
+    if (cmd->getOnline())
+        reader.getOnlineWUs();
 }
 
 

dali/sasha/sacmd.cpp

@@ -22,8 +22,10 @@ class CSashaCommand: public CInterface, implements ISashaCommand
     CDateTime *dts;
     unsigned numdts;
 
-    StringAttr after;
-    StringAttr before;
+    StringAttr after; //datetime
+    StringAttr before; //datetime
+    StringAttr afterWU;
+    StringAttr beforeWU;
     StringAttr state;
     StringAttr owner;
     StringAttr cluster;
@@ -47,6 +49,7 @@ class CSashaCommand: public CInterface, implements ISashaCommand
     bool archived;
     bool dfu;
     bool wuservices;
+    bool sortDescending = false;
     unsigned start;
     unsigned limit;
     CMessageBuffer msgbuf;  // used for reply
@@ -150,6 +153,12 @@ public:
                 }
             }
         }
+        if (mb.remaining() > 0)
+            mb.read(sortDescending);
+        if (mb.remaining() > 0) {
+            mb.read(afterWU);
+            mb.read(beforeWU);
+        }
     }
 
     void serialize(MemoryBuffer &mb)
@@ -195,6 +204,11 @@ public:
         for (i=0;i<numdts;i++) 
             dts[i].serialize(mb);
 
+        mb.append(sortDescending);
+        if (afterWU.get() || beforeWU.get()) {
+            mb.append(afterWU);
+            mb.append(beforeWU);
+        }
     }
 
     SashaCommandAction getAction()
@@ -280,6 +294,36 @@ public:
         before.set(val);
     }
 
+    const char *queryAfterWU() const
+    {
+        return afterWU.get();
+    }
+
+    void setAfterWU(const char *val)
+    {
+        afterWU.set(val);
+    }
+
+    const char *queryBeforeWU() const
+    {
+        return beforeWU.get();
+    }
+
+    void setBeforeWU(const char *val)
+    {
+        beforeWU.set(val);
+    }
+
+    bool querySortDescending()
+    {
+        return sortDescending;
+    }
+
+    void setSortDescending(bool _sortDescending)
+    {
+        sortDescending = _sortDescending;
+    }
+
     const char *queryState()
     {
         return retstr(state);

dali/sasha/sacmd.hpp

@@ -35,6 +35,12 @@ interface ISashaCommand: extends IInterface
     virtual void setAfter(const char *val) = 0;
     virtual const char *queryBefore() = 0;
     virtual void setBefore(const char *val) = 0;
+    virtual const char *queryAfterWU() const = 0;
+    virtual void setAfterWU(const char *val) = 0;
+    virtual const char *queryBeforeWU() const = 0;
+    virtual void setBeforeWU(const char *val) = 0;
+    virtual bool querySortDescending() = 0;
+    virtual void setSortDescending(bool val) = 0;
     virtual const char *queryState() = 0;
     virtual void setState(const char *val) = 0;
     virtual const char *queryOwner() = 0;

esp/scm/ws_workunits.ecm

@@ -437,9 +437,11 @@ ESPrequest [nil_remove] WUQueryRequest
     [depr_ver("1.57")] string ApplicationKey;
     [depr_ver("1.57")] string ApplicationData;
     [min_ver("1.57")] ESParray<ESPstruct ApplicationValue> ApplicationValues;
+    [min_ver("1.72")] string BeforeWU;
+    [min_ver("1.72")] string AfterWU;
 
-    string After;
-    string Before;
+    [depr_ver("1.02")] string After; //Not used since 1.02
+    [depr_ver("1.02")] string Before; //Not used since 1.02
     int Count;
     [min_ver("1.03")] int64 PageSize(0);
     [min_ver("1.03")] int64 PageStartFrom(0);
@@ -499,6 +501,8 @@ ESPrequest [nil_remove] WULightWeightQueryRequest
     string JobName;
     string StartDate;
     string EndDate;
+    [min_ver("1.72")] string BeforeWU;
+    [min_ver("1.72")] string AfterWU;
     string State;
 
     ESParray<ESPstruct ApplicationValue> ApplicationValues;
@@ -2119,7 +2123,7 @@ ESPResponse [exceptions_inline] WUDetailsMetaResponse
 // ----------------------------------------------------------------------------------
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.71"), default_client_version("1.71"),
+    version("1.72"), default_client_version("1.72"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2119,7 +2119,7 @@ void doWULightWeightQueryWithSort(IEspContext &context, IEspWULightWeightQueryRe
 class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
 {
     IEspContext& context;
-    unsigned pageFrom, pageSize;
+    unsigned pageSize;
     StringAttr sashaServerIP;
     unsigned sashaServerPort;
     unsigned cacheMinutes;
@@ -2137,7 +2137,7 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
             timeTo.setString(endDateReq, NULL, true);
             timeTo.getDate(year, month, day, true);
             timeTo.getTime(hour, minute, second, nano, true);
-            to.setf("%4d%02d%02d%02d%02d%02d", year, month, day, hour, minute, second);
+            to.setf("%4d%02d%02d-%02d%02d%02d", year, month, day, hour, minute, second);
         }
 
         if(isEmpty(startDateReq))
@@ -2150,7 +2150,7 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
         unsigned year0, month0, day0, hour0, minute0, second0, nano0;
         timeFrom.getDate(year0, month0, day0, true);
         timeFrom.getTime(hour0, minute0, second0, nano0, true);
-        from.setf("%4d%02d%02d%02d%02d%02d", year0, month0, day0, hour0, minute0, second0);
+        from.setf("%4d%02d%02d-%02d%02d%02d", year0, month0, day0, hour0, minute0, second0);
         return;
     }
 
@@ -2180,7 +2180,9 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
         addToFilterString("state", req.getState());
         addToFilterString("timeFrom", req.getStartDate());
         addToFilterString("timeTo", req.getEndDate());
-        addToFilterString("pageStart", pageFrom);
+        addToFilterString("beforeWU", req.getBeforeWU());
+        addToFilterString("afterWU", req.getAfterWU());
+        addToFilterString("descending", req.getDescending());
         addToFilterString("pageSize", pageSize);
         if (sashaServerIP && *sashaServerIP)
         {
@@ -2199,7 +2201,9 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
         addToFilterString("state", req.getState());
         addToFilterString("timeFrom", req.getStartDate());
         addToFilterString("timeTo", req.getEndDate());
-        addToFilterString("pageStart", pageFrom);
+        addToFilterString("beforeWU", req.getBeforeWU());
+        addToFilterString("afterWU", req.getAfterWU());
+        addToFilterString("descending", req.getDescending());
         addToFilterString("pageSize", pageSize);
         if (sashaServerIP && *sashaServerIP)
         {
@@ -2214,7 +2218,6 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
         cmd->setOutputFormat("owner,jobname,cluster,state");
         cmd->setOnline(false);
         cmd->setArchived(true);
-        cmd->setStart(pageFrom);
         cmd->setLimit(pageSize+1); //read an extra WU to check hasMoreWU
     }
 
@@ -2237,6 +2240,11 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
             cmd->setAfter(timeFrom.str());
         if (timeTo.length())
             cmd->setBefore(timeTo.str());
+        if (notEmpty(req.getBeforeWU()))
+            cmd->setBeforeWU(req.getBeforeWU());
+        if (notEmpty(req.getAfterWU()))
+            cmd->setAfterWU(req.getAfterWU());
+        cmd->setSortDescending(req.getDescending());
         return;
     }
 
@@ -2259,6 +2267,11 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader
             cmd->setAfter(timeFrom.str());
         if (timeTo.length())
             cmd->setBefore(timeTo.str());
+        if (notEmpty(req.getBeforeWU()))
+            cmd->setBeforeWU(req.getBeforeWU());
+        if (notEmpty(req.getAfterWU()))
+            cmd->setAfterWU(req.getAfterWU());
+        cmd->setSortDescending(req.getDescending());
 
         return;
     }
@@ -2327,9 +2340,9 @@ public:
     IMPLEMENT_IINTERFACE_USING(CInterface);
 
     CArchivedWUsReader(IEspContext& _context, const char* _sashaServerIP, unsigned _sashaServerPort, ArchivedWuCache& _archivedWuCache,
-        unsigned _cacheMinutes, unsigned _pageFrom, unsigned _pageSize)
+        unsigned _cacheMinutes, unsigned _pageSize)
         : context(_context), sashaServerIP(_sashaServerIP), sashaServerPort(_sashaServerPort),
-        archivedWuCache(_archivedWuCache), cacheMinutes(_cacheMinutes), pageFrom(_pageFrom), pageSize(_pageSize)
+        archivedWuCache(_archivedWuCache), cacheMinutes(_cacheMinutes), pageSize(_pageSize)
     {
         hasMoreWU = false;
         numberOfWUsReturned = 0;
@@ -2418,10 +2431,6 @@ public:
                 archivedLWWUs.append(*info.getClear());
             }
         }
-        if (!lightWeight)
-            archivedWUs.sort(compareWuids);
-        else
-            archivedLWWUs.sort(compareLWWuids);
 
         archivedWuCache.add(filterStr, "AddWhenAvailable", hasMoreWU, numberOfWUsReturned, archivedWUs, archivedLWWUs);
         return;
@@ -2434,12 +2443,12 @@ public:
 void doWUQueryFromArchive(IEspContext &context, const char* sashaServerIP, unsigned sashaServerPort,
        ArchivedWuCache &archivedWuCache, unsigned cacheMinutes, IEspWUQueryRequest & req, IEspWUQueryResponse & resp)
 {
-    unsigned pageStart = (unsigned) req.getPageStartFrom();
+    //Sasha server does noy support the PageStartFrom due to inefficient access to archived workunits for pages>1.
     unsigned pageSize = (unsigned) req.getPageSize();
     if(pageSize < 1)
         pageSize=500;
     Owned<IArchivedWUsReader> archiveWUsReader = new CArchivedWUsReader(context, sashaServerIP, sashaServerPort, archivedWuCache,
-        cacheMinutes, pageStart, pageSize);
+        cacheMinutes, pageSize);
 
     IArrayOf<IEspECLWorkunit> archivedWUs;
     IArrayOf<IEspECLWorkunitLW> dummyWUs;
@@ -2453,29 +2462,15 @@ void doWUQueryFromArchive(IEspContext &context, const char* sashaServerIP, unsig
 
     resp.setType("archived only");
     resp.setPageSize(pageSize);
-    resp.setPageStartFrom(pageStart+1);
-    resp.setPageEndAt(pageStart + archiveWUsReader->getNumberOfWUsReturned());
-    if(pageStart > 0)
-    { //This is not the first page;
-        resp.setFirst(false);
-        resp.setPrevPage((pageStart > pageSize) ? pageStart - pageSize: 0);
-    }
-    if (archiveWUsReader->getHasMoreWU())
-        resp.setNextPage(pageStart + pageSize);
     return;
 }
 
 void doWULightWeightQueryFromArchive(IEspContext &context, const char* sashaServerIP, unsigned sashaServerPort,
        ArchivedWuCache &archivedWuCache, unsigned cacheMinutes, IEspWULightWeightQueryRequest & req, IEspWULightWeightQueryResponse & resp)
 {
-    int pageStart = 0;
-    int pageSize=500;
-    if (!req.getPageStartFrom_isNull())
-        pageStart = req.getPageStartFrom();
-    if (!req.getPageSize_isNull())
-        pageSize = req.getPageSize();
+    int pageSize = req.getPageSize_isNull()? 500 : req.getPageSize();
     Owned<IArchivedWUsReader> archiveWUsReader = new CArchivedWUsReader(context, sashaServerIP, sashaServerPort, archivedWuCache,
-        cacheMinutes, pageStart, pageSize);
+        cacheMinutes, pageSize);
     Owned<CWUQueryRequest> dummyReq = new CWUQueryRequest("WsWorkunits");
     IArrayOf<IEspECLWorkunit> dummyWUs;
     IArrayOf<IEspECLWorkunitLW> archivedWUs;

plugins/workunitservices/workunitservices.ipp

@@ -114,7 +114,7 @@ inline const char* readModifyTime(IPropertyTree& pt, StringBuffer& time, bool fo
     return time.str();
 }
 
-inline bool serializeWUSrow(IPropertyTree &pt,MemoryBuffer &mb, bool isonline)
+inline bool serializeWUSrow(IPropertyTree &pt, MemoryBuffer &mb, size32_t maxBufferLength, bool isonline)
 {
     mb.setEndian(__LITTLE_ENDIAN);
     fixedAppend(mb,24,pt.queryName());
@@ -152,7 +152,8 @@ inline bool serializeWUSrow(IPropertyTree &pt,MemoryBuffer &mb, bool isonline)
     mb.append(online);
     byte prot = pt.getPropBool("@protected")?1:0;
     mb.append(prot);
-    if (mb.length()>WORKUNIT_SERVICES_BUFFER_MAX) {
+    if (mb.length() > maxBufferLength)
+    {
         mb.clear().append(WUS_STATUS_OVERFLOWED);
         return false;
     }