HPCC-9595 Implement generalized paging/cache for DFU files

This fix replaces the existing paging/no cache function for
browsing logical files with new paging/cache function. The
MDFS_ITERATE_FILTEREDFILES flag and processing code are added to
support more filters on dali server side. In ESP side, a new method
getLogicalFilesSorted() and supporting code are added to set/send
the MDFS_ITERATE_FILTEREDFILES request with filters to dali server,
read the response into IPropertyTreeIterator for sorting, and sort/
cache the logical files. The old paging code is moved into a new
method which will be called for legacy dali.

3 minor changes: (1) Inside the response, replace file attribute
IsKeyFile to ContentType. (2) In dfu.xslt, rename the sort field
'Size' to 'FileSize'. (3) In dfu_search.xslt, merge the Sort Filter
FirstNType with FileSize.

Signed-off-by: Kevin Wang <kevin.wang@lexisnexis.com>

dali/base/dadfs.cpp

@@ -75,6 +75,7 @@ enum MDFSRequestKind
     MDFS_SET_FILE_ACCESSED,
     MDFS_ITERATE_RELATIONSHIPS,
     MDFS_SET_FILE_PROTECT,
+    MDFS_ITERATE_FILTEREDFILES,
     MDFS_MAX
 };
 
@@ -899,6 +900,8 @@ public:
 
     IDistributedFileIterator *getIterator(const char *wildname, bool includesuper,IUserDescriptor *user);
     IDFAttributesIterator *getDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive, bool includesuper,INode *foreigndali,unsigned foreigndalitimeout);
+    IPropertyTreeIterator *getDFAttributesTreeIterator(const char *filters, DFUQResultField* localFilters, const char *localFilterBuf,
+        IUserDescriptor *user, INode *foreigndali,unsigned foreigndalitimeout);
     IDFAttributesIterator *getForeignDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive=true, bool includesuper=false, const char *foreigndali="", unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT)
     {
         Owned<INode> foreign;
@@ -969,6 +972,8 @@ public:
     unsigned queryProtectedCount(const CDfsLogicalFileName &logicalname, const char *owner);                    
     bool getProtectedInfo(const CDfsLogicalFileName &logicalname, StringArray &names, UnsignedArray &counts);
     IDFProtectedIterator *lookupProtectedFiles(const char *owner=NULL,bool notsuper=false,bool superonly=false);
+    IDFAttributesIterator* getLogicalFilesSorted(IUserDescriptor* udesc, DFUQResultField *sortOrder, const void *filterBuf, DFUQResultField *specialFilters,
+            const void *specialFilterBuf, unsigned startOffset, unsigned maxNum, __int64 *cacheHint, unsigned *total);
 
     void setFileProtect(CDfsLogicalFileName &dlfn,IUserDescriptor *user, const char *owner, bool set, const INode *foreigndali=NULL,unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT);
 
@@ -1869,7 +1874,14 @@ public:
         }
         index = 0;
     }
-    
+
+    CDFAttributeIterator(IArrayOf<IPropertyTree>& trees)
+    {
+        ForEachItemIn(t, trees)
+            attrs.append(*LINK(&trees.item(t)));
+        index = 0;
+    }
+
     ~CDFAttributeIterator()
     {
         attrs.kill();
@@ -8021,8 +8033,6 @@ GetFileClusterNamesType CDistributedFileDirectory::getFileClusterNames(const cha
     return GFCN_NotFound;
 }
 
-
-
 // --------------------------------------------------------
 
 
@@ -8203,6 +8213,378 @@ public:
 typedef CIArrayOf<CScope> CScopeArray;
 
 
+const char* DFUQFilterFieldNames[] = { "", "@description", "@directory", "@group", "@modified", "@name", "@numclusters", "@numparts",
+    "@partmask", "@OrigName", "Attr", "Attr/@job", "Attr/@owner", "Attr/@recordCount", "Attr/@recordSize", "Attr/@size",
+    "Attr/@compressedsize", "Attr/@workunit", "Cluster", "Cluster/@defaultBaseDir", "Cluster/@defaultReplDir", "Cluster/@mapFlags",
+    "Cluster/@name", "Part", "Part/@name", "Part/@num", "Part/@size", "SuperOwner", "SuperOwner/@name",
+    "SubFile", "SubFile/@name", "SubFile/@num" };
+
+extern da_decl const char* getDFUQFilterFieldName(DFUQFilterField feild)
+{
+    return DFUQFilterFieldNames[feild];
+}
+
+class CDFUSFFilter : public CInterface
+{
+    DFUQFilterType filterType;
+    StringAttr attrPath;
+    bool hasFilter;
+    bool hasFilterHigh;
+    StringAttr filterValue;
+    StringAttr filterValueHigh;
+    int filterValueInt;
+    int filterValueHighInt;
+    __int64 filterValueInt64;
+    __int64 filterValueHighInt64;
+    bool filterValueBoolean;
+    StringAttr sep;
+    StringArray filterArray;
+
+public:
+    CDFUSFFilter(DFUQFilterType _filterType, const char *_attrPath, const char *_filterValue, const char *_filterValueHigh)
+        : filterType(_filterType), attrPath(_attrPath), filterValue(_filterValue), filterValueHigh(_filterValueHigh) {};
+    CDFUSFFilter(DFUQFilterType _filterType, const char *_attrPath, bool _hasFilter, const int _filterValue, bool _hasFilterHigh, const int _filterValueHigh)
+        : filterType(_filterType), attrPath(_attrPath), hasFilter(_hasFilter), filterValueInt(_filterValue), hasFilterHigh(_hasFilterHigh), filterValueHighInt(_filterValueHigh) {};
+    CDFUSFFilter(DFUQFilterType _filterType, const char *_attrPath, bool _hasFilter, const __int64 _filterValue, bool _hasFilterHigh, const __int64 _filterValueHigh)
+        : filterType(_filterType), attrPath(_attrPath), hasFilter(_hasFilter), filterValueInt64(_filterValue), hasFilterHigh(_hasFilterHigh), filterValueHighInt64(_filterValueHigh) {};
+    CDFUSFFilter(DFUQFilterType _filterType, const char *_attrPath, bool _filterValue)
+        : filterType(_filterType), attrPath(_attrPath), filterValueBoolean(_filterValue) {};
+    CDFUSFFilter(DFUQFilterType _filterType, const char *_attrPath, const char *_filterValue, const char *_sep, StringArray& _filterArray)
+        : filterType(_filterType), attrPath(_attrPath), filterValue(_filterValue), sep(_sep)
+    {
+        ForEachItemIn(i,_filterArray)
+        {
+            const char* filter = _filterArray.item(i);
+            if (filter && *filter)
+                filterArray.append(filter);
+        }
+    };
+
+    DFUQFilterType getFilterType() { return filterType;}
+    const char * getAttrPath() { return attrPath.get();}
+    const char * getFilterValue() { return filterValue.get();}
+    const char * getFilterValueHigh() { return filterValueHigh.get();}
+    const int getFilterValueInt() { return filterValueInt;}
+    const int getFilterValueHighInt() { return filterValueHighInt;}
+    const __int64 getFilterValueInt64() { return filterValueInt64;}
+    const __int64 getFilterValueHighInt64() { return filterValueHighInt64;}
+    const bool getFilterValueBoolean() { return filterValueBoolean;}
+    const char * getSep() { return sep.get();}
+    void getFilterArray(StringArray &filters)
+    {
+        ForEachItemIn(c, filterArray)
+            filters.append(filterArray.item(c));
+    }
+
+    bool checkFilter(IPropertyTree &file)
+    {
+        bool match = true;
+        switch(filterType)
+        {
+        case DFUQFTwildcardMatch:
+            match = doWildMatch(file);
+            break;
+        case DFUQFTbooleanMatch:
+            match = doBooleanMatch(file);
+            break;
+        case DFUQFThasProp:
+            match = checkHasPropFilter(file);
+            break;
+        case DFUQFTcontainString:
+            match = checkContainStringFilter(file);
+            break;
+        case DFUQFTstringRange:
+            match = checkStringRangeFilter(file);
+            break;
+        case DFUQFTintegerRange:
+            match = checkIntegerRangeFilter(file);
+            break;
+        case DFUQFTinteger64Range:
+            match = checkInteger64RangeFilter(file);
+            break;
+        }
+        return match;
+    }
+    bool doWildMatch(IPropertyTree &file)
+    {
+        const char* filter = filterValue.get();
+        if (!attrPath.get() || !filter || !*filter || streq(filter, "*"))
+            return true;
+
+        const char* prop = file.queryProp(attrPath.get());
+        if (prop && WildMatch(prop, filter, true))
+            return true;
+        return false;
+    }
+    bool doBooleanMatch(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+
+        return filterValueBoolean == file.getPropBool(attrPath.get(), true);
+    }
+    bool checkHasPropFilter(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+
+        return filterValueBoolean == file.hasProp(attrPath.get());
+    }
+    bool checkContainStringFilter(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+        const char* prop = file.queryProp(attrPath.get());
+        if (!prop || !*prop)
+            return false;
+
+        bool found = false;
+        if (!sep.get())
+        {
+            if (filterArray.find(prop) != NotFound) //Match with one of values in the filter
+               found = true;
+            return found;
+        }
+        StringArray propArray;
+        propArray.appendListUniq(prop, sep.get());
+        ForEachItemIn(i,propArray)
+        {
+            const char* value = propArray.item(i);
+            if (!value || !*value)
+                continue;
+            if (filterArray.find(value) != NotFound) //Match with one of values in the filter
+            {
+                found = true;
+                break;
+            }
+        }
+        return found;
+    }
+    bool checkStringRangeFilter(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+        const char* prop = file.queryProp(attrPath.get());
+        if (!prop || !*prop)
+            return false;
+        if (filterValue && (strcmp(filterValue, prop) > 0))
+            return false;
+        if (filterValueHigh && (strcmp(filterValueHigh, prop) < 0))
+            return false;
+        return true;
+    }
+    bool checkIntegerRangeFilter(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+        int prop = file.getPropInt(attrPath.get());
+        if (hasFilter && (prop < filterValueInt))
+            return false;
+        if (hasFilterHigh && (prop > filterValueHighInt))
+            return false;
+        return true;
+    }
+    bool checkInteger64RangeFilter(IPropertyTree &file)
+    {
+        if (!attrPath.get())
+            return true;
+        __int64 prop = file.getPropInt64(attrPath.get());
+        if (hasFilter && (prop < filterValueInt64))
+            return false;
+        if (hasFilterHigh && (prop > filterValueHighInt64))
+            return false;
+        return true;
+    }
+};
+typedef CIArrayOf<CDFUSFFilter> CDFUSFFilterArray;
+
+class CFileScanFilterContainer : public CInterface
+{
+    StringAttr filterBuf; //Hold original filter string just in case
+    StringAttr wildNameFilter;
+    DFUQFileTypeFilter fileTypeFilter;
+    CIArrayOf<CDFUSFFilter> filters;
+    //The 'filters' contains the file scan filters other than wildNameFilter and fileTypeFilter. Those filters are used for
+    //filtering the files using File Attributes tree and CDFUSFFilter::checkFilter(). The wildNameFilter and fileTypeFilter need
+    //special code to filter the files.
+
+    bool isValidInteger(const char *s)
+    {
+        if (!s)
+            return false;
+        while (*s)
+        {
+            if ((*s != '-') && !isdigit(*s))
+                return false;
+            s++;
+        }
+        return true;
+    }
+    void addFilter(DFUQFilterType filterType, const char* attr, const char* value, const char* valueHigh)
+    {
+        if (!attr || !*attr)
+            return;
+        if ((DFUQFTwildcardMatch == filterType) || (DFUQFTstringRange == filterType))
+        {
+            filters.append(*new CDFUSFFilter(filterType, attr, value, valueHigh));
+            return;
+        }
+        if ((DFUQFTbooleanMatch == filterType) || (DFUQFThasProp == filterType))
+        {
+            bool filter = true;
+            if (value && (streq(value, "0") || strieq(value, "false")))
+                filter = false;
+            filters.append(*new CDFUSFFilter(filterType, attr, filter));
+            return;
+        }
+        if ((DFUQFTintegerRange == filterType) || (DFUQFTinteger64Range == filterType))
+        {
+            bool hasFilter = false;
+            bool hasFilterHigh = false;
+            if (value && isValidInteger(value))
+                hasFilter = true;
+            if (valueHigh && isValidInteger(valueHigh))
+                hasFilterHigh = true;
+            if (!hasFilter && !hasFilterHigh)
+                return;
+            if (DFUQFTintegerRange == filterType)
+                filters.append(*new CDFUSFFilter(filterType, attr, hasFilter, atoi(value), hasFilterHigh, atoi(valueHigh)));
+            else
+                filters.append(*new CDFUSFFilter(filterType, attr, hasFilter, (__int64) atol(value), hasFilterHigh, (__int64) atol(valueHigh)));
+            return;
+        }
+    }
+    void addFilterArray(DFUQFilterType filterType, const char* attr, const char* value, const char* sep)
+    {
+        if (!attr || !*attr || !value || !*value)
+            return;
+
+        StringArray filterArray;
+        filterArray.appendListUniq(value, sep);
+        filters.append(*new CDFUSFFilter(filterType, attr, value, sep, filterArray));
+    }
+    void addSpecialFilter(const char* attr, const char* value)
+    {
+        if (!attr || !*attr || !value || !*value)
+            return;
+        if (!isdigit(*attr))
+        {
+            PROGLOG("Unsupported Speical Filter: %s", attr);
+            return;
+        }
+        DFUQSpecialFilter filterName = (DFUQSpecialFilter) atoi(attr);
+        switch(filterName)
+        {
+        case DFUQSFFileNameWithPrefix:
+            wildNameFilter.set(value);
+            break;
+        case DFUQSFFileType:
+            if (isdigit(*value))
+                fileTypeFilter = (DFUQFileTypeFilter) atoi(value);
+            else
+                PROGLOG("Unsupported Speical Filter: %s, value %s", attr, value);
+            break;
+        default:
+            PROGLOG("Unsupported Speical Filter: %d", filterName);
+            break;
+        }
+    }
+
+    bool doWildMatch(const char* filter, const char* value)
+    {
+        if (!filter || !*filter || streq(filter, "*") || (value && WildMatch(value, filter, true)))
+            return true;
+
+        return false;
+    }
+
+public:
+    CFileScanFilterContainer()
+    {
+        fileTypeFilter = DFUQFFTall;
+        wildNameFilter.set("*");
+        filterBuf.clear();
+    };
+    void readScanFilters(const char *filterStr)
+    {
+        if (!filterStr || !*filterStr)
+            return;
+
+        filterBuf.set(filterStr);
+        StringArray filterStringArray;
+        char sep[] = { DFUQFilterSeparator, '\0' };
+        filterStringArray.appendList(filterStr, sep);
+
+        unsigned filterFieldsToRead = filterStringArray.length();
+        ForEachItemIn(i,filterStringArray)
+        {
+            const char* filterTypeStr = filterStringArray.item(i);
+            if (!filterTypeStr || !*filterTypeStr)
+                continue;
+            if (!isdigit(*filterTypeStr))
+                continue;
+            unsigned filterSize = 4;
+            DFUQFilterType filterType = (DFUQFilterType) atoi(filterTypeStr);
+            switch(filterType)
+            {
+            case DFUQFTcontainString:
+                if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | separator | filter value separated by the separator
+                    addFilterArray(DFUQFTcontainString, filterStringArray.item(i+1), (const char*)filterStringArray.item(i+2), (const char*)filterStringArray.item(i+3));
+                break;
+            case DFUQFThasProp:
+            case DFUQFTbooleanMatch:
+            case DFUQFTwildcardMatch:
+                filterSize = 3;
+                if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | filter value
+                    addFilter(filterType, filterStringArray.item(i+1), (const char*)filterStringArray.item(i+2), NULL);
+                break;
+            case DFUQFTstringRange:
+            case DFUQFTintegerRange:
+            case DFUQFTinteger64Range:
+                if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | from filter | to filter
+                    addFilter(filterType, filterStringArray.item(i+1), (const char*)filterStringArray.item(i+2), (const char*)filterStringArray.item(i+3));
+                break;
+            case DFUQFTspecial:
+                filterSize = 3;
+                if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | filter value
+                    addSpecialFilter(filterStringArray.item(i+1), (const char*)filterStringArray.item(i+2));
+                break;
+            }
+            filterFieldsToRead -= filterSize;
+            i += (filterSize - 1);
+        }
+    }
+    bool matchFileScanFilter(const char* name, IPropertyTree &file)
+    {
+        if (!doWildMatch(wildNameFilter.get(), name))
+            return false;
+
+        if (!filters.length())
+            return true;
+        ForEachItemIn(i,filters)
+        {
+            CDFUSFFilter &filter = filters.item(i);
+            bool match = filter.checkFilter(file);
+            if (!match)
+                return match;
+        }
+        return true;
+    }
+
+    DFUQFileTypeFilter getFileTypeFilter() { return fileTypeFilter; }
+    void setFileTypeFilter(DFUQFileTypeFilter _fileType)
+    {
+        fileTypeFilter = _fileType;
+    }
+    const char* getNameFilter() { return wildNameFilter.get(); }
+    void setNameFilter(const char* _wildName)
+    {
+        if (!_wildName || !*_wildName)
+            return;
+        wildNameFilter.set(_wildName);
+    }
+};
+
 class CFileScanner
 {
     bool recursive;
@@ -8210,14 +8592,20 @@ class CFileScanner
     StringAttr wildname;
     Owned<CScope> topLevelScope;
     CScope *currentScope;
+    bool fileScanWithFilter;
+    CFileScanFilterContainer fileScanFilterContainer;
 
     bool scopeMatch(const char *name)
     {   // name has trailing '::'
-        if (!*name)
+        if (!name || !*name)
             return true;
-        if (wildname.isEmpty())
+        const char *s1 = NULL;
+        if (!fileScanWithFilter)
+            s1 = wildname.get();
+        else
+            s1 = fileScanFilterContainer.getNameFilter();
+        if (!s1 || !*s1)
             return true;
-        const char *s1 = wildname.get();
         const char *s2 = name;
         while (*s2) {
             if (*s1=='*') {
@@ -8269,7 +8657,10 @@ class CFileScanner
                     }
                 } while (iter->next());
             }
-            ret |= processFiles(root,name);
+            if (!fileScanWithFilter)
+                ret |= processFiles(root,name);
+            else
+                ret |= processFilesWithFilters(root,name);
         }
         if (!ret && parentScope)
             parentScope->popLastScope(); // discard scopes where no matches
@@ -8313,9 +8704,37 @@ class CFileScanner
         return ret;
     }
 
+    bool processFilesWithFilters(IPropertyTree &root, StringBuffer &name)
+    {
+        bool ret = false;
+        size32_t ns = name.length();
+        DFUQFileTypeFilter fileTypeFilter = fileScanFilterContainer.getFileTypeFilter();
+        if (fileTypeFilter != DFUQFFTsuperfileonly)
+            addMatchedFiles(root.getElements(queryDfsXmlBranchName(DXB_File)), false, name, ns, ret);
+        if ((fileTypeFilter == DFUQFFTall) || (fileTypeFilter == DFUQFFTsuperfileonly))
+            addMatchedFiles(root.getElements(queryDfsXmlBranchName(DXB_SuperFile)), true, name, ns, ret);
+        return ret;
+    }
+
+    void addMatchedFiles(IPropertyTreeIterator* files, bool isSuper, StringBuffer &name, size32_t ns, bool& ret)
+    {
+        Owned<IPropertyTreeIterator> iter = files;
+        ForEach(*iter)
+        {
+            IPropertyTree &file = iter->query();
+            name.append(file.queryProp("@name"));
+            if (fileScanFilterContainer.matchFileScanFilter(name.str(), file))
+            {
+                currentScope->addMatch(name,file,isSuper);
+                ret = true;
+            }
+            name.setLength(ns);
+        }
+    }
 public:
     void scan(IPropertyTree *sroot, const char *_wildname,bool _recursive,bool _includesuper)
     {
+        fileScanWithFilter =  false;
         if (_wildname)
             wildname.set(_wildname);
         else
@@ -8327,6 +8746,17 @@ public:
         currentScope = NULL;
         processScopes(*sroot->queryPropTree(querySdsFilesRoot()),name);
     }
+    void scan(IPropertyTree *sroot, const char *filters, bool _recursive)
+    {
+        fileScanWithFilter =  true;
+        recursive = _recursive;
+        fileScanFilterContainer.readScanFilters(filters);
+
+        StringBuffer name;
+        topLevelScope.clear();
+        currentScope = NULL;
+        processScopes(*sroot->queryPropTree(querySdsFilesRoot()),name);
+    }
     void _getResults(bool auth, IUserDescriptor *user, CScope &scope, CFileMatchArray &matchingFiles, StringArray &authScopes, unsigned &count)
     {
         if (auth)
@@ -9050,6 +9480,59 @@ public:
         mb.writeDirect(0,sizeof(count),&count);
     }
 
+    void iterateFilteredFiles(CMessageBuffer &mb,StringBuffer &trc)
+    {
+        TransactionLog transactionLog(*this, MDFS_ITERATE_FILTEREDFILES, mb.getSender());
+
+        Owned<IUserDescriptor> udesc;
+        StringAttr filters;
+        bool recursive;
+        mb.read(filters).read(recursive);
+        trc.appendf("iterateFilteredFiles(%s,%s)",filters.sget(),recursive?"recursive":"");
+        if (queryTransactionLogging())
+            transactionLog.log("%s", trc.str());
+        if (mb.getPos()<mb.length())
+        {
+            udesc.setown(createUserDescriptor());
+            udesc->deserialize(mb);
+        }
+
+        mb.clear();
+        unsigned count=0;
+        mb.append(count);
+
+        CFileScanner scanner;
+        CSDSServerLockBlock sdsLock; // lock sds while scanning
+        unsigned start = msTick();
+        scanner.scan(sdsLock, filters.get(), recursive);
+        unsigned tookMs = msTick()-start;
+        if (tookMs>100)
+            PROGLOG("TIMING(filescan): %s: took %dms",trc.str(), tookMs);
+        sdsLock.unlock(); // unlock to perform authentification
+
+        bool auth = querySessionManager().checkScopeScansLDAP()&&getScopePermissions(NULL,udesc,(unsigned)-1);
+        StringArray authScopes;
+        CIArrayOf<CFileMatch> matchingFiles;
+        start = msTick();
+        count = scanner.getResults(auth, udesc, matchingFiles, authScopes);
+        tookMs = msTick()-start;
+        if (tookMs>100)
+            PROGLOG("TIMING(LDAP): %s: took %dms, %d lookups, file matches = %d", trc.str(), tookMs, authScopes.ordinality(), count);
+
+        sdsLock.lock(); // re-lock sds while serializing
+        start = msTick();
+        ForEachItemIn(m, matchingFiles)
+        {
+            CFileMatch &fileMatch = matchingFiles.item(m);
+            CDFAttributeIterator::serializeFileAttributes(mb, fileMatch.queryFileTree(), fileMatch.queryName(), fileMatch.queryIsSuper());
+        }
+        tookMs = msTick()-start;
+        if (tookMs>100)
+            PROGLOG("TIMING(filescan-serialization): %s: took %dms, %d files",trc.str(), tookMs, count);
+
+        mb.writeDirect(0,sizeof(count),&count);
+    }
+
     void iterateRelationships(CMessageBuffer &mb,StringBuffer &trc)
     {
         TransactionLog transactionLog(*this, MDFS_ITERATE_RELATIONSHIPS, mb.getSender());
@@ -9274,6 +9757,10 @@ public:
                     iterateFiles(mb,trc);                    
                 }
                 break;
+            case MDFS_ITERATE_FILTEREDFILES: {
+                    iterateFilteredFiles(mb,trc);
+                }
+                break;
             case MDFS_ITERATE_RELATIONSHIPS: {
                     iterateRelationships(mb,trc);                    
                 }
@@ -9325,6 +9812,8 @@ public:
         {
         case MDFS_ITERATE_FILES:
             return ret.append("MDFS_ITERATE_FILES");
+        case MDFS_ITERATE_FILTEREDFILES:
+            return ret.append("MDFS_ITERATE_FILTEREDFILES");
         case MDFS_ITERATE_RELATIONSHIPS:
             return ret.append("MDFS_ITERATE_RELATIONSHIPS");
         case MDFS_GET_FILE_TREE:
@@ -11197,6 +11686,261 @@ IDFProtectedIterator *CDistributedFileDirectory::lookupProtectedFiles(const char
     return new CDFProtectedIterator(owner,notsuper,superonly,defaultTimeout);
 }
 
+const char* DFUQResultFieldNames[] = { "@name", "@description", "@group", "@kind", "@modified", "@job", "@owner",
+    "@DFUSFrecordCount", "@recordCount", "@recordSize", "@DFUSFsize", "@size", "@workunit", "@DFUSFcluster", "@numsubfiles",
+    "@accessed", "@numparts", "@compressedSize", "@directory", "@partmask" };
+
+extern da_decl const char* getDFUQResultFieldName(DFUQResultField feild)
+{
+    return DFUQResultFieldNames[feild];
+}
+
+IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, DFUQResultField* localFilters, const char* localFilterBuf)
+{
+    class CFileAttrIterator: public CInterface, implements IPropertyTreeIterator
+    {
+        Owned<IPropertyTree> cur;
+        StringArray fileClusterGroups;
+
+        void setFileCluster(IPropertyTree *attr, const char* group, StringArray& clusterFilter)
+        {
+            if (!group || !*group)
+                return;
+
+            //The group may contain multiple clusters and some of them may match with the clusterFilter.
+            if (clusterFilter.length() == 1)
+                attr->setProp(getDFUQResultFieldName(DFUQRFcluster), clusterFilter.item(0));//Filter has been handled on server side.
+            else
+            {
+                StringArray clusters;
+                clusters.appendListUniq(group, ",");
+                ForEachItemIn(i,clusters)
+                {
+                    //Add a cluster if no cluster filter or the cluster matchs with cluster filter
+                    const char* cluster = clusters.item(i);
+                    if (cluster && *cluster && ((!clusterFilter.length()) || (clusterFilter.find(cluster) != NotFound)))
+                        fileClusterGroups.append(cluster);
+                }
+                if (fileClusterGroups.length())
+                {
+                    //if this file exists on multiple clusters, set one of the clusters as the "@DFUSFcluster" prop for
+                    //this attr, leaving the rest inside the fileClusterGroups array. Those clusters will be used by the
+                    //duplicateFileAttrOnOtherClusterGroup() to duplicate this file attr on other clusters.
+                    attr->setProp(getDFUQResultFieldName(DFUQRFcluster), fileClusterGroups.item(fileClusterGroups.length() -1));
+                    fileClusterGroups.pop();
+                }
+            }
+        }
+
+        void setRecordCount(IPropertyTree* file)
+        {
+            __int64 recordCount = 0;
+            if (file->hasProp(getDFUQResultFieldName(DFUQRForigrecordcount)))
+                recordCount = file->getPropInt64(getDFUQResultFieldName(DFUQRForigrecordcount));
+            else
+            {
+                __int64 recordSize=file->getPropInt64(getDFUQResultFieldName(DFUQRFrecordsize),0);
+                if(recordSize)
+                {
+                    __int64 size=file->getPropInt64(getDFUQResultFieldName(DFUQRForigsize),-1);
+                    recordCount = size/recordSize;
+                }
+            }
+            file->setPropInt64(getDFUQResultFieldName(DFUQRFrecordcount),recordCount);
+            return;
+        }
+
+        IPropertyTree *deserializeFileAttr(MemoryBuffer &mb, StringArray& clusterFilter)
+        {
+            IPropertyTree *attr = getEmptyAttr();
+            StringAttr val;
+            unsigned n;
+            mb.read(val);
+            attr->setProp(getDFUQResultFieldName(DFUQRFname),val.get());
+            mb.read(val);
+            if (strieq(val,"!SF"))
+            {
+                mb.read(n);
+                attr->setPropInt(getDFUQResultFieldName(DFUQRFnumsubfiles),n);
+                mb.read(val);   // not used currently
+            }
+            else
+            {
+                attr->setProp(getDFUQResultFieldName(DFUQRFdirectory),val.get());
+                mb.read(n);
+                attr->setPropInt(getDFUQResultFieldName(DFUQRFnumparts),n);
+                mb.read(val);
+                attr->setProp(getDFUQResultFieldName(DFUQRFpartmask),val.get());
+            }
+            mb.read(val);
+            attr->setProp(getDFUQResultFieldName(DFUQRFtimemodified),val.get());
+            unsigned count;
+            mb.read(count);
+            StringAttr at;
+            while (count--)
+            {
+                mb.read(at);
+                mb.read(val);
+                attr->setProp(at.get(),val.get());
+                if (strieq(at.get(), getDFUQResultFieldName(DFUQRFgroup)))
+                    setFileCluster(attr, val.get(), clusterFilter);
+            }
+            attr->setPropInt64(getDFUQResultFieldName(DFUQRFsize), attr->getPropInt64(getDFUQResultFieldName(DFUQRForigsize), -1));//Sort the files with empty size to front
+            setRecordCount(attr);
+            return attr;
+        }
+
+        IPropertyTree *duplicateFileAttrOnOtherClusterGroup(IPropertyTree *previousAttr)
+        {
+            IPropertyTree *attr = getEmptyAttr();
+            Owned<IAttributeIterator> ai = previousAttr->getAttributes();
+            ForEach(*ai)
+                attr->setProp(ai->queryName(),ai->queryValue());
+            attr->setProp(getDFUQResultFieldName(DFUQRFcluster), fileClusterGroups.item(fileClusterGroups.length()-1));
+            fileClusterGroups.pop();
+            return attr;
+        }
+
+    public:
+        IMPLEMENT_IINTERFACE;
+        MemoryBuffer mb;
+        unsigned numfiles;
+        StringArray clusterFilter;
+
+        bool first()
+        {
+            mb.reset();
+            mb.read(numfiles);
+
+            return next();
+        }
+
+        bool next()
+        {
+            if (fileClusterGroups.length())
+            {
+                IPropertyTree *attr = duplicateFileAttrOnOtherClusterGroup(cur);
+                cur.clear();
+                cur.setown(attr);
+                return true;
+            }
+            cur.clear();
+            if (mb.getPos()>=mb.length())
+                return false;
+            cur.setown(deserializeFileAttr(mb, clusterFilter));
+            return true;
+        }
+
+        bool isValid()
+        {
+            return cur.get()!=NULL;
+        }
+
+        IPropertyTree  & query()
+        {
+            return *cur;
+        }
+
+        void setLocalFilters(DFUQResultField* localFilters, const char* localFilterBuf)
+        {
+            if (!localFilters || !localFilterBuf || !*localFilterBuf)
+                return;
+
+            const char *fv = localFilterBuf;
+            for (unsigned i=0;localFilters[i]!=DFUQRFterm;i++)
+            {
+                int fmt = localFilters[i];
+                int subfmt = (fmt&0xff);
+                if ((subfmt==DFUQRFcluster) && fv && *fv)
+                    clusterFilter.appendListUniq(fv, ",");
+                //Add more if needed
+                fv = fv + strlen(fv)+1;
+            }
+        }
+
+    } *fai = new CFileAttrIterator;
+    mb.swapWith(fai->mb);
+    fai->setLocalFilters(localFilters, localFilterBuf);
+    return fai;
+}
+
+IPropertyTreeIterator *CDistributedFileDirectory::getDFAttributesTreeIterator(const char* filters, DFUQResultField* localFilters,
+    const char* localFilterBuf, IUserDescriptor* user, INode* foreigndali, unsigned foreigndalitimeout)
+{
+    CMessageBuffer mb;
+    mb.append((int)MDFS_ITERATE_FILTEREDFILES).append(filters).append(true);
+    if (user)
+        user->serialize(mb);
+
+    if (foreigndali)
+        foreignDaliSendRecv(foreigndali,mb,foreigndalitimeout);
+    else
+        queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DFS_REQUEST);
+    checkDfsReplyException(mb);
+
+    return deserializeFileAttrIterator(mb, localFilters, localFilterBuf);
+}
+
+IDFAttributesIterator* CDistributedFileDirectory::getLogicalFilesSorted(
+    IUserDescriptor* udesc,
+    DFUQResultField *sortOrder, // list of fields to sort by (terminated by DFUSFterm)
+    const void *filters,  // (appended) string values for filters used by dali server
+    DFUQResultField *localFilters, //used for filtering query result received from dali server.
+    const void *localFilterBuf,
+    unsigned startOffset,
+    unsigned maxNum,
+    __int64 *cacheHint,
+    unsigned *total)
+{
+    class CDFUPager : public CSimpleInterface, implements IElementsPager
+    {
+        IUserDescriptor* udesc;
+        //StringAttr clusterFilter;
+        StringAttr filters;
+        DFUQResultField *localFilters;
+        StringAttr localFilterBuf;
+        StringAttr sortOrder;
+
+    public:
+        IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+        CDFUPager(IUserDescriptor* _udesc, const char*_filters, DFUQResultField*_localFilters, const char*_localFilterBuf,
+            const char*_sortOrder) : udesc(_udesc), filters(_filters), localFilters(_localFilters), localFilterBuf(_localFilterBuf),
+            sortOrder(_sortOrder)
+        {
+        }
+        virtual IRemoteConnection* getElements(IArrayOf<IPropertyTree> &elements)
+        {
+            Owned<IPropertyTreeIterator> fi = queryDistributedFileDirectory().getDFAttributesTreeIterator(filters.get(),
+                localFilters, localFilterBuf.get(), udesc);
+            sortElements(fi, sortOrder.get(), NULL, NULL, elements);
+            return NULL;
+        }
+    };
+
+    StringBuffer so;
+    if (sortOrder)
+    {
+        for (unsigned i=0;sortOrder[i]!=DFUQRFterm;i++)
+        {
+            if (so.length())
+                so.append(',');
+            int fmt = sortOrder[i];
+            if (fmt&DFUQRFreverse)
+                so.append('-');
+            if (fmt&DFUQRFnocase)
+                so.append('?');
+            if (fmt&DFUQRFnumeric)
+                so.append('#');
+            so.append(getDFUQResultFieldName((DFUQResultField) (fmt&0xff)));
+        }
+    }
+    IArrayOf<IPropertyTree> results;
+    Owned<IElementsPager> elementsPager = new CDFUPager(udesc, (const char*) filters, localFilters, (const char*) localFilterBuf,
+        so.length()?so.str():NULL );
+    getElementsPaged(elementsPager,startOffset,maxNum,NULL,"",cacheHint,results,total,false);
+    return new CDFAttributeIterator(results);
+}
 #ifdef _USE_CPPUNIT
 /*
  * This method removes files only logically. removeEntry() used to do that, but the only
@@ -11211,5 +11955,5 @@ extern da_decl void removeLogical(const char *fname, IUserDescriptor *user) {
         f->detachLogical();
     }
 }
-
+}
 #endif // _USE_CPPUNIT

dali/base/dadfs.hpp

@@ -168,6 +168,105 @@ typedef IIteratorOf<IDistributedFilePart> IDistributedFilePartIterator;
 
 class CDFAction ;
 
+#define DFUQFilterSeparator '|' // | is used as a separator because it is not a valid character for logical file name
+
+enum DFUQFilterType
+{
+    DFUQFTwildcardMatch,
+    DFUQFThasProp,
+    DFUQFTcontainString,
+    DFUQFTbooleanMatch,
+    DFUQFTstringRange,
+    DFUQFTintegerRange,
+    DFUQFTinteger64Range,
+    DFUQFTspecial
+};
+
+enum DFUQSpecialFilter
+{
+    DFUQSFFileNameWithPrefix = 1,
+    DFUQSFFileType = 2
+};
+
+enum DFUQFileTypeFilter
+{
+    DFUQFFTall = 1,
+    DFUQFFTsuperfileonly = 2,
+    DFUQFFTnonsuperfileonly = 3
+};
+
+enum DFUQFilterField
+{
+    DFUQFFfiletype = 0,
+    DFUQFFdescription = 1,
+    DFUQFFdirectory = 2,
+    DFUQFFgroup = 3,
+    DFUQFFtimemodified = 4,
+    DFUQFFname = 5,
+    DFUQFFnumclusters = 6,
+    DFUQFFnumparts = 7,
+    DFUQFFpartmask = 8,
+    DFUQFForigname = 9,
+    DFUQFFattr = 10,
+    DFUQFFattrjob = 11,
+    DFUQFFattrowner = 12,
+    DFUQFFattrrecordcount = 13,
+    DFUQFFattrrecordsize = 14,
+    DFUQFFattrsize = 15,
+    DFUQFFattrcompressedsize = 16,
+    DFUQFFattrworkunit = 17,
+    DFUQFFcluster = 18,
+    DFUQFFclusterdefaultbasedir = 19,
+    DFUQFFclusterdefaultrepldir = 20,
+    DFUQFFclustermapflags = 21,
+    DFUQFFclustername = 22,
+    DFUQFFpart = 23,
+    DFUQFFpartname = 24,
+    DFUQFFpartnum = 25,
+    DFUQFFpartsize = 26,
+    DFUQFFsuperowner = 27,
+    DFUQFFsuperownername = 28,
+    DFUQFFsubfile = 29,
+    DFUQFFsubfilename = 30,
+    DFUQFFsubfilenum = 31,
+    DFUQFFterm = 32,
+    DFUQFFreverse = 256,
+    DFUQFFnocase = 512,
+    DFUQFFnumeric = 1024,
+    DFUQFFwild = 2048
+};
+
+enum DFUQResultField
+{
+    DFUQRFname = 0,
+    DFUQRFdescription = 1,
+    DFUQRFgroup = 2,
+    DFUQRFkind = 3,
+    DFUQRFtimemodified = 4,
+    DFUQRFjob = 5,
+    DFUQRFowner = 6,
+    DFUQRFrecordcount = 7,
+    DFUQRForigrecordcount = 8,
+    DFUQRFrecordsize = 9,
+    DFUQRFsize = 10,
+    DFUQRForigsize = 11,
+    DFUQRFworkunit = 12,
+    DFUQRFcluster = 13,
+    DFUQRFnumsubfiles = 14,
+    DFUQRFaccessed = 15,
+    DFUQRFnumparts = 16,
+    DFUQRFcompressedsize = 17,
+    DFUQRFdirectory = 18,
+    DFUQRFpartmask = 19,
+    DFUQRFterm = 20,
+    DFUQRFreverse = 256,
+    DFUQRFnocase = 512,
+    DFUQRFnumeric = 1024
+};
+
+extern da_decl const char* getDFUQFilterFieldName(DFUQFilterField feild);
+extern da_decl const char* getDFUQResultFieldName(DFUQResultField feild);
+
 /**
  * File operations can be included in a transaction to ensure that multiple
  * updates are handled atomically. This is the interface to a transaction
@@ -269,7 +368,7 @@ interface IDistributedFile: extends IInterface
 
     virtual bool getFormatCrc(unsigned &crc) =0;   // CRC for record format 
     virtual bool getRecordSize(size32_t &rsz) =0;   
-    virtual bool getRecordLayout(MemoryBuffer &layout) =0;   
+    virtual bool getRecordLayout(MemoryBuffer &layout) =0;
 
 
     virtual void enqueueReplicate()=0;
@@ -448,6 +547,8 @@ interface IDistributedFileDirectory: extends IInterface
     virtual IDistributedFileIterator *getIterator(const char *wildname, bool includesuper, IUserDescriptor *user) = 0;
             // wildname is in form scope/name and may contain wild components for either
     virtual IDFAttributesIterator *getDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive=true, bool includesuper=false, INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
+    virtual IPropertyTreeIterator *getDFAttributesTreeIterator(const char *filters, DFUQResultField* localFilters,
+        const char *localFilterBuf, IUserDescriptor *user, INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
     virtual IDFAttributesIterator *getForeignDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive=true, bool includesuper=false, const char *foreigndali="", unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
 
     virtual IDFScopeIterator *getScopeIterator(IUserDescriptor *user, const char *subscope=NULL,bool recursive=true,bool includeempty=false)=0;
@@ -551,6 +652,8 @@ interface IDistributedFileDirectory: extends IInterface
                                            StringArray &names, UnsignedArray &counts) = 0;
 
     virtual IDFProtectedIterator *lookupProtectedFiles(const char *owner=NULL,bool notsuper=false,bool superonly=false)=0; // if owner = NULL then all
+    virtual IDFAttributesIterator* getLogicalFilesSorted(IUserDescriptor* udesc, DFUQResultField *sortOrder, const void* filters, DFUQResultField *localFilters,
+            const void *specialFilterBuf, unsigned startOffset, unsigned maxNum, __int64 *cacheHint, unsigned *total) = 0;
 
     virtual unsigned setDefaultTimeout(unsigned timems) = 0;                                // sets default timeout for SDS connections and locking
                                                                                             // returns previous value

dali/base/dautils.cpp

@@ -1678,9 +1678,19 @@ public:
                 char *&v2 = vals[i2*nk+i];
                 if (!v2)
                     getkeyval(i2,i,v2);
+                if (!v1 || !v2)
+                    return 0;
                 int ret;
                 if (mod&SORT_NUMERIC)
-                    ret = (int)(_atoi64(v1)-_atoi64(v2));
+                {
+                    __int64 ret0 = _atoi64(v1)-_atoi64(v2);
+                    if (ret0 > 0)
+                        ret = 1;
+                    else if (ret0 < 0)
+                        ret = -1;
+                    else
+                        ret = 0;
+                }
                 else if (mod&SORT_NOCASE)
                     ret = stricmp(v1,v2);
                 else
@@ -1915,7 +1925,8 @@ IRemoteConnection *getElementsPaged( IElementsPager *elementsPager,
                                      const char *owner,
                                      __int64 *hint,
                                      IArrayOf<IPropertyTree> &results,
-                                     unsigned *total)
+                                     unsigned *total,
+                                     bool checkConn)
 {
     if ((pagesize==0) || !elementsPager)
         return NULL;
@@ -1932,11 +1943,12 @@ IRemoteConnection *getElementsPaged( IElementsPager *elementsPager,
         elem.setown(QUERYINTERFACE(pagedElementsCache->get(owner,*hint),CPECacheElem)); // NB: removes from cache in process, added back at end
         postfilter = elem->postFilter; // reuse cached postfilter
     }
-    if (!elem)
+    else
+    {
         elem.setown(new CPECacheElem(owner, postfilter));
-    if (!elem->conn)
         elem->conn.setown(elementsPager->getElements(elem->totalres));
-    if (!elem->conn)
+    }
+    if (checkConn && !elem->conn)
         return NULL;
     unsigned n;
     if (total)
@@ -1986,7 +1998,9 @@ IRemoteConnection *getElementsPaged( IElementsPager *elementsPager,
             results.append(item);
         }
     }
-    IRemoteConnection *ret = elem->conn.getLink();
+    IRemoteConnection *ret = NULL;
+    if (elem->conn)
+        ret = elem->conn.getLink();
     if (hint) {
         *hint = elem->hint;
         pagedElementsCache->add(elem.getClear());

dali/base/dautils.hpp

@@ -254,10 +254,6 @@ extern da_decl void expandFileTree(IPropertyTree *file,bool expandnodes,const ch
 extern da_decl bool shrinkFileTree(IPropertyTree *file); // compresses parts into Parts blob
 extern da_decl void filterParts(IPropertyTree *file,UnsignedArray &partslist); // only include parts in list (in expanded tree)
 
-
-
-
-
 IRemoteConnection *getSortedElements( const char *basexpath, 
                                      const char *xpath, 
                                      const char *sortorder, 
@@ -285,7 +281,8 @@ extern da_decl IRemoteConnection *getElementsPaged(IElementsPager *elementsPager
                                      const char *owner,
                                      __int64 *hint,                         // if non null points to in/out cache hint
                                      IArrayOf<IPropertyTree> &results,
-                                     unsigned *total); // total possible filtered matches, i.e. irrespective of startoffset and pagesize
+                                     unsigned *total,
+                                     bool checkConn = true); // total possible filtered matches, i.e. irrespective of startoffset and pagesize
 
 extern da_decl void clearPagedElementsCache();
 

esp/eclwatch/ws_XSLT/dfu.xslt

@@ -42,6 +42,7 @@
     <xsl:variable name="filters" select="/DFUQueryResponse/Filters"/>
     <xsl:variable name="parametersforpaging" select="/DFUQueryResponse/ParametersForPaging"/>
     <xsl:variable name="basicquery" select="/DFUQueryResponse/BasicQuery"/>
+    <xsl:variable name="cachehint" select="/DFUQueryResponse/CacheHint"/>
 
     <xsl:template match="/DFUQueryResponse">
         <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
@@ -74,8 +75,7 @@
                     var pageSize = '<xsl:value-of select="$pagesize"/>';;
                     var sortBy = '<xsl:value-of select="$sortby"/>';;
                     var descending = '<xsl:value-of select="$descending"/>';;
-                    //var firstN = '<xsl:value-of select="$firstn"/>';;
-                    //var firstNType = '<xsl:value-of select="$firstntype"/>';;
+                    var cacheHint='<xsl:value-of select="$cachehint"/>';
 
             var oMenu;
 
@@ -390,7 +390,7 @@
                           
                             function headerClicked(headername, descending)
                             {
-                                document.location.href='/WsDfu/DFUQuery?'+currentFilters+'&Sortby='+headername+'&Descending='+descending;
+                                document.location.href='/WsDfu/DFUQuery?'+currentFilters.replace(/\&amp;/g,'&')+'&Sortby='+headername+'&Descending='+descending;
                             }                
                             
                             function onLoad()
@@ -412,9 +412,9 @@
                                 var size = pageEndAt - startFrom + 1;
 
                                 if (basicQuery.length > 0)
-                                    document.location.href = '/WsDfu/DFUQuery?PageSize='+size+'&'+basicQuery+'&PageStartFrom='+startFrom;
+                                    document.location.href = '/WsDfu/DFUQuery?PageSize='+size+'&'+basicQuery.replace(/\&amp;/g,'&')+'&PageStartFrom='+startFrom + '&CacheHint=' + cacheHint;
                                 else
-                                    document.location.href = '/WsDfu/DFUQuery?PageStartFrom='+startFrom+'&PageSize='+size;
+                                    document.location.href = '/WsDfu/DFUQuery?PageStartFrom='+startFrom+'&PageSize='+size + '&CacheHint=' + cacheHint;
 
                                 return false;
                             }
@@ -460,11 +460,11 @@
         <form id="listitems" action="/WsDFU/DFUArrayAction" method="post">
             <xsl:choose>
                 <xsl:when test="$basicquery!=''">
-          <input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery?PageSize={$pagesize}&amp;{$basicquery}&amp;PageStartFrom={/DFUQueryResponse/PageStartFrom}">&#160;</input> 
+          <input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery?PageSize={$pagesize}&amp;{$basicquery}&amp;PageStartFrom={/DFUQueryResponse/PageStartFrom}&amp;CacheHint={$cachehint}">&#160;</input>
                     <!--input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery{$pagesize}&amp;{/DFUQueryResponse/PageStartFrom}"/-->
                 </xsl:when>
                 <xsl:otherwise>
-                    <input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery?PageSize={$pagesize}&amp;PageStartFrom={/DFUQueryResponse/PageStartFrom}">&#160;</input>
+                    <input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery?PageSize={$pagesize}&amp;PageStartFrom={/DFUQueryResponse/PageStartFrom}&amp;CacheHint={$cachehint}">&#160;</input>
           <!--input type="hidden" id="BackToPage" name="BackToPage" value="/WsDfu/DFUQuery{$pagesize}&amp;{/DFUQueryResponse/PageStartFrom}"/-->
                 </xsl:otherwise>
             </xsl:choose>
@@ -519,14 +519,14 @@
                        </xsl:otherwise>
                    </xsl:choose>
                    <xsl:choose>
-                       <xsl:when test="$sortby='Size' and $descending &lt; 1">
-                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('Size', 1)">Size<img src="/esp/files/img/upsimple.png" width="10" height="10"></img></th>
+                       <xsl:when test="$sortby='FileSize' and $descending &lt; 1">
+                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('FileSize', 1)">Size<img src="/esp/files/img/upsimple.png" width="10" height="10"></img></th>
                        </xsl:when>
-                       <xsl:when test="$sortby='Size'">
-                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('Size', 0)">Size<img src="/esp/files/img/downsimple.png" width="10" height="10"></img></th>
+                       <xsl:when test="$sortby='FileSize'">
+                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('FileSize', 0)">Size<img src="/esp/files/img/downsimple.png" width="10" height="10"></img></th>
                        </xsl:when>
                        <xsl:otherwise>
-                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('Size', 1)">Size</th>
+                          <th align="center" style="cursor:pointer" onmouseover="bgColor='#FFFFFF'" onmouseout="bgColor='#CCCCCC'" onclick="headerClicked('FileSize', 1)">Size</th>
                        </xsl:otherwise>
                    </xsl:choose>
                    <xsl:choose>
@@ -593,12 +593,12 @@
       <table>
             <tr>
             <xsl:if test="$prevpagefrom &gt; 0">
-                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}')">First</a></td>
-                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$prevpagefrom}')">Prev</a></td>
+                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;CacheHint={$cachehint}')">First</a></td>
+                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$prevpagefrom}&amp;CacheHint={$cachehint}')">Prev</a></td>
             </xsl:if>
             <xsl:if test="$nextpagefrom &gt; 0">
-                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$nextpagefrom}')">Next</a></td>
-                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$lastpagefrom}')">Last</a></td>
+                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$nextpagefrom}&amp;CacheHint={$cachehint}')">Next</a></td>
+                <td><a href="javascript:go('/WsDfu/DFUQuery?{$parametersforpaging}&amp;PageStartFrom={$lastpagefrom}&amp;CacheHint={$cachehint}')">Last</a></td>
             </xsl:if>
             </tr>
       </table>  
@@ -660,7 +660,7 @@
               </xsl:if>
             </td>
             <td>
-              <xsl:if test="IsKeyFile=1">
+              <xsl:if test="ContentType='key'">
                 <img border="0" src="/esp/files/img/keyfile.png" title="Indexed" width="16" height="16"/>
               </xsl:if>
             </td>

esp/eclwatch/ws_XSLT/dfu_search.xslt

@@ -243,22 +243,25 @@
                             else
                             {
                                 var firstNTypeS = document.getElementById("FirstNType").selectedIndex;
-                                var firstNType = "newest";
-                                if (firstNTypeS > 2)
-                                    firstNType = "smallest";
-                                else if (firstNTypeS > 1)
-                                    firstNType = "largest";
-                                else if (firstNTypeS > 0)
-                                    firstNType = "oldest";
-
+                                var sortBy = "Modified";
+                                var descending = true;
+                                if (firstNTypeS > 2)//largest
+                                {
+                                    sortBy = "FileSize";
+                                    descending = "false";
+                                }
+                                else if (firstNTypeS > 1)//smallest
+                                    sortBy = "FileSize";
+                                else if (firstNTypeS > 0)//oldest
+                                    descending = "false";
                                 if (first)
                                 {
-                                    url += "?FirstN=" + firstN + "&FirstNType=" + firstNType;
+                                    url += "?FirstN=" + firstN + "&Sortby=" + sortBy + "&Descending=" + descending;
                                     first = false;
                                 }
                                 else
                                 {
-                                    url += "&FirstN=" + firstN + "&FirstNType=" + firstNType;
+                                    url += "&FirstN=" + firstN + "&Sortby=" + sortBy + "&Descending=" + descending;
                                 }
                             }
                         

esp/scm/ws_dfu.ecm

@@ -52,8 +52,9 @@ ESPStruct DFULogicalFile
     [min_ver("1.01")] int64 IntRecordCount;
     [min_ver("1.02")] bool FromRoxieCluster;
     [min_ver("1.03")] bool BrowseData;
-    [min_ver("1.14")] bool IsKeyFile;
+    [min_ver("1.14"), depr_ver("1.24")] bool IsKeyFile;
     [min_ver("1.22")] bool IsCompressed;
+    [min_ver("1.24")] string ContentType;
     [min_ver("1.22")] int64 CompressedFileSize;
 };
 
@@ -124,10 +125,7 @@ ESPStruct DFUSpaceItem
     string SmallestSize;
 };
 
-ESPrequest 
-[
-]
-DFUQueryRequest
+ESPrequest [nil_remove] DFUQueryRequest
 {
     string Prefix;
     string ClusterName;
@@ -140,7 +138,7 @@ DFUQueryRequest
     int64 FileSizeFrom(-1);
     int64 FileSizeTo(-1);
     int FirstN(-1);
-    string FirstNType;
+    [depr_ver("1.24")] string FirstNType;
 
     int PageSize;
     int PageStartFrom;
@@ -149,6 +147,7 @@ DFUQueryRequest
     bool Descending(false);
 
     bool OneLevelDirFileReturn(false);
+    [min_ver("1.24")] int64 CacheHint;
 };
 
 ESPresponse 
@@ -170,7 +169,7 @@ DFUQueryResponse
     int64 FileSizeFrom(-1);
     int64 FileSizeTo(-1);
     int FirstN(-1);
-    string FirstNType;
+    [depr_ver("1.24")] string FirstNType;
 
     int PageSize(20);
     int64 PageStartFrom(1);
@@ -185,6 +184,7 @@ DFUQueryResponse
     string BasicQuery;
     string ParametersForPaging;
     string Filters;
+    [min_ver("1.24")] int64 CacheHint;
 };
 
 ESPrequest 
@@ -631,7 +631,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUSearchDataRespons
 
 //  ===========================================================================
 ESPservice [
-    version("1.23"), default_client_version("1.23"),
+    version("1.24"), default_client_version("1.24"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/ws_dfu/ws_dfuService.hpp

@@ -28,23 +28,7 @@
 
 #include "fileview.hpp"
 #include "fvrelate.hpp"
-
-/*
-class CSpaceItem64 : public CInterface
-{
-public:
-    char Name[256];
-    char LargestFile[256];
-    char SmallestFile[256];
-    __int64 NumOfFilesInt;
-    __int64 NumOfFilesIntUnknown;
-    __int64 TotalSizeInt;
-    __int64 LargestSizeInt;
-    __int64 SmallestSizeInt;
-
-    IMPLEMENT_IINTERFACE;
-};*/
-
+#include "dadfs.hpp"
 
 class CWsDfuSoapBindingEx : public CWsDfuSoapBinding
 {
@@ -88,9 +72,20 @@ public:
     virtual bool onSuperfileAction(IEspContext &context, IEspSuperfileActionRequest &req, IEspSuperfileActionResponse &resp);
 
 private:
+    const char* getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix);
+    const char* getShortDescription(const char* description, StringBuffer& shortDesc);
+    bool addDFUQueryFilter(DFUQResultField *filters, unsigned short &count, MemoryBuffer &buff, const char* value, DFUQResultField name);
+    void appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, StringBuffer& filterBuf);
+    void appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, const char *valueHigh, StringBuffer& filterBuf);
+    void setFileTypeFilter(const char* fileType, StringBuffer& filterBuf);
+    void setFileNameFilter(const char* fname, const char* prefix, StringBuffer &buff);
+    void setDFUQueryFilters(IEspDFUQueryRequest& req, StringBuffer& filterBuf);
+    void setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sortBy, bool& descending, DFUQResultField* sortOrder);
+    bool addToLogicalFileList(IPropertyTree& file, double version, IArrayOf<IEspDFULogicalFile>& logicalFiles);
+    void setDFUQueryResponse(IEspContext &context, unsigned totalFiles, StringBuffer& sortBy, bool descending, unsigned pageStart,
+        unsigned pageSize, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
     void getLogicalFileAndDirectory(IEspContext &context, IUserDescriptor* udesc, const char *dirname, IArrayOf<IEspDFULogicalFile>& LogicalFiles, int& numFiles, int& numDirs);
     bool doLogicalFileSearch(IEspContext &context, IUserDescriptor* udesc, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
-    //bool doLogicalFileSearch(IUserDescriptor* udesc, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
     void doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, const char *name,const char *cluster,
         const char *description,IEspDFUFileDetail& FileDetails);
     bool createSpaceItemsByDate(IArrayOf<IEspSpaceItem>& SpaceItems, StringBuffer interval, unsigned& yearFrom, 
@@ -98,7 +93,7 @@ private:
     bool setSpaceItemByScope(IArrayOf<IEspSpaceItem>& SpaceItems64, const char*scopeName, const char*logicalName, __int64 size);
     bool setSpaceItemByOwner(IArrayOf<IEspSpaceItem>& SpaceItems64, const char *owner, const char *logicalName, __int64 size);
     bool setSpaceItemByDate(IArrayOf<IEspSpaceItem>& SpaceItems, StringBuffer interval, StringBuffer mod, const char*logicalName, __int64 size);
-    bool findPositionToAdd(const char *datetime, const __int64 size, const int numNeeded, const unsigned orderType, 
+    bool findPositionToAdd(const char *datetime, const __int64 size, const int numNeeded, const unsigned orderType,
                        IArrayOf<IEspDFULogicalFile>& LogicalFiles, int& addToPos, bool& reachLimit);
     __int64 findPositionByParts(const __int64 parts, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionBySize(const __int64 size, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
@@ -109,6 +104,7 @@ private:
     __int64 findPositionByDate(const char *datetime, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionByDescription(const char *description, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     bool checkDescription(const char *description, const char *descriptionFilter);
+    void getAPageOfSortedLogicalFile(IEspContext &context, IUserDescriptor* udesc, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
     void getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuffer& returnStr);
     void xsltTransformer(const char* xsltPath,StringBuffer& source,StringBuffer& returnStr);
     bool onDFUAction(IUserDescriptor* udesc, const char* LogicalFileName, const char* ClusterName, const char* ActionType, StringBuffer& returnStr);