Merge pull request #7206 from wangkx/h13259

HPCC-13390 Return persist/superowners in WsDfu.DFUQuery

Reviewed By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed By: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dadfs.cpp

@@ -1914,15 +1914,25 @@ public:
     }
 };
 
+struct SerializeFileAttrOptions
+{
+    bool includeSuperOwner;
+    //Add more as needed
+
+    SerializeFileAttrOptions()
+    {
+        includeSuperOwner = false;
+    }
+};
 
 class CDFAttributeIterator: public CInterface, implements IDFAttributesIterator
 {
-    IArrayOf<IPropertyTree> attrs; 
+    IArrayOf<IPropertyTree> attrs;
     unsigned index;
 public:
     IMPLEMENT_IINTERFACE;
 
-    static MemoryBuffer &serializeFileAttributes(MemoryBuffer &mb, IPropertyTree &root, const char *name, bool issuper)
+    static MemoryBuffer &serializeFileAttributes(MemoryBuffer &mb, IPropertyTree &root, const char *name, bool issuper, SerializeFileAttrOptions& option)
     {
         StringBuffer buf;
         mb.append(name);
@@ -1932,7 +1942,7 @@ public:
             mb.append("");
         }
         else {
-            mb.append(root.queryProp("@directory"));        
+            mb.append(root.queryProp("@directory"));
             mb.append(root.getPropInt("@numparts",0));
             mb.append(root.queryProp("@partmask"));
         }
@@ -1980,6 +1990,25 @@ public:
                 mb.append(plist.str());
             }
         }
+        if (option.includeSuperOwner) {
+            //add superowners
+            StringBuffer soList;
+            Owned<IPropertyTreeIterator> superOwners = root.getElements("SuperOwner");
+            ForEach(*superOwners) {
+                IPropertyTree &superOwner = superOwners->query();
+                const char *name = superOwner.queryProp("@name");
+                if (name&&*name) {
+                    if (soList.length())
+                        soList.append(",");
+                    soList.append(name);
+                }
+            }
+            if (soList.length()) {
+                count++;
+                mb.append("@superowners");
+                mb.append(soList.str());
+            }
+        }
         mb.writeDirect(countpos,sizeof(count),&count);
         return mb;
     }
@@ -2001,7 +2030,7 @@ public:
                 mb.read(val);   // not used currently
             }
             else {
-                attr->setProp("@directory",val.get());     
+                attr->setProp("@directory",val.get());
                 mb.read(n);
                 attr->setPropInt("@numparts",n);
                 mb.read(val);
@@ -2045,7 +2074,7 @@ public:
         index++;
         return (index<attrs.ordinality());
     }
-                            
+
     bool  isValid()
     {
         return (index<attrs.ordinality());
@@ -2068,7 +2097,7 @@ class CDFProtectedIterator: public CInterface, implements IDFProtectedIterator
     Owned<IPropertyTreeIterator> fiter;
     Owned<IPropertyTreeIterator> piter;
     unsigned defaultTimeout;
-    
+
     bool notsuper;
     bool superonly;
 
@@ -7968,7 +7997,7 @@ void CDistributedFileDirectory::removeEmptyScope(const char *scope)
     if (scope&&*scope) {
         StringBuffer fn(scope);
         fn.append("::x");
-        CDfsLogicalFileName dlfn;   
+        CDfsLogicalFileName dlfn;
         dlfn.set(fn.str());
         removeFileEmptyScope(dlfn,defaultTimeout);
     }
@@ -7985,7 +8014,7 @@ void CDistributedFileDirectory::renamePhysical(const char *oldname,const char *n
 #endif
         user = defaultudesc.get();
     }
-    CDfsLogicalFileName oldlogicalname; 
+    CDfsLogicalFileName oldlogicalname;
     oldlogicalname.set(oldname);
     checkLogicalName(oldlogicalname,user,true,true,false,"rename");
 
@@ -8006,7 +8035,7 @@ void CDistributedFileDirectory::renamePhysical(const char *oldname,const char *n
 
 void CDistributedFileDirectory::fixDates(IDistributedFile *file)
 {
-    // should do in parallel 
+    // should do in parallel
     unsigned width = file->numParts();
     CriticalSection crit;
     class casyncfor: public CAsyncFor
@@ -8061,7 +8090,7 @@ void CDistributedFileDirectory::fixDates(IDistributedFile *file)
 
 void CDistributedFileDirectory::addEntry(CDfsLogicalFileName &dlfn,IPropertyTree *root,bool superfile, bool ignoreexists)
 {
-    // add bit awkward 
+    // add bit awkward
     bool external;
     bool foreign;
     external = dlfn.isExternal();
@@ -8107,7 +8136,7 @@ IDistributedFileIterator *CDistributedFileDirectory::getIterator(const char *wil
 
 GetFileClusterNamesType CDistributedFileDirectory::getFileClusterNames(const char *_logicalname,StringArray &out)
 {
-    CDfsLogicalFileName logicalname;    
+    CDfsLogicalFileName logicalname;
     logicalname.set(_logicalname);
     if (logicalname.isForeign())
         return GFCN_Foreign;
@@ -8121,7 +8150,7 @@ GetFileClusterNamesType CDistributedFileDirectory::getFileClusterNames(const cha
             getFileGroups(froot,out);
             return GFCN_Normal;
         }
-        if (bkind==DXB_SuperFile) 
+        if (bkind==DXB_SuperFile)
             return GFCN_Super;
     }
     return GFCN_NotFound;
@@ -8142,7 +8171,7 @@ IDistributedFileDirectory &queryDistributedFileDirectory()
 {
     if (!DFdir) {
         CriticalBlock block(dfdirCrit);
-        if (!DFdir) 
+        if (!DFdir)
             DFdir = new CDistributedFileDirectory();
     }
     return *DFdir;
@@ -8154,7 +8183,7 @@ IDistributedFileDirectory &queryDistributedFileDirectory()
 void closedownDFS()  // called by dacoven
 {
     CriticalBlock block(dfdirCrit);
-    try { 
+    try {
         delete DFdir;
     }
     catch (IMP_Exception *e) {
@@ -8193,17 +8222,17 @@ public:
             return;
         while (*s) {
             if (isdigit(*s)) {
-                pn = pn*10+(*s-'0'); 
+                pn = pn*10+(*s-'0');
                 if (pn>max)
-                    max = pn;       
+                    max = pn;
             }
             else
                 pn = 0;
             s++;
         }
-        if (max==0) 
+        if (max==0)
             return;
-        partincluded = new bool[max];       
+        partincluded = new bool[max];
         unsigned i;
         for (i=0;i<max;i++)
             partincluded[i] = false;
@@ -8224,9 +8253,9 @@ public:
                 pn = 0;
             }
             else if (isdigit(*s)) {
-                pn = pn*10+(*s-'0'); 
+                pn = pn*10+(*s-'0');
                 if (pn>max)
-                    max = pn;       
+                    max = pn;
                 if (s[1]=='-') {
                     s++;
                     start = pn;
@@ -8254,7 +8283,7 @@ public:
 
 IDFPartFilter *createPartFilter(const char *filter)
 {
-    return new CDFPartFilter(filter); 
+    return new CDFPartFilter(filter);
 
 }
 
@@ -8492,7 +8521,7 @@ public:
 };
 typedef CIArrayOf<CDFUSFFilter> CDFUSFFilterArray;
 
-class CFileScanFilterContainer : public CInterface
+class CIterateFileFilterContainer : public CInterface
 {
     StringAttr filterBuf; //Hold original filter string just in case
     StringAttr wildNameFilter;
@@ -8502,6 +8531,8 @@ class CFileScanFilterContainer : public CInterface
     //filtering the files using File Attributes tree and CDFUSFFilter::checkFilter(). The wildNameFilter and fileTypeFilter need
     //special code to filter the files.
 
+    SerializeFileAttrOptions options;
+
     bool isValidInteger(const char *s)
     {
         if (!s)
@@ -8514,6 +8545,20 @@ class CFileScanFilterContainer : public CInterface
         }
         return true;
     }
+    void addOption(const char* optionStr)
+    {
+        if (!optionStr || !*optionStr || !isdigit(*optionStr))
+            return;
+
+        DFUQSerializeFileAttrOption option = (DFUQSerializeFileAttrOption) atoi(optionStr);
+        switch(option)
+        {
+        case DFUQSFAOincludeSuperOwner:
+            options.includeSuperOwner =  true;
+            break;
+        //Add more when needed
+        }
+    }
     void addFilter(DFUQFilterType filterType, const char* attr, const char* value, const char* valueHigh)
     {
         if (!attr || !*attr)
@@ -8593,13 +8638,13 @@ class CFileScanFilterContainer : public CInterface
     }
 
 public:
-    CFileScanFilterContainer()
+    CIterateFileFilterContainer()
     {
         fileTypeFilter = DFUQFFTall;
         wildNameFilter.set("*");
         filterBuf.clear();
     };
-    void readScanFilters(const char *filterStr)
+    void readFilters(const char *filterStr)
     {
         if (!filterStr || !*filterStr)
             return;
@@ -8638,6 +8683,11 @@ public:
                 if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | from filter | to filter
                     addFilter(filterType, filterStringArray.item(i+1), (const char*)filterStringArray.item(i+2), (const char*)filterStringArray.item(i+3));
                 break;
+            case DFUQFTincludeFileAttr:
+                filterSize = 2;
+                if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter
+                    addOption(filterStringArray.item(i+1));
+                break;
             case DFUQFTspecial:
                 filterSize = 3;
                 if (filterFieldsToRead >= filterSize) //DFUQFilterType | filter name | filter value
@@ -8677,6 +8727,7 @@ public:
             return;
         wildNameFilter.set(_wildName);
     }
+    SerializeFileAttrOptions& getSerializeFileAttrOptions() { return options; }
 };
 
 class CFileScanner
@@ -8686,18 +8737,17 @@ class CFileScanner
     StringAttr wildname;
     Owned<CScope> topLevelScope;
     CScope *currentScope;
-    bool fileScanWithFilter;
-    CFileScanFilterContainer fileScanFilterContainer;
+    Owned<CIterateFileFilterContainer> iterateFileFilterContainer;
 
     bool scopeMatch(const char *name)
     {   // name has trailing '::'
         if (!name || !*name)
             return true;
         const char *s1 = NULL;
-        if (!fileScanWithFilter)
+        if (!iterateFileFilterContainer)
             s1 = wildname.get();
         else
-            s1 = fileScanFilterContainer.getNameFilter();
+            s1 = iterateFileFilterContainer->getNameFilter();
         if (!s1 || !*s1)
             return true;
         const char *s2 = name;
@@ -8751,7 +8801,7 @@ class CFileScanner
                     }
                 } while (iter->next());
             }
-            if (!fileScanWithFilter)
+            if (!iterateFileFilterContainer)
                 ret |= processFiles(root,name);
             else
                 ret |= processFilesWithFilters(root,name);
@@ -8802,7 +8852,7 @@ class CFileScanner
     {
         bool ret = false;
         size32_t ns = name.length();
-        DFUQFileTypeFilter fileTypeFilter = fileScanFilterContainer.getFileTypeFilter();
+        DFUQFileTypeFilter fileTypeFilter = iterateFileFilterContainer->getFileTypeFilter();
         if (fileTypeFilter != DFUQFFTsuperfileonly)
             addMatchedFiles(root.getElements(queryDfsXmlBranchName(DXB_File)), false, name, ns, ret);
         if ((fileTypeFilter == DFUQFFTall) || (fileTypeFilter == DFUQFFTsuperfileonly))
@@ -8817,7 +8867,7 @@ class CFileScanner
         {
             IPropertyTree &file = iter->query();
             name.append(file.queryProp("@name"));
-            if (fileScanFilterContainer.matchFileScanFilter(name.str(), file))
+            if (iterateFileFilterContainer->matchFileScanFilter(name.str(), file))
             {
                 currentScope->addMatch(name,file,isSuper);
                 ret = true;
@@ -8828,7 +8878,6 @@ class CFileScanner
 public:
     void scan(IPropertyTree *sroot, const char *_wildname,bool _recursive,bool _includesuper)
     {
-        fileScanWithFilter =  false;
         if (_wildname)
             wildname.set(_wildname);
         else
@@ -8840,11 +8889,10 @@ public:
         currentScope = NULL;
         processScopes(*sroot->queryPropTree(querySdsFilesRoot()),name);
     }
-    void scan(IPropertyTree *sroot, const char *filters, bool _recursive)
+    void scan(IPropertyTree *sroot, CIterateFileFilterContainer* _iterateFileFilterContainer, bool _recursive)
     {
-        fileScanWithFilter =  true;
+        iterateFileFilterContainer.setown(_iterateFileFilterContainer);
         recursive = _recursive;
-        fileScanFilterContainer.readScanFilters(filters);
 
         StringBuffer name;
         topLevelScope.clear();
@@ -9452,9 +9500,6 @@ bool removeClusterSpares(const char *clusterName, const char *type, SocketEndpoi
     return init.removeSpares(clusterName, type, eps, response);
 }
 
-
-
-
 class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements IDaliServer, implements IExceptionHandler
 {  // Coven size
     
@@ -9574,10 +9619,11 @@ public:
 
         sdsLock.lock(); // re-lock sds while serializing
         start = msTick();
+        SerializeFileAttrOptions options; //The options is needed for the serializeFileAttributes()
         ForEachItemIn(m, matchingFiles)
         {
             CFileMatch &fileMatch = matchingFiles.item(m);
-            CDFAttributeIterator::serializeFileAttributes(mb, fileMatch.queryFileTree(), fileMatch.queryName(), fileMatch.queryIsSuper());
+            CDFAttributeIterator::serializeFileAttributes(mb, fileMatch.queryFileTree(), fileMatch.queryName(), fileMatch.queryIsSuper(), options);
         }
         tookMs = msTick()-start;
         if (tookMs>100)
@@ -9607,10 +9653,13 @@ public:
         unsigned count=0;
         mb.append(count);
 
+        Owned<CIterateFileFilterContainer> iterateFileFilterContainer =  new CIterateFileFilterContainer();
+        iterateFileFilterContainer->readFilters(filters);
+
         CFileScanner scanner;
         CSDSServerLockBlock sdsLock; // lock sds while scanning
         unsigned start = msTick();
-        scanner.scan(sdsLock, filters.get(), recursive);
+        scanner.scan(sdsLock, iterateFileFilterContainer.getLink(), recursive);
         unsigned tookMs = msTick()-start;
         if (tookMs>100)
             PROGLOG("TIMING(filescan): %s: took %dms",trc.str(), tookMs);
@@ -9630,7 +9679,7 @@ public:
         ForEachItemIn(m, matchingFiles)
         {
             CFileMatch &fileMatch = matchingFiles.item(m);
-            CDFAttributeIterator::serializeFileAttributes(mb, fileMatch.queryFileTree(), fileMatch.queryName(), fileMatch.queryIsSuper());
+            CDFAttributeIterator::serializeFileAttributes(mb, fileMatch.queryFileTree(), fileMatch.queryName(), fileMatch.queryIsSuper(), iterateFileFilterContainer->getSerializeFileAttrOptions());
         }
         tookMs = msTick()-start;
         if (tookMs>100)
@@ -11795,7 +11844,7 @@ IDFProtectedIterator *CDistributedFileDirectory::lookupProtectedFiles(const char
 
 const char* DFUQResultFieldNames[] = { "@name", "@description", "@group", "@kind", "@modified", "@job", "@owner",
     "@DFUSFrecordCount", "@recordCount", "@recordSize", "@DFUSFsize", "@size", "@workunit", "@DFUSFcluster", "@numsubfiles",
-    "@accessed", "@numparts", "@compressedSize", "@directory", "@partmask" };
+    "@accessed", "@numparts", "@compressedSize", "@directory", "@partmask", "@superowners", "@persistent" };
 
 extern da_decl const char* getDFUQResultFieldName(DFUQResultField feild)
 {

dali/base/dadfs.hpp

@@ -179,7 +179,14 @@ enum DFUQFilterType
     DFUQFTstringRange,
     DFUQFTintegerRange,
     DFUQFTinteger64Range,
-    DFUQFTspecial
+    DFUQFTspecial,
+    DFUQFTincludeFileAttr
+};
+
+enum DFUQSerializeFileAttrOption
+{
+    DFUQSFAOincludeSuperOwner = 1
+    //May add more
 };
 
 enum DFUQSpecialFilter
@@ -258,7 +265,9 @@ enum DFUQResultField
     DFUQRFcompressedsize = 17,
     DFUQRFdirectory = 18,
     DFUQRFpartmask = 19,
-    DFUQRFterm = 20,
+    DFUQRFsuperowners = 20,
+    DFUQRFpersistent = 21,
+    DFUQRFterm = 22,
     DFUQRFreverse = 256,
     DFUQRFnocase = 512,
     DFUQRFnumeric = 1024

esp/scm/ws_dfu.ecm

@@ -57,6 +57,8 @@ ESPStruct DFULogicalFile
     [min_ver("1.22")] bool IsCompressed;
     [min_ver("1.24")] string ContentType;
     [min_ver("1.22")] int64 CompressedFileSize;
+    [min_ver("1.30")] string SuperOwners;
+    [min_ver("1.30")] bool Persistent;
 };
 
 ESPStruct DFUPart
@@ -169,6 +171,7 @@ ESPrequest [nil_remove] DFUQueryRequest
 
     bool OneLevelDirFileReturn(false);
     [min_ver("1.24")] int64 CacheHint;
+    [min_ver("1.30")] bool IncludeSuperOwner;
 };
 
 ESPresponse 
@@ -679,7 +682,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUGetFileMetaDataRe
 
 //  ===========================================================================
 ESPservice [
-    version("1.29"),
+    version("1.30"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -3064,6 +3064,8 @@ void CWsDfuEx::setDFUQueryFilters(IEspDFUQueryRequest& req, StringBuffer& filter
     appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFdescription), DFUQFTwildcardMatch, req.getDescription(), filterBuf);
     appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFattrowner), DFUQFTwildcardMatch, req.getOwner(), filterBuf);
     appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFgroup), DFUQFTcontainString, req.getNodeGroup(), ",", filterBuf);
+    if (!req.getIncludeSuperOwner_isNull() && req.getIncludeSuperOwner())
+        filterBuf.append(DFUQFTincludeFileAttr).append(DFUQFilterSeparator).append(DFUQSFAOincludeSuperOwner).append(DFUQFilterSeparator);
 
     __int64 sizeFrom = req.getFileSizeFrom();
     __int64 sizeTo = req.getFileSizeTo();
@@ -3204,6 +3206,15 @@ bool CWsDfuEx::addToLogicalFileList(IPropertyTree& file, const char* nodeGroup,
         }
         lFile->setBrowseData(numSubFiles > 1 ? false : true); ////Bug 41379 - ViewKeyFile Cannot handle superfile with multiple subfiles
 
+        if (version >= 1.30)
+        {
+            bool persistent = file.getPropBool(getDFUQResultFieldName(DFUQRFpersistent), false);
+            if (persistent)
+                lFile->setPersistent(true);
+            if (file.hasProp(getDFUQResultFieldName(DFUQRFsuperowners)))
+                lFile->setSuperOwners(file.queryProp(getDFUQResultFieldName(DFUQRFsuperowners)));
+        }
+
         __int64 size = file.getPropInt64(getDFUQResultFieldName(DFUQRForigsize),0);
         if (size > 0)
         {