HPCC-20437 Set limit of returned files in DropZoneFileSearch

The existing DropZoneFileSearch fails when too many files
match with a wildcard filter. In this fix, a max file limit
is introduced. If >1000 files match the search filter, only
1000 files are returned with a warning message. The code for
the file search is also simplified by passing the 'sub=true'
option. And a client has to specify a valid dropzone name, a
valid dropzone server, and a valid file name filter to do
the search.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/scm/ws_fs.ecm

@@ -558,6 +558,7 @@ ESPrequest [nil_remove] DropZoneFileSearchRequest
 ESPresponse [exceptions_inline] DropZoneFileSearchResponse
 {
     ESParray<ESPStruct PhysicalFileStruct> Files;
+    [min_ver("1.16")] string Warning;
 };
 
 ESPrequest OpenSaveRequest
@@ -679,7 +680,7 @@ ESPresponse [exceptions_inline, nil_remove] GetDFUServerQueuesResponse
 
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.15"),
+    version("1.16"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPmethod EchoDateTime(EchoDateTime, EchoDateTimeResponse);

esp/services/ws_fs/ws_fsService.cpp

@@ -46,6 +46,8 @@
 #define FILE_DESPRAY_URL    "FileDesprayAccess"
 #define WUDETAILS_REFRESH_MINS 1
 
+const unsigned dropZoneFileSearchMaxFiles = 1000;
+
 void SetResp(StringBuffer &resp, IConstDFUWorkUnit * wu, bool array);
 int Schedule::run()
 {
@@ -2833,16 +2835,30 @@ bool CFileSprayEx::checkDropZoneIPAndPath(double clientVersion, const char* drop
     return false;
 }
 
-void CFileSprayEx::addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* dropZonePath,
-    StringBuffer& relativePath, const char pathSep, IArrayOf<IEspPhysicalFileStruct>& filesInFolder, IArrayOf<IEspPhysicalFileStruct>& files)
+void CFileSprayEx::addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* path, IArrayOf<IEspPhysicalFileStruct>& files)
 {
-    StringBuffer path = dropZonePath;
-    if (!relativePath.isEmpty())
-        path.append(pathSep).append(relativePath.str());
-
     Owned<IEspPhysicalFileStruct> aFile = createPhysicalFileStruct();
-    aFile->setName(name);
-    aFile->setPath(path);
+
+    const char pathSep = getPathSepChar(path); 
+    const char* pName = strrchr(name, pathSep);
+    if (!pName)
+    {
+        aFile->setName(name);
+        aFile->setPath(path);
+    }
+    else
+    {
+        StringBuffer sPath = path;
+        unsigned len = strlen(path);
+        if (sPath.charAt(len - 1) != pathSep)
+            sPath.append(pathSep);
+        sPath.append(pName - name, name);
+        aFile->setPath(sPath.str());
+
+        pName++; //skip the PathSepChar
+        aFile->setName(pName);
+    }
+
     aFile->setIsDir(di->isDir());
     CDateTime modtime;
     StringBuffer timestr;
@@ -2853,16 +2869,20 @@ void CFileSprayEx::addDropZoneFile(IEspContext& context, IDirectoryIterator* di,
     timestr.appendf("%04d-%02d-%02d %02d:%02d:%02d", y,m,d,h,min,sec);
     aFile->setModifiedtime(timestr.str());
     aFile->setFilesize(di->getFileSize());
-    if (di->isDir() && filesInFolder.ordinality())
-        aFile->setFiles(filesInFolder);
     files.append(*aFile.getLink());
 }
 
-bool CFileSprayEx::searchDropZoneFileInFolder(IEspContext& context, IFile* f, const char* nameFilter,
-    bool returnAll, const char* dropZonePath, StringBuffer& relativePath, const char pathSep, IArrayOf<IEspPhysicalFileStruct>& files)
+void CFileSprayEx::searchDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files, unsigned& filesFound)
 {
-    bool foundMatch = false;
-    Owned<IDirectoryIterator> di = f->directoryFiles(NULL, false, true);
+    RemoteFilename rfn;
+    SocketEndpoint ep;
+    ep.ipset(ip);
+    rfn.setPath(ep, dir);
+    Owned<IFile> f = createIFile(rfn);
+    if(!f->isDirectory())
+        throw MakeStringException(ECLWATCH_INVALID_DIRECTORY, "%s is not a directory.", dir);
+
+    Owned<IDirectoryIterator> di = f->directoryFiles(nameFilter, true, true);
     ForEach(*di)
     {
         StringBuffer fname;
@@ -2870,49 +2890,12 @@ bool CFileSprayEx::searchDropZoneFileInFolder(IEspContext& context, IFile* f, co
         if (!fname.length())
             continue;
 
-        StringBuffer newPath = relativePath;
-        if (!newPath.isEmpty())
-            newPath.append(pathSep);
-        newPath.append(fname.str());
-
-        IArrayOf<IEspPhysicalFileStruct> filesInFolder;
-        if (returnAll) //Every files and subfolders in this folder have to be returned
-        {
-            if (di->isDir())
-                searchDropZoneFileInFolder(context, &di->get(), NULL, returnAll, dropZonePath, newPath, pathSep, filesInFolder);
-            addDropZoneFile(context, di, fname.str(), dropZonePath, relativePath, pathSep, filesInFolder, files);
-            continue;
-        }
-
-        bool foundMatchNew = WildMatch(fname.str(), nameFilter, true);
-        if (di->isDir() && searchDropZoneFileInFolder(context, &di->get(), nameFilter, foundMatchNew, dropZonePath,
-            newPath, pathSep, filesInFolder))
-        {
-            foundMatchNew = true;
-        }
+        filesFound++;
+        if (filesFound > dropZoneFileSearchMaxFiles)
+            break;
 
-        if (foundMatchNew)
-        {
-            addDropZoneFile(context, di, fname.str(), dropZonePath, relativePath, pathSep, filesInFolder, files);
-            foundMatch = true;
-        }
+        addDropZoneFile(context, di, fname.str(), dir, files);
     }
-    return foundMatch;
-}
-
-void CFileSprayEx::appendDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files)
-{
-    RemoteFilename rfn;
-    SocketEndpoint ep;
-    ep.ipset(ip);
-    rfn.setPath(ep, dir);
-    Owned<IFile> f = createIFile(rfn);
-    if(!f->isDirectory())
-        throw MakeStringException(ECLWATCH_INVALID_DIRECTORY, "%s is not a directory.", dir);
-
-    StringBuffer relativePath;
-    searchDropZoneFileInFolder(context, f, nameFilter, !nameFilter || !*nameFilter, dir,
-        relativePath, getPathSepChar(dir), files);
 }
 
 bool CFileSprayEx::onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSearchRequest &req, IEspDropZoneFileSearchResponse &resp)
@@ -2924,6 +2907,14 @@ bool CFileSprayEx::onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSe
         const char* dropZoneName = req.getDropZoneName();
         if (isEmptyString(dropZoneName))
             throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone not specified.");
+        const char* dropZoneServerReq = req.getServer(); //IP or hostname
+        if (isEmptyString(dropZoneServerReq))
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone server not specified.");
+        const char* nameFilter = req.getNameFilter();
+        if (isEmptyString(nameFilter))
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Name Filter not specified.");
+        if (streq(nameFilter, "*"))
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid Name Filter '*'");
 
         Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
         Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
@@ -2937,14 +2928,13 @@ bool CFileSprayEx::onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSe
             throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone Directory not found for %s.", dropZoneName);
 
         IpAddress ipToMatch;
-        const char* dropZoneServerReq = req.getServer(); //IP or hostname
-        if (!isEmptyString(dropZoneServerReq))
-        {
-            ipToMatch.ipset(dropZoneServerReq);
-            if (ipToMatch.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid server %s specified.", dropZoneServerReq);
-        }
+        ipToMatch.ipset(dropZoneServerReq);
+        if (ipToMatch.isNull())
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid server %s specified.", dropZoneServerReq);
 
+        double version = context.getClientVersion();
+        bool serverFound = false;
+        unsigned filesFound = 0;
         IArrayOf<IEspPhysicalFileStruct> files;
         Owned<IConstDropZoneServerInfoIterator> dropZoneServerItr = dropZoneInfo->getServers();
         ForEach(*dropZoneServerItr)
@@ -2957,8 +2947,21 @@ bool CFileSprayEx::onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSe
 
             IpAddress ipAddr;
             ipAddr.ipset(server.str());
-            if (isEmptyString(dropZoneServerReq) || ipAddr.ipequals(ipToMatch))
-                appendDropZoneFiles(context, ipAddr, directory.str(), req.getNameFilter(), files);
+            if (ipAddr.ipequals(ipToMatch))
+            {
+                serverFound = true;
+                searchDropZoneFiles(context, ipAddr, directory.str(), nameFilter, files, filesFound);
+                break;
+            }
+        }
+
+        if (!serverFound)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Server %s not found in dropzone %s.", dropZoneServerReq, dropZoneName);
+
+        if ((version >= 1.16) && (filesFound > dropZoneFileSearchMaxFiles))
+        {
+            VStringBuffer msg("More than %u files are found. Only %u files are returned.", dropZoneFileSearchMaxFiles, dropZoneFileSearchMaxFiles);
+            resp.setWarning(msg.str());
         }
         resp.setFiles(files);
     }

esp/services/ws_fs/ws_fsService.hpp

@@ -149,13 +149,10 @@ protected:
     bool getOneDFUWorkunit(IEspContext& context, const char* wuid, IEspGetDFUWorkunitsResponse& resp);
     void getDropZoneInfoByIP(double clientVersion, const char* destIP, const char* destFile, StringBuffer& path, StringBuffer& mask);
     bool checkDropZoneIPAndPath(double clientVersion, const char* dropZone, const char* netAddr, const char* path);
-    void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* dropZonePath, StringBuffer& relativePath,
-        const char pathSep, IArrayOf<IEspPhysicalFileStruct>& filesInFolder, IArrayOf<IEspPhysicalFileStruct>&files);
-    bool searchDropZoneFileInFolder(IEspContext& context, IFile* f, const char* nameFilter, bool returnAll,
-        const char* dropZonePath, StringBuffer& relativePath, const char pathSep, IArrayOf<IEspPhysicalFileStruct>& files);
+    void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* path, IArrayOf<IEspPhysicalFileStruct>&files);
+    void searchDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files, unsigned& filesFound);
     void setDFUServerQueueReq(const char* dfuServerQueue, IDFUWorkUnit* wu);
     void setUserAuth(IEspContext &context, IDFUWorkUnit* wu);
-    void appendDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files);
 };
 
 #endif //_ESPWIZ_FileSpray_HPP__