Merge pull request #6024 from afishbeck/eclcmdListQueryFiles

HPCC-11435 ecl-queries "file" command to list files used by query

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/referencedfilelist.cpp

@@ -123,8 +123,8 @@ class ReferencedFile : public CInterface, implements IReferencedFile
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs)
-    : flags(_flags), pkgid(_pkgid), noDfsResolution(noDfs)
+    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize)
+    : flags(_flags), pkgid(_pkgid), noDfsResolution(noDfs), calcFileSize(calcSize), fileSize(0), numParts(0)
     {
         logicalName.set(skipForeign(lfn, &daliip)).toLowerCase();
         if (daliip.length())
@@ -165,6 +165,14 @@ public:
     virtual void cloneInfo(IDFUhelper *helper, IUserDescriptor *user, const char *dstCluster, const char *srcCluster, bool overwrite=false, bool cloneForeign=false);
     void cloneSuperInfo(ReferencedFileList *list, IUserDescriptor *user, INode *remote, bool overwrite);
     virtual const char *queryPackageId() const {return pkgid.get();}
+    virtual __int64 getFileSize()
+    {
+        return fileSize;
+    }
+    virtual unsigned getNumParts()
+    {
+        return numParts;
+    }
 
 public:
     StringBuffer logicalName;
@@ -172,15 +180,19 @@ public:
     StringBuffer daliip;
     StringBuffer filePrefix;
     StringAttr fileSrcCluster;
+    __int64 fileSize;
+    unsigned numParts;
     unsigned flags;
     bool noDfsResolution;
+    bool calcFileSize;
 };
 
 class ReferencedFileList : public CInterface, implements IReferencedFileList
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFileList(const char *username, const char *pw, bool allowForeignFiles) : allowForeign(allowForeignFiles)
+    ReferencedFileList(const char *username, const char *pw, bool allowForeignFiles, bool allowFileSizeCalc)
+        : allowForeign(allowForeignFiles), allowSizeCalc(allowFileSizeCalc)
     {
         if (username && pw)
         {
@@ -189,7 +201,8 @@ public:
         }
     }
 
-    ReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles) : allowForeign(allowForeignFiles)
+    ReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles, bool allowFileSizeCalc)
+        : allowForeign(allowForeignFiles), allowSizeCalc(allowFileSizeCalc)
     {
         if (userDesc)
             user.set(userDesc);
@@ -227,6 +240,7 @@ public:
     StringAttr srcCluster;
     StringAttr remotePrefix;
     bool allowForeign;
+    bool allowSizeCalc;
 };
 
 void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstCluster, const char *srcCluster, StringArray *subfiles)
@@ -259,6 +273,8 @@ void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstC
         if (srcCluster && *srcCluster)
             if (NotFound == df->findCluster(srcCluster))
                 flags |= RefFileNotOnSource;
+        fileSize = df->getFileSize(calcFileSize, false);
+        numParts = df->numParts();
     }
 }
 
@@ -283,6 +299,8 @@ void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcC
         VStringBuffer xpath("Cluster[@name='%s']", srcCluster);
         if (!tree->hasProp(xpath))
             flags |= RefFileNotOnSource;
+        numParts = tree->getPropInt("@numparts", 0);
+        fileSize = tree->getPropInt64("Attr/@size", 0);
     }
 
 }
@@ -506,7 +524,7 @@ void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *
     if (!allowForeign && checkForeign(ln))
         throw MakeStringException(-1, "Foreign file not allowed%s: %s", (flags & RefFileInPackage) ? " (declared in package)" : "", ln);
 
-    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution);
+    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution, allowSizeCalc);
     if (!file->logicalName.length())
         return;
     ReferencedFile *existing = map.getValue(file->getLogicalName());
@@ -646,7 +664,7 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
         if (!allowForeign && checkForeign(lfn))
             throw MakeStringException(-1, "Foreign sub file not allowed: %s", lfn);
 
-        Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL, false);
+        Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL, false, allowSizeCalc);
         if (file->logicalName.length() && !map.getValue(file->getLogicalName()))
         {
             file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, resolveForeign);
@@ -725,12 +743,12 @@ IReferencedFileIterator *ReferencedFileList::getFiles()
     return new ReferencedFileIterator(this);
 }
 
-IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles)
+IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles, bool allowFileSizeCalc)
 {
-    return new ReferencedFileList(user, pw, allowForeignFiles);
+    return new ReferencedFileList(user, pw, allowForeignFiles, allowFileSizeCalc);
 }
 
-IReferencedFileList *createReferencedFileList(IUserDescriptor *user, bool allowForeignFiles)
+IReferencedFileList *createReferencedFileList(IUserDescriptor *user, bool allowForeignFiles, bool allowFileSizeCalc)
 {
-    return new ReferencedFileList(user, allowForeignFiles);
+    return new ReferencedFileList(user, allowForeignFiles, allowFileSizeCalc);
 }

common/workunit/referencedfilelist.hpp

@@ -42,6 +42,8 @@ interface IReferencedFile : extends IInterface
     virtual unsigned getFlags() const =0;
     virtual const SocketEndpoint &getForeignIP(SocketEndpoint &ep) const =0;
     virtual const char *queryPackageId() const =0;
+    virtual __int64 getFileSize()=0;
+    virtual unsigned getNumParts()=0;
 };
 
 interface IReferencedFileIterator : extends IIteratorOf<IReferencedFile> { };
@@ -65,8 +67,8 @@ interface IReferencedFileList : extends IInterface
 
 extern WORKUNIT_API const char *skipForeign(const char *name, StringBuffer *ip=NULL);
 
-extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles);
-extern WORKUNIT_API IReferencedFileList *createReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles);
+extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles, bool allowFileSizeCalc);
+extern WORKUNIT_API IReferencedFileList *createReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles, bool allowFileSizeCalc);
 
 extern WORKUNIT_API void splitDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster);
 extern WORKUNIT_API void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix);

ecl/eclcmd/queries/ecl-queries.cpp

@@ -300,6 +300,108 @@ private:
     bool optInactive;
 };
 
+class EclCmdQueryFiles : public EclCmdCommon
+{
+public:
+    EclCmdQueryFiles()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else if (optQuery.isEmpty())
+                    optQuery.set(arg);
+                else
+                {
+                    fprintf(stderr, "\n%s option not recognized\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (optTarget.isEmpty())
+        {
+            fputs("Target must be specified.\n\n", stderr);
+            return false;
+        }
+        if (optTarget.isEmpty())
+        {
+            fputs("Query must be specified.\n\n", stderr);
+            return false;
+        }
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        return true;
+    }
+
+    virtual int processCMD()
+    {
+        Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
+        Owned<IClientWUQueryFilesRequest> req = client->createWUQueryFilesRequest();
+        req->setTarget(optTarget.get());
+        req->setQueryId(optQuery.get());
+
+        Owned<IClientWUQueryFilesResponse> resp = client->WUQueryFiles(req);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+        else
+        {
+            IArrayOf<IConstFileUsedByQuery> &files = resp->getFiles();
+            if (!files.length())
+                fputs("No files used.\n", stdout);
+            else
+                fputs("Files used:\n", stdout);
+            ForEachItemIn(i, files)
+            {
+                IConstFileUsedByQuery &file = files.item(i);
+                StringBuffer line("  ");
+                line.append(file.getFileName()).append(", ");
+                line.append(file.getFileSize()).append(" bytes, ");
+                line.append(file.getNumberOfParts()).append(" part(s)\n");
+                fputs(line, stdout);
+            }
+            fputs("\n", stdout);
+        }
+        return 0;
+    }
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+            "\n"
+            "The 'queries files' command displays a list of the files currently in use by\n"
+            "the given query.\n"
+            "\n"
+            "ecl queries files <target> <query>\n\n"
+            " Options:\n"
+            "   <target>               Name of target cluster the query is published on\n"
+            "   <query>                Name of the query to get a list of files in use by\n"
+            " Common Options:\n",
+            stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optTarget;
+    StringAttr optQuery;
+};
+
 class EclCmdQueriesCopy : public EclCmdCommon
 {
 public:
@@ -771,6 +873,8 @@ IEclCommand *createEclQueriesCommand(const char *cmdname)
         return NULL;
     if (strieq(cmdname, "list"))
         return new EclCmdQueriesList();
+    if (strieq(cmdname, "files"))
+        return new EclCmdQueryFiles();
     if (strieq(cmdname, "config"))
         return new EclCmdQueriesConfig();
     if (strieq(cmdname, "copy"))
@@ -796,6 +900,7 @@ public:
             "ecl queries <command> [command options]\n\n"
             "   Queries Commands:\n"
             "      list         list queries on target cluster(s)\n"
+            "      files        list the files currently used by a query\n"
             "      config       update query settings\n"
             "      copy         copy a query from one target cluster to another\n"
             "      copy-set     copy queries from one target cluster to another\n"

esp/scm/ws_workunits.ecm

@@ -1303,6 +1303,24 @@ ESPresponse [exceptions_inline] WUListQueriesUsingFileResponse
     ESParray<ESPstruct TargetQueriesUsingFile> Targets;
 };
 
+ESPrequest [nil_remove] WUQueryFilesRequest
+{
+    string Target;
+    string QueryId;
+};
+
+ESPStruct [nil_remove] FileUsedByQuery
+{
+    string FileName;
+    int64 FileSize;
+    unsigned NumberOfParts;
+};
+
+ESPresponse [exceptions_inline] WUQueryFilesResponse
+{
+    ESParray<ESPstruct FileUsedByQuery, File> Files;
+};
+
 ESPrequest WUQueryDetailsRequest
 {
     string QueryId;
@@ -1607,6 +1625,7 @@ ESPservice [
     ESPmethod [resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
     ESPmethod WUQueryConfig(WUQueryConfigRequest, WUQueryConfigResponse);
     ESPmethod WUListQueries(WUListQueriesRequest, WUListQueriesResponse);
+    ESPmethod WUQueryFiles(WUQueryFilesRequest, WUQueryFilesResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/QueriesUsingFile.xslt")] WUListQueriesUsingFile(WUListQueriesUsingFileRequest, WUListQueriesUsingFileResponse);
     ESPmethod WUCreateZAPInfo(WUCreateZAPInfoRequest, WUCreateZAPInfoResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUZAPInfoForm.xslt")] WUGetZAPInfo(WUGetZAPInfoRequest, WUGetZAPInfoResponse);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -134,7 +134,7 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
         userdesc->getPassword(password);
     }
 
-    Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles);
+    Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles, false);
     wufiles->addFilesFromPackageMap(packageMap);
     SCMStringBuffer processName;
     dstInfo->getRoxieProcess(processName);
@@ -787,7 +787,7 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
 
     if (req.getCheckDFS())
     {
-        Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true);
+        Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true, false);
         pmfiles->addFilesFromPackageMap(mapTree);
         pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -343,7 +343,7 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
             queryTree->setProp("@pkgid", pkgid);
 
         IUserDescriptor **roxieUser = roxieUserMap.getValue(target);
-        Owned<IReferencedFileList> wufiles = createReferencedFileList(roxieUser ? *roxieUser : NULL, true);
+        Owned<IReferencedFileList> wufiles = createReferencedFileList(roxieUser ? *roxieUser : NULL, true, true);
         wufiles->addFilesFromQuery(cw, pm, queryid);
         if (aborting)
             return;
@@ -372,6 +372,10 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
                 const char *fpkgid = rf.queryPackageId();
                 if (fpkgid && *fpkgid)
                     fileTree->setProp("@pkgid", fpkgid);
+                if (rf.getFileSize())
+                    fileTree->setPropInt64("@size", rf.getFileSize());
+                if (rf.getNumParts())
+                    fileTree->setPropInt("@numparts", rf.getNumParts());
             }
         }
     }
@@ -664,7 +668,7 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         clusterInfo->getRoxieProcess(process);
         if (!process.length())
             return;
-        Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), allowForeignFiles);
+        Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), allowForeignFiles, false);
         Owned<IHpccPackageSet> ps = createPackageSet(process.str());
         StringBuffer queryid;
         if (queryname && *queryname)
@@ -1403,6 +1407,44 @@ bool CWsWorkunitsEx::onWUListQueriesUsingFile(IEspContext &context, IEspWUListQu
     return true;
 }
 
+bool CWsWorkunitsEx::onWUQueryFiles(IEspContext &context, IEspWUQueryFilesRequest &req, IEspWUQueryFilesResponse &resp)
+{
+    const char *target = req.getTarget();
+    const char *query = req.getQueryId();
+    if (!target || !*target)
+        throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Target not specified");
+    if (!isValidCluster(target))
+        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid target name: %s", target);
+    if (!query || !*query)
+        throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not specified");
+    Owned<IPropertyTree> registeredQuery = resolveQueryAlias(target, query, true);
+    if (!registeredQuery)
+        throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found");
+    StringAttr queryid(registeredQuery->queryProp("@id"));
+    registeredQuery.clear();
+
+    Owned<IPropertyTree> tree = filesInUse.getTree();
+    VStringBuffer xpath("%s/Query[@id='%s']", target, queryid.get());
+    IPropertyTree *queryTree = tree->queryPropTree(xpath);
+    if (!queryTree)
+       throw MakeStringException(ECLWATCH_QUERYID_NOT_FOUND, "Query not found in file cache (%s)", xpath.str());
+
+    IArrayOf<IEspFileUsedByQuery> referencedFiles;
+    Owned<IPropertyTreeIterator> files = queryTree->getElements("File");
+    ForEach(*files)
+    {
+        IPropertyTree &file = files->query();
+        if (file.getPropBool("@super", 0))
+            continue;
+        Owned<IEspFileUsedByQuery> respFile = createFileUsedByQuery();
+        respFile->setFileName(file.queryProp("@lfn"));
+        respFile->setFileSize(file.getPropInt64("@size"));
+        respFile->setNumberOfParts(file.getPropInt("@numparts"));
+        referencedFiles.append(*respFile.getClear());
+    }
+    resp.setFiles(referencedFiles);
+    return true;
+}
 
 bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRequest & req, IEspWUQueryDetailsResponse & resp)
 {
@@ -1920,7 +1962,7 @@ public:
         cloneFilesEnabled = true;
         overwriteDfs = _overwriteDfs;
         splitDerivedDfsLocation(dfsServer, srcCluster, dfsIP, srcPrefix, sourceProcess, sourceProcess, NULL, NULL);
-        wufiles.setown(createReferencedFileList(context->queryUserId(), context->queryPassword(), allowForeign));
+        wufiles.setown(createReferencedFileList(context->queryUserId(), context->queryPassword(), allowForeign, false));
         Owned<IHpccPackageSet> ps = createPackageSet(destProcess);
         pm.set(ps->queryActiveMap(target));
         process.set(destProcess);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -193,6 +193,7 @@ public:
     bool onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRequest & req, IEspWUQueryDetailsResponse & resp);
     bool onWUListQueries(IEspContext &context, IEspWUListQueriesRequest &req, IEspWUListQueriesResponse &resp);
     bool onWUListQueriesUsingFile(IEspContext &context, IEspWUListQueriesUsingFileRequest &req, IEspWUListQueriesUsingFileResponse &resp);
+    bool onWUQueryFiles(IEspContext &context, IEspWUQueryFilesRequest &req, IEspWUQueryFilesResponse &resp);
 
     bool onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEspWUInfoResponse &resp);
     bool onWUInfoDetails(IEspContext &context, IEspWUInfoRequest &req, IEspWUInfoResponse &resp);