HPCC-9559 ecl-packagemap-query-files command to show query file mappings

The "ecl packagemap query-files" command will list the files in
a given query and show if/how they are mapped to SuperFiles defined
in the active packagemap for the given cluster.  --pmid option allows
an inactive packagemap to be used instead.

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

common/workunit/package.h

@@ -27,6 +27,7 @@
 #define PACKAGE_MISSING_ID            PACKAGE_ERROR_START+1
 #define PACKAGE_NO_SUBFILES           PACKAGE_ERROR_START+2
 #define PACKAGE_NOT_FOUND             PACKAGE_ERROR_START+3
+#define PACKAGE_QUERY_NOT_FOUND           PACKAGE_ERROR_START+4
 
 
 interface IHpccPackage : extends IInterface
@@ -46,6 +47,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
     virtual bool validate(const char *queryid, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const = 0;
+    virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface

common/workunit/pkgimpl.hpp

@@ -367,6 +367,42 @@ public:
         load(getPackageMapById(id, true));
     }
 
+    virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const
+    {
+        Owned<IPropertyTree> query = resolveQueryAlias(querySet, queryname, true);
+        if (!query)
+            throw MakeStringException(PACKAGE_QUERY_NOT_FOUND, "Query %s not found", queryname);
+        Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL);
+        Owned<IWorkUnitFactory> wufactory = getWorkUnitFactory(NULL, NULL);
+        Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(query->queryProp("@wuid"), false);
+
+        const IHpccPackage *pkg = matchPackage(query->queryProp("@id"));
+        filelist->addFilesFromQuery(cw, pkg);
+        Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
+        ForEach(*refFiles)
+        {
+            IReferencedFile &rf = refFiles->query();
+            if (!(rf.getFlags() & RefFileInPackage))
+                fileInfo->addProp("File", rf.getLogicalName());
+            else
+            {
+                Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(rf.getLogicalName());
+                if (ssfe && ssfe->numSubFiles()>0)
+                {
+                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile", createPTree());
+                    superInfo->setProp("@name", rf.getLogicalName());
+                    unsigned count = ssfe->numSubFiles();
+                    while (count--)
+                    {
+                        StringBuffer subfile;
+                        ssfe->getSubFileName(count, subfile);
+                        superInfo->addProp("SubFile", subfile.str());
+                    }
+                }
+            }
+        }
+    }
+
     virtual bool validate(const char *queryToCheck, StringArray &warn, StringArray &err, 
         StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
     {

common/workunit/referencedfilelist.cpp

@@ -126,6 +126,7 @@ public:
     virtual void addFiles(StringArray &files);
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw);
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
+    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
 
     virtual IReferencedFileIterator *getFiles();
     virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo);
@@ -389,12 +390,8 @@ void ReferencedFileList::addFiles(StringArray &files)
         addFile(files.item(i));
 }
 
-void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)
+void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
 {
-    const IHpccPackage *pkg = NULL;
-    if (pm && queryid && *queryid)
-        pkg = pm->matchPackage(queryid);
-
     Owned<IConstWUGraphIterator> graphs = &cw->getGraphs(GraphTypeActivities);
     ForEach(*graphs)
     {
@@ -436,6 +433,14 @@ void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
     }
 }
 
+void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)
+{
+    const IHpccPackage *pkg = NULL;
+    if (pm && queryid && *queryid)
+        pkg = pm->matchPackage(queryid);
+    addFilesFromQuery(cw, pkg);
+}
+
 void ReferencedFileList::addFilesFromWorkUnit(IConstWorkUnit *cw)
 {
     addFilesFromQuery(cw, NULL, NULL);

common/workunit/referencedfilelist.hpp

@@ -49,6 +49,8 @@ interface IReferencedFileList : extends IInterface
 {
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw)=0;
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)=0;
+    virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
+
     virtual void addFile(const char *ln)=0;
     virtual void addFiles(StringArray &files)=0;
 

ecl/ecl-package/ecl-package.cpp

@@ -784,6 +784,131 @@ private:
     bool optValidateActive;
 };
 
+class EclCmdPackageQueryFiles : public EclCmdCommon
+{
+public:
+    EclCmdPackageQueryFiles()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else if (optQueryId.isEmpty())
+                    optQueryId.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (iter.matchOption(optPMID, ECLOPT_PMID) || iter.matchOption(optPMID, ECLOPT_PMID_S))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optTarget.isEmpty())
+            err.append("\n ... A target cluster must be specified\n\n");
+        if (optQueryId.isEmpty())
+            err.append("\n ... A query must be specified\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        Owned<IClientGetQueryFileMappingRequest> request = packageProcessClient->createGetQueryFileMappingRequest();
+
+        request->setTarget(optTarget);
+        request->setQueryName(optQueryId);
+        request->setPMID(optPMID);
+
+        Owned<IClientGetQueryFileMappingResponse> resp = packageProcessClient->GetQueryFileMapping(request);
+        if (resp->getExceptions().ordinality()>0)
+            outputMultiExceptions(resp->getExceptions());
+
+        StringArray &unmappedFiles = resp->getUnmappedFiles();
+        if (!unmappedFiles.ordinality())
+            fputs("No undefined files found.\n", stderr);
+        else
+        {
+            fputs("Files not defined in PackageMap:\n", stderr);
+            ForEachItemIn(i, unmappedFiles)
+                fprintf(stderr, "  %s\n", unmappedFiles.item(i));
+        }
+        IArrayOf<IConstSuperFile> &superFiles = resp->getSuperFiles();
+        if (!superFiles.ordinality())
+            fputs("\nNo matching SuperFiles found in PackageMap.\n", stderr);
+        else
+        {
+            fputs("\nSuperFiles defined in PackageMap:\n", stderr);
+            ForEachItemIn(i, superFiles)
+            {
+                IConstSuperFile &super = superFiles.item(i);
+                fprintf(stderr, "  %s\n", super.getName());
+                StringArray &subfiles = super.getSubFiles();
+                if (subfiles.ordinality()>0)
+                {
+                    ForEachItemIn(sbi, subfiles)
+                        fprintf(stderr, "   > %s\n", subfiles.item(sbi));
+                }
+            }
+        }
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'query-files' command will list the files referenced by a query, showing if/how they\n"
+                    "are mapped as SuperFiles in the active packagemap.  --pmid option allows an inactive\n"
+                    "packagemap to be used instead.\n"
+                    "\n"
+                    "ecl packagemap query-files <target> <queryid>\n"
+                    " Options:\n"
+                    "   <target>                    name of target to use when validating package map information\n"
+                    "   <queryid>                   name of query to get file mappings for\n"
+                    "   -pm, --pmid                 optional id of packagemap to validate, defaults to active\n",
+                    stdout);
+
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optTarget;
+    StringAttr optQueryId;
+    StringAttr optPMID;
+};
+
 IEclCommand *createPackageSubCommand(const char *cmdname)
 {
     if (!cmdname || !*cmdname)
@@ -802,6 +927,8 @@ IEclCommand *createPackageSubCommand(const char *cmdname)
         return new EclCmdPackageList();
     if (strieq(cmdname, "validate"))
         return new EclCmdPackageValidate();
+    if (strieq(cmdname, "query-files"))
+        return new EclCmdPackageQueryFiles();
 return NULL;
 }
 
@@ -827,6 +954,7 @@ public:
             "      list         list loaded package map names\n"
             "      info         return active package map information\n"
             "      validate     validate information in the package map file \n"
+            "      query-files  show files used by a query and if/how they are mapped\n"
         );
     }
 };

esp/scm/ws_packageprocess.ecm

@@ -146,6 +146,26 @@ ESPresponse [exceptions_inline] ValidatePackageResponse
     ESPstruct ValidatePackageFiles files;
 };
 
+ESPrequest GetQueryFileMappingRequest
+{
+    string Target;
+    string PMID;
+    string QueryName;
+};
+
+ESPstruct SuperFile
+{
+    string Name;
+    ESParray<string, File> SubFiles;
+};
+
+ESPresponse [exceptions_inline] GetQueryFileMappingResponse
+{
+    ESParray<string, File> UnmappedFiles;
+    ESParray<ESPstruct SuperFile> SuperFiles;
+};
+
+
 ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
@@ -156,6 +176,7 @@ ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("
     ESPmethod ListPackage(ListPackageRequest, ListPackageResponse);
     ESPmethod GetPackage(GetPackageRequest, GetPackageResponse);
     ESPmethod ValidatePackage(ValidatePackageRequest, ValidatePackageResponse);
+    ESPmethod GetQueryFileMapping(GetQueryFileMappingRequest, GetQueryFileMappingResponse);
 };
 
 SCMexportdef(WsPackageProcess);

esp/services/ws_packageprocess/packageprocess_errors.h

@@ -20,7 +20,7 @@
 
 #include "errorlist.h"
 
-#define PKG_NAME_EXISTS   PKG_PROCESS_ERROR_START
+#define PKG_NAME_EXISTS     PKG_PROCESS_ERROR_START
 #define PKG_MISSING_PARAM   PKG_PROCESS_ERROR_START+1
 #define PKG_DALI_LOOKUP_ERROR    PKG_PROCESS_ERROR_START+2
 #define PKG_MISSING_DALI_LOOKUP_IP  PKG_PROCESS_ERROR_START+3
@@ -32,5 +32,7 @@
 #define PKG_CREATE_PACKAGESET_FAILED   PKG_PROCESS_ERROR_START+9
 #define PKG_PACKAGEMAP_NOT_FOUND   PKG_PROCESS_ERROR_START+10
 #define PKG_LOAD_PACKAGEMAP_FAILED   PKG_PROCESS_ERROR_START+11
+#define PKG_INVALID_CLUSTER_TYPE   PKG_PROCESS_ERROR_START+12
+#define PKG_INVALID_QUERY_NAME   PKG_PROCESS_ERROR_START+13
 
 #endif

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -566,3 +566,69 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     resp.updateFiles().setUnmatched(unmatchedFiles);
     return true;
 }
+
+bool CWsPackageProcessEx::onGetQueryFileMapping(IEspContext &context, IEspGetQueryFileMappingRequest &req, IEspGetQueryFileMappingResponse &resp)
+{
+    const char *queryname = req.getQueryName();
+    if (!queryname || !*queryname)
+        throw MakeStringException(PKG_INVALID_QUERY_NAME, "Query name required");
+    const char *target = req.getTarget();
+    if (!target || !*target)
+        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
+
+    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+    if (!clusterInfo)
+        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Unable to find target cluster");
+    if (clusterInfo->getPlatform()!=RoxieCluster)
+        throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Roxie target required");
+
+    Owned<IHpccPackageSet> set;
+    Owned<IHpccPackageMap> ownedmap;
+    const IHpccPackageMap *map = NULL;
+
+    const char *pmid = req.getPMID();
+    if (pmid && *pmid)
+    {
+        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(pmid, true), target, pmid));
+        if (!ownedmap)
+            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
+        map = ownedmap;
+    }
+    else
+    {
+        SCMStringBuffer process;
+        clusterInfo->getRoxieProcess(process);
+        set.setown(createPackageSet(process.str()));
+        if (!set)
+            throw MakeStringException(PKG_CREATE_PACKAGESET_FAILED, "Unable to create PackageSet");
+        map = set->queryActiveMap(target);
+        if (!map)
+            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
+    }
+    Owned<IPropertyTree> fileInfo = createPTree();
+    map->gatherFileMappingForQuery(queryname, fileInfo);
+
+    StringArray unmappedFiles;
+    Owned<IPropertyTreeIterator> it = fileInfo->getElements("File");
+    ForEach(*it)
+        unmappedFiles.append(it->query().queryProp(NULL));
+    resp.setUnmappedFiles(unmappedFiles);
+
+    IArrayOf<IEspSuperFile> superArray;
+    it.setown(fileInfo->getElements("SuperFile"));
+    ForEach(*it)
+    {
+        IPropertyTree &superTree = it->query();
+        Owned<IEspSuperFile> superItem = createSuperFile();
+        superItem->setName(superTree.queryProp("@name"));
+        StringArray subArray;
+        Owned<IPropertyTreeIterator> subfiles = superTree.getElements("SubFile");
+        ForEach(*subfiles)
+            subArray.append(subfiles->query().queryProp(NULL));
+        superItem->setSubFiles(subArray);
+        superArray.append(*superItem.getClear());
+    }
+
+    resp.setSuperFiles(superArray);
+    return true;
+}

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -52,6 +52,7 @@ public:
     virtual bool onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp);
     virtual bool onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp);
     virtual bool onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp);
+    virtual bool onGetQueryFileMapping(IEspContext &context, IEspGetQueryFileMappingRequest &req, IEspGetQueryFileMappingResponse &resp);
 };
 
 #endif //_ESPWIZ_ws_packageprocess_HPP__

roxie/ccd/ccdstate.cpp

@@ -502,7 +502,10 @@ public:
     {
         return BASE::validate(queryid, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles);
     }
-
+    virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const
+    {
+        BASE::gatherFileMappingForQuery(queryname, fileInfo);
+    }
     virtual const IRoxiePackage *queryRoxiePackage(const char *name) const
     {
         return queryResolvedPackage(name);