HPCC-9545 ecl-packagemap-validate option to check subfiles exist in dfs

Adds a new --check-dfs option to ecl-packagemap-validate which will
verify that all subfiles in the packagemap are defined in the local
dfs.

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

common/workunit/package.cpp

@@ -183,20 +183,23 @@ void CHpccPackageSet::load(IPropertyTree *xml)
     }
 }
 
-const IHpccPackageMap *CHpccPackageSet::queryActiveMap(const char *queryset) const
+bool checkPackageMapMatchesTarget(const char *target, const char *match)
 {
+    if (!match || !*match)
+        return false;
+    if (isWildString(match))
+        return WildMatch(target, match);
+    return streq(target, match);
+}
+
+const IHpccPackageMap *CHpccPackageSet::queryActiveMap(const char *target) const
+{
+    if (!target || !*target)
+        return NULL;
     ForEachItemIn(i, packageMaps)
     {
         CHpccPackageMap &pm = packageMaps.item(i);
-        StringAttr &match = pm.querySet;
-        if (!match.length())
-            continue;
-        if (isWildString(match))
-        {
-            if (WildMatch(queryset, match))
-                return ±
-        }
-        else if (streq(queryset, match))
+        if (checkPackageMapMatchesTarget(target, pm.querySet.get()))
             return ±
     }
     return NULL;
@@ -219,6 +222,8 @@ extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool read
 
 extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly)
 {
+    if (!id || !*id)
+        return NULL;
     StringBuffer xpath;
     xpath.append("/PackageSets/PackageSet[@id=\"").append(id).append("\"]");
     Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
@@ -227,8 +232,27 @@ extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool read
     return conn->getRoot();
 }
 
+extern WORKUNIT_API IPropertyTree * resolveActivePackageMap(const char *process, const char *target, bool readonly)
+{
+    if (!target || !*target)
+        return NULL;
+    Owned<IPropertyTree> ps = resolvePackageSetRegistry(process, true);
+    if (!ps)
+        return NULL;
+    Owned<IPropertyTreeIterator> it = ps->getElements("PackageMap[@active='1']");
+    ForEach(*it)
+    {
+        IPropertyTree &pm = it->query();
+        if (checkPackageMapMatchesTarget(target, pm.queryProp("@querySet")))
+            return getPackageMapById(pm.queryProp("@id"), readonly);
+    }
+    return NULL;
+}
+
 extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly)
 {
+    if (!process || !*process)
+        return NULL;
     Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
     Owned<IPropertyTree> psroot = globalLock->getRoot();
 

common/workunit/package.h

@@ -62,6 +62,7 @@ extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process);
 extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly);
 extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly);
 extern WORKUNIT_API IPropertyTree * resolvePackageSetRegistry(const char *process, bool readonly);
+extern WORKUNIT_API IPropertyTree * resolveActivePackageMap(const char *process, const char *target, bool readonly);
 extern WORKUNIT_API hash64_t pkgHash64Data(size32_t len, const void *buf, hash64_t hval);
 
 

common/workunit/referencedfilelist.cpp

@@ -127,6 +127,7 @@ public:
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw);
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
+    virtual void addFilesFromPackageMap(IPropertyTree *pm);
 
     virtual IReferencedFileIterator *getFiles();
     virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo);
@@ -390,6 +391,13 @@ void ReferencedFileList::addFiles(StringArray &files)
         addFile(files.item(i));
 }
 
+void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)
+{
+    Owned<IPropertyTreeIterator> files = pm->getElements("Package/SuperFile/SubFile[@value]");
+    ForEach(*files)
+        addFile(files->query().queryProp("@value"));
+}
+
 void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
 {
     Owned<IConstWUGraphIterator> graphs = &cw->getGraphs(GraphTypeActivities);

common/workunit/referencedfilelist.hpp

@@ -50,6 +50,7 @@ interface IReferencedFileList : extends IInterface
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw)=0;
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid)=0;
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
+    virtual void addFilesFromPackageMap(IPropertyTree *pm)=0;
 
     virtual void addFile(const char *ln)=0;
     virtual void addFiles(StringArray &files)=0;

ecl/ecl-package/ecl-package.cpp

@@ -621,7 +621,7 @@ private:
 class EclCmdPackageValidate : public EclCmdCommon
 {
 public:
-    EclCmdPackageValidate() : optValidateActive(false)
+    EclCmdPackageValidate() : optValidateActive(false), optCheckDFS(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -650,6 +650,8 @@ public:
             }
             if (iter.matchFlag(optValidateActive, ECLOPT_ACTIVE))
                 continue;
+            if (iter.matchFlag(optCheckDFS, ECLOPT_CHECK_DFS))
+                continue;
             if (iter.matchOption(optPMID, ECLOPT_PMID) || iter.matchOption(optPMID, ECLOPT_PMID_S))
                 continue;
             if (iter.matchOption(optQueryId, ECLOPT_QUERYID))
@@ -706,6 +708,7 @@ public:
         request->setPMID(optPMID);
         request->setTarget(optTarget);
         request->setQueryIdToVerify(optQueryId);
+        request->setCheckDFS(optCheckDFS);
 
         bool validateMessages = false;
         Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
@@ -754,6 +757,14 @@ public:
                 fprintf(stderr, "      %s\n", unusedFiles.item(i));
         }
 
+        StringArray &notInDFS = resp->getFiles().getNotInDFS();
+        if (notInDFS.ordinality()>0)
+        {
+            fputs("\n   Packagemap SubFiles not found in DFS:\n", stderr);
+            ForEachItemIn(i, notInDFS)
+                fprintf(stderr, "      %s\n", notInDFS.item(i));
+        }
+
         if (!validateMessages)
             fputs("   Validation was successful\n", stdout);
 
@@ -771,6 +782,7 @@ public:
                     "   <target>                    name of target to use when validating package map information\n"
                     "   <filename>                  name of file containing package map information\n"
                     "   --active                    validate the active packagemap\n"
+                    "   --check-dfs                 verify that subfiles exist in DFS\n"
                     "   -pm, --pmid                 id of packagemap to validate\n",
                     stdout);
 
@@ -782,6 +794,7 @@ private:
     StringAttr optPMID;
     StringAttr optQueryId;
     bool optValidateActive;
+    bool optCheckDFS;
 };
 
 class EclCmdPackageQueryFiles : public EclCmdCommon

ecl/eclcmd/eclcmd_common.hpp

@@ -82,6 +82,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_DELETE_PREVIOUS_S "-dp"
 #define ECLOPT_DELETE_PREVIOUS_INI "deletePrevDefault"
 #define ECLOPT_DELETE_PREVIOUS_ENV "ACTIVATE_DELETE_PREVIOUS"
+#define ECLOPT_CHECK_DFS "--check-dfs"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

esp/scm/ws_packageprocess.ecm

@@ -118,6 +118,7 @@ ESPrequest ValidatePackageRequest
     bool Active;
     string PMID;
     string QueryIdToVerify;
+    bool CheckDFS;
 };
 
 ESPstruct ValidatePackageInfo
@@ -133,6 +134,7 @@ ESPstruct ValidatePackageQueries
 ESPstruct ValidatePackageFiles
 {
     ESParray<string> Unmatched;
+    ESParray<string, File> NotInDFS;
 };
 
 ESPresponse [exceptions_inline] ValidatePackageResponse

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -529,33 +529,60 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     StringArray unmatchedFiles;
 
     Owned<IHpccPackageSet> set;
-    Owned<IHpccPackageMap> ownedmap;
-    const IHpccPackageMap *map = NULL;
+    Owned<IPropertyTree> mapTree;
+
+    const char *target = req.getTarget();
+    if (!target || !*target)
+        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
+    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+    if (!clusterInfo)
+        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
+    SCMStringBuffer process;
+    clusterInfo->getRoxieProcess(process);
+    if (!process.length())
+        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
 
+    const char *pmid = req.getPMID();
     if (req.getActive()) //validate active map
     {
-        set.setown(createPackageSet("*"));
-        if (!set)
-            throw MakeStringException(PKG_CREATE_PACKAGESET_FAILED, "Unable to create PackageSet");
-        map = set->queryActiveMap(req.getTarget());
-        if (!map)
+        mapTree.setown(resolveActivePackageMap(process.str(), target, true));
+        if (!mapTree)
             throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
     }
-    else if (req.getPMID())
+    else if (pmid && *pmid)
     {
-        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(req.getPMID(), true), req.getTarget(), req.getPMID()));
-        if (!ownedmap)
-            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
-        map = ownedmap;
+        mapTree.setown(getPackageMapById(pmid, true));
+        if (!mapTree)
+            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", req.getPMID());
     }
     else
     {
-        ownedmap.setown(createPackageMapFromXml(req.getInfo(), req.getTarget(), NULL));
-        if (!ownedmap)
+        mapTree.setown(createPTreeFromXMLString(req.getInfo()));
+        if (!mapTree)
             throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
-        map = ownedmap;
     }
 
+    if (req.getCheckDFS())
+    {
+        Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword());
+        pmfiles->addFilesFromPackageMap(mapTree);
+        pmfiles->resolveFiles(process.str(), NULL, true, false);
+        Owned<IReferencedFileIterator> files = pmfiles->getFiles();
+        StringArray notInDFS;
+        ForEach(*files)
+        {
+            IReferencedFile &file = files->query();
+            if (file.getFlags() & RefFileNotFound)
+                notInDFS.append(file.getLogicalName());
+        }
+        resp.updateFiles().setNotInDFS(notInDFS);
+    }
+
+    const char *id = mapTree->queryProp("@id");
+    Owned<IHpccPackageMap> map = createPackageMapFromPtree(mapTree, target, id);
+    if (!map)
+        throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
+
     map->validate(req.getQueryIdToVerify(), warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles);
 
     resp.setPMID(map->queryPackageId());