HPCC-2839 Command line tool to validate package content

Adds a WsPackageProcess web service method "ValidatePackage" and an
"ecl packagemap validate" command line option to validate a packagemap
file.

Checks some common definition errors, and (based on a specified
target cluster) will check for unused package definitions and unmatched
queries.

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

common/workunit/package.cpp

@@ -98,7 +98,7 @@ void CPackageNode::loadEnvironment()
         {
             StringBuffer s;
             toXML(&env, s);
-            throw MakeStringException(0, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
+            throw MakeStringException(PACKAGE_MISSING_ID, "PACKAGE_ERROR: Environment element missing id or value: %s", s.str());
         }
     }
     Owned<IAttributeIterator> attrs = node->getAttributes();
@@ -110,6 +110,28 @@ void CPackageNode::loadEnvironment()
     }
 }
 
+bool CPackageNode::validate(IMultiException *me) const
+{
+    if (!node)
+        return true;
+    const char *packageId = node->queryProp("@id");
+    if (!packageId || !*packageId)
+        me->append(*MakeStringExceptionDirect(PACKAGE_MISSING_ID, "Package has no id attribute"));
+    Owned<IPropertyTreeIterator> files = node->getElements("SuperFile");
+    ForEach(*files)
+    {
+        IPropertyTree &super = files->query();
+        const char *superId = super.queryProp("@id");
+        if (!superId || !*superId)
+            me->append(*MakeStringExceptionDirect(PACKAGE_MISSING_ID, "SuperFile has no id attribute"));
+
+        if (!super.hasProp("SubFile"))
+            me->append(*MakeStringException(PACKAGE_NO_SUBFILES, "Warning: Package['%s']/SuperFile['%s'] has no SubFiles defined", packageId, superId ? superId : ""));
+    }
+    return true;
+}
+
+
 CHpccPackage *createPackage(IPropertyTree *p)
 {
     return new CHpccPackage(p);
@@ -119,6 +141,20 @@ CHpccPackage *createPackage(IPropertyTree *p)
 // CPackageMap - an implementation of IPackageMap using a string map
 //================================================================================================
 
+IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryset, const char *id)
+{
+    Owned<CHpccPackageMap> pm = new CHpccPackageMap(id, queryset, true);
+    pm->load(t);
+    return pm.getClear();
+}
+
+IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id)
+{
+    Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
+    return createPackageMapFromPtree(t, queryset, id);
+}
+
+
 //================================================================================================
 // CHpccPackageSet - an implementation of IHpccPackageSet
 //================================================================================================

common/workunit/package.h

@@ -18,9 +18,17 @@
 #ifndef WUPACKAGE_H
 #define WUPACKAGE_H
 
+#include "errorlist.h"
 #include "dadfs.hpp"
 #include "workunit.hpp"
 
+// error codes
+#define PACKAGE_TARGET_NOT_FOUND      PACKAGE_ERROR_START
+#define PACKAGE_MISSING_ID            PACKAGE_ERROR_START+1
+#define PACKAGE_NO_SUBFILES           PACKAGE_ERROR_START+2
+#define PACKAGE_NOT_FOUND             PACKAGE_ERROR_START+3
+
+
 interface IHpccPackage : extends IInterface
 {
     virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const = 0;
@@ -37,6 +45,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
+    virtual bool validate(IMultiException *me, StringArray &unmatchedQueries, StringArray &unusedPackages) const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface
@@ -44,6 +53,9 @@ interface IHpccPackageSet : extends IInterface
      virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const = 0;
 };
 
+extern WORKUNIT_API IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id);
+extern WORKUNIT_API IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryset, const char *id);
+
 extern WORKUNIT_API IHpccPackageSet *createPackageSet(const char *process);
 extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool readonly);
 extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly);

common/workunit/pkgimpl.hpp

@@ -115,7 +115,6 @@ protected:
         return NULL;
     }
 
-
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -149,6 +148,7 @@ public:
             return NULL;
         return node->getPropTree("QuerySets");
     }
+    virtual bool validate(IMultiException *me) const;
 };
 
 enum baseResolutionState
@@ -207,10 +207,10 @@ public:
                     IPropertyTree &baseElem = baseIterator->query();
                     const char *baseId = baseElem.queryProp("@id");
                     if (!baseId)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base element missing id attribute");
+                        throw MakeStringException(PACKAGE_MISSING_ID, "PACKAGE_ERROR: base element missing id attribute");
                     const IHpccPackage *base = packages->queryPackage(baseId);
                     if (!base)
-                        throw MakeStringException(0, "PACKAGE_ERROR: base package %s not found", baseId);
+                        throw MakeStringException(PACKAGE_NOT_FOUND, "PACKAGE_ERROR: base package %s not found", baseId);
                     CResolvedPackage<TYPE> *rebase = dynamic_cast<CResolvedPackage<TYPE> *>(const_cast<IHpccPackage *>(base));
                     rebase->resolveBases(packages);
                     appendBase(base);
@@ -255,6 +255,11 @@ public:
 
         return false;
     }
+
+    virtual bool validate(IMultiException *me) const
+    {
+        return TYPE::validate(me);
+    }
 };
 
 
@@ -333,7 +338,7 @@ public:
             IPropertyTree &packageTree = allpackages->query();
             const char *id = packageTree.queryProp("@id");
             if (!id || !*id)
-                throw MakeStringException(-1, "Invalid package map - Package element missing id attribute");
+                throw MakeStringException(PACKAGE_MISSING_ID, "Invalid package map - Package element missing id attribute");
             Owned<packageType> package = new packageType(&packageTree);
             packages.setValue(id, package.get());
             package->loadEnvironment();
@@ -357,6 +362,56 @@ public:
     {
         load(getPackageMapById(id, true));
     }
+
+    virtual bool validate(IMultiException *me, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    {
+        bool isValid = true;
+        MapStringTo<bool> referencedPackages;
+        Owned<IPropertyTree> qs = getQueryRegistry(querySet, true);
+        if (!qs)
+            throw MakeStringException(PACKAGE_TARGET_NOT_FOUND, "Target %s not found", querySet.sget());
+        Owned<IPropertyTreeIterator> queries = qs->getElements("Query");
+        HashIterator it(packages);
+        ForEach (it)
+        {
+            const char *packageId = (const char *)it.query().getKey();
+            packageType *pkg = packages.getValue(packageId);
+            if (!pkg)
+                continue;
+            if (!pkg->validate(me))
+                isValid = false;
+            Owned<IPropertyTreeIterator> baseNodes = pkg->queryTree()->getElements("Base");
+            ForEach(*baseNodes)
+            {
+                const char *baseId = baseNodes->query().queryProp("@id");
+                if (!referencedPackages.getValue(baseId))
+                    referencedPackages.setValue(baseId, true);
+            }
+        }
+        ForEach(*queries)
+        {
+            const char *queryid = queries->query().queryProp("@id");
+            if (queryid && *queryid)
+            {
+                const IHpccPackage *matched = matchPackage(queryid);
+                if (matched)
+                {
+                    const char *matchId = matched->queryTree()->queryProp("@id");
+                    if (!referencedPackages.getValue(matchId))
+                        referencedPackages.setValue(matchId, true);
+                }
+                else
+                    unmatchedQueries.append(queryid);
+            }
+        }
+        ForEach (it)
+        {
+            const char *packageId = (const char *)it.query().getKey();
+            if (!referencedPackages.getValue(packageId))
+                unusedPackages.append(packageId);
+        }
+        return isValid;
+    }
 };
 
 typedef CPackageMapOf<CPackageNode, IHpccPackage> CHpccPackageMap;

ecl/ecl-package/ecl-package.cpp

@@ -281,8 +281,8 @@ public:
             {
                 IConstPackageListMapData& req = pkgMapInfo.item(i);
                 printf("\nPackage Name = %s  active = %d\n", req.getId(), req.getActive());
-                IArrayOf<IConstPackageListData> &pkgInfo = req.getPkgListData();
 
+                IArrayOf<IConstPackageListData> &pkgInfo = req.getPkgListData();
                 unsigned int numPkgs = pkgInfo.ordinality();
                 for (unsigned int j = 0; j <numPkgs; j++)
                 {
@@ -600,6 +600,115 @@ private:
     StringAttr optDaliIP;
 };
 
+class EclCmdPackageValidate : public EclCmdCommon
+{
+public:
+    EclCmdPackageValidate()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else if (optFileName.isEmpty())
+                    optFileName.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n\n");
+        else if (optTarget.isEmpty())
+            err.append("\n ... Specify a cluster name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        StringBuffer pkgInfo;
+        pkgInfo.loadFile(optFileName);
+
+        fprintf(stdout, "\nvalidating packagemap file %s\n\n", optFileName.sget());
+
+        Owned<IClientValidatePackageRequest> request = packageProcessClient->createValidatePackageRequest();
+        request->setInfo(pkgInfo);
+        request->setTarget(optTarget);
+
+        Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
+        if (resp->getExceptions().ordinality()==0)
+            fputs("   No errors found\n", stdout);
+        else
+            outputMultiExceptions(resp->getExceptions());
+        StringArray &unmatchedQueries = resp->getQueries().getUnmatched();
+        if (unmatchedQueries.ordinality()>0)
+        {
+            fputs("\n   Queries without matching package:\n", stderr);
+            ForEachItemIn(i, unmatchedQueries)
+                fprintf(stderr, "      %s\n", unmatchedQueries.item(i));
+        }
+        StringArray &unusedPackages = resp->getPackages().getUnmatched();
+        if (unusedPackages.ordinality()>0)
+        {
+            fputs("\n   Packages without matching queries:\n", stderr);
+            ForEachItemIn(i, unusedPackages)
+                fprintf(stderr, "      %s\n", unusedPackages.item(i));
+        }
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'validate' command will checkout the contents of the package map file \n"
+                    "\n"
+                    "ecl packagemap validate <target> <filename>\n"
+                    " Options:\n"
+                    "   <target>                    name of target to use when adding package map information\n"
+                    "   <filename>                  name of file containing package map information\n",
+                    stdout);
+
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optFileName;
+    StringAttr optTarget;
+};
 
 IEclCommand *createPackageSubCommand(const char *cmdname)
 {
@@ -617,7 +726,9 @@ IEclCommand *createPackageSubCommand(const char *cmdname)
         return new EclCmdPackageInfo();
     if (strieq(cmdname, "list"))
         return new EclCmdPackageList();
-    return NULL;
+    if (strieq(cmdname, "validate"))
+        return new EclCmdPackageValidate();
+return NULL;
 }
 
 //=========================================================================================
@@ -641,6 +752,7 @@ public:
             "      deactivate   deactivate a package map (package map will not get loaded)\n"
             "      list         list loaded package map names\n"
             "      info         return active package map information\n"
+            "      validate     validate information in the package map file \n"
         );
     }
 };

esp/scm/ws_packageprocess.ecm

@@ -110,6 +110,29 @@ ESPresponse [exceptions_inline] ListPackageResponse
     ESParray<ESPstruct PackageListMapData> PkgListMapData;
 };
 
+ESPrequest ValidatePackageRequest
+{
+    string Info;
+    string Target;
+};
+
+ESPstruct ValidatePackageInfo
+{
+    ESParray<string> Unmatched;
+};
+
+ESPstruct ValidatePackageQueries
+{
+    ESParray<string> Unmatched;
+};
+
+ESPresponse [exceptions_inline] ValidatePackageResponse
+{
+    ESPstruct BasePackageStatus status;
+    ESPstruct ValidatePackageInfo packages;
+    ESPstruct ValidatePackageQueries queries;
+};
+
 ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
@@ -119,6 +142,7 @@ ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("
     ESPmethod DeActivatePackage(DeActivatePackageRequest, DeActivatePackageResponse);
     ESPmethod ListPackage(ListPackageRequest, ListPackageResponse);
     ESPmethod GetPackage(GetPackageRequest, GetPackageResponse);
+    ESPmethod ValidatePackage(ValidatePackageRequest, ValidatePackageResponse);
 };
 
 SCMexportdef(WsPackageProcess);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -26,6 +26,7 @@
 #include "ws_workunits.hpp"
 #include "packageprocess_errors.h"
 #include "referencedfilelist.hpp"
+#include "package.h"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -525,3 +526,21 @@ bool CWsPackageProcessEx::onGetPackage(IEspContext &context, IEspGetPackageReque
     resp.setInfo(info);
     return true;
 }
+
+bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp)
+{
+    Owned<IHpccPackageMap> map = createPackageMapFromXml(req.getInfo(), req.getTarget(), NULL);
+    Owned<IMultiException> me = MakeMultiException("PackageMap");
+    StringArray unmatchedQueries;
+    StringArray unusedPackages;
+    map->validate(me, unmatchedQueries, unusedPackages);
+    resp.updateQueries().setUnmatched(unmatchedQueries);
+    resp.updatePackages().setUnmatched(unusedPackages);
+    if (me->ordinality()>0)
+    {
+        IArrayOf<IException>& exceptions = me->getArray();
+        ForEachItemIn(i, exceptions)
+            resp.noteException(*LINK(&exceptions.item(i)));
+    }
+    return true;
+}

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -51,6 +51,7 @@ public:
     virtual bool onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp);
     virtual bool onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp);
     virtual bool onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp);
+    virtual bool onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp);
 };
 
 #endif //_ESPWIZ_ws_packageprocess_HPP__

roxie/ccd/ccdstate.cpp

@@ -192,28 +192,6 @@ protected:
     virtual aindex_t getBaseCount() const = 0;
     virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
 
-    //map ambiguous IHpccPackage
-    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
-    {
-        return CPackageNode::resolveSuperFile(superFileName);
-    }
-    virtual const char *queryEnv(const char *varname) const
-    {
-        return CPackageNode::queryEnv(varname);
-    }
-    virtual bool getEnableFieldTranslation() const
-    {
-        return CPackageNode::getEnableFieldTranslation();
-    }
-    virtual const IPropertyTree *queryTree() const
-    {
-        return CPackageNode::queryTree();
-    }
-    virtual hash64_t queryHash() const
-    {
-        return CPackageNode::queryHash();
-    }
-
     virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
 
     // Add a filename and the corresponding IResolvedFile to the cache
@@ -467,6 +445,28 @@ public:
         else
             return NULL;
     }
+
+    //map ambiguous IHpccPackage
+    virtual ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const
+    {
+        return CPackageNode::resolveSuperFile(superFileName);
+    }
+    virtual const char *queryEnv(const char *varname) const
+    {
+        return CPackageNode::queryEnv(varname);
+    }
+    virtual bool getEnableFieldTranslation() const
+    {
+        return CPackageNode::getEnableFieldTranslation();
+    }
+    virtual const IPropertyTree *queryTree() const
+    {
+        return CPackageNode::queryTree();
+    }
+    virtual hash64_t queryHash() const
+    {
+        return CPackageNode::queryHash();
+    }
 };
 
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
@@ -512,6 +512,10 @@ public:
     {
         return BASE::isActive();
     }
+    virtual bool validate(IMultiException *me, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    {
+        return BASE::validate(me, unmatchedQueries, unusedPackages);
+    }
 
     virtual const IRoxiePackage *queryRoxiePackage(const char *name) const
     {

system/include/errorlist.h

@@ -45,6 +45,9 @@
 #define WORKUNIT_ERROR_START    5000
 #define WORKUNIT_ERROR_END      5099
 
+#define PACKAGE_ERROR_START    5200
+#define PACKAGE_ERROR_END      5299
+
 #define WUWEB_ERROR_START       5500
 #define WUWEB_ERROR_END         5599