Merge branch 'candidate-7.2.x' into candidate-7.4.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/package.h

@@ -52,7 +52,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
-    virtual bool validate(StringArray &queriesToVerify, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const = 0;
+    virtual bool validate(const StringArray &queriesToVerify, const StringArray &queriesToIgnore, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const = 0;
     virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const = 0;
     virtual const StringArray &getPartIds() const = 0;
 };

common/workunit/pkgimpl.hpp

@@ -457,8 +457,27 @@ public:
         }
     }
 
-    virtual bool validate(StringArray &queriesToCheck, StringArray &warn, StringArray &err, 
-        StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
+    bool checkIgnoreQuery(const StringArray &queriesToIgnore, const char *queryid) const
+    {
+        ForEachItemIn(i, queriesToIgnore)
+        {
+            const char *match = queriesToIgnore.item(i);
+            if (match && *match)
+            {
+                if (containsWildcard(match))
+                {
+                    if (WildMatch(queryid, match, true))
+                        return true;
+                }
+                else if (streq(match, queryid))
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    virtual bool validate(const StringArray &queriesToCheck, const StringArray &queriesToIgnore, StringArray &warn, StringArray &err,
+        StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const
     {
         bool isValid = true;
         MapStringTo<bool> referencedPackages;
@@ -520,6 +539,8 @@ public:
             const char *queryid = queries->query().queryProp("@id");
             if (queryid && *queryid)
             {
+                if (checkIgnoreQuery(queriesToIgnore, queryid))
+                    continue;
                 Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, true, false);
                 Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(queries->query().queryProp("@wuid"));
 
@@ -569,7 +590,11 @@ public:
                     }
                     VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
                     if (!(flags & RefFileNotOptional))
+                    {
+                        if (ignoreOptionalFiles)
+                            continue;
                         fullname.append("/Optional");
+                    }
                     else if (isCompulsory)
                         fullname.append("/Compulsory");
                     unmatchedFiles.append(fullname);

ecl/ecl-package/ecl-package.cpp

@@ -843,12 +843,21 @@ public:
                 continue;
             if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
                 continue;
+            if (iter.matchFlag(optIgnoreWarnings, ECLOPT_IGNORE_WARNINGS))
+                continue;
+            if (iter.matchFlag(optIgnoreOptionalFiles, ECLOPT_IGNORE_OPTIONAL))
+                continue;
             StringAttr queryIds;
             if (iter.matchOption(queryIds, ECLOPT_QUERYID))
             {
                 optQueryIds.appendList(queryIds.get(), ",");
                 continue;
             }
+            if (iter.matchOption(queryIds, ECLOPT_IGNORE_QUERIES))
+            {
+                optIgnoreQueries.appendList(queryIds.get(), ",");
+                continue;
+            }
             eclCmdOptionMatchIndicator ind = EclCmdCommon::matchCommandLineOption(iter, true);
             if (ind != EclCmdOptionMatch)
                 return ind;
@@ -928,8 +937,11 @@ public:
         request->setPMID(optPMID);
         request->setTarget(optTarget);
         request->setQueriesToVerify(optQueryIds);
+        request->setQueriesToIgnore(optIgnoreQueries);
         request->setCheckDFS(optCheckDFS);
         request->setGlobalScope(optGlobalScope);
+        request->setIgnoreWarnings(optIgnoreWarnings);
+        request->setIgnoreOptionalFiles(optIgnoreOptionalFiles);
 
         bool validateMessages = false;
         Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
@@ -1025,6 +1037,11 @@ public:
                     "   --queryid                   Query to verify against packagemap, multiple queries can be\n"
                     "                               specified using a comma separated list, or by using --queryid\n"
                     "                               more than once. Default is all queries in the target queryset\n"
+                    "   --ignore-queries            Queries to exclude from verification, multiple queries can be\n"
+                    "                               specified using wildcards, a comma separated list, or by using\n"
+                    "                               --ignore-queries more than once.\n"
+                    "   --ignore-optional           Doesn't warn when optional files are not defined in packagemap.\n"
+                    "   --ignore-warnings           Doesn't output general packagemap warnings.\n"
                     "   --global-scope              The specified packagemap can be shared across multiple targets\n",
                     stdout);
 
@@ -1032,12 +1049,15 @@ public:
     }
 private:
     StringArray optQueryIds;
+    StringArray optIgnoreQueries;
     StringAttr optFileName;
     StringAttr optTarget;
     StringAttr optPMID;
     bool optValidateActive;
     bool optCheckDFS;
     bool optGlobalScope;
+    bool optIgnoreWarnings = false;
+    bool optIgnoreOptionalFiles = false;
 };
 
 class EclCmdPackageQueryFiles : public EclCmdCommon

ecl/eclcmd/eclcmd_common.hpp

@@ -117,6 +117,9 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_PART_NAME "--part-name"
 #define ECLOPT_PROTECT "--protect"
 #define ECLOPT_USE_EXISTING "--use-existing"
+#define ECLOPT_IGNORE_WARNINGS "--ignore-warnings"
+#define ECLOPT_IGNORE_OPTIONAL "--ignore-optional"
+#define ECLOPT_IGNORE_QUERIES "--ignore-queries"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

esp/scm/ws_packageprocess.ecm

@@ -189,8 +189,11 @@ ESPrequest ValidatePackageRequest
     string PMID;
     string QueryIdToVerify;
     ESParray<string> QueriesToVerify;
+    ESParray<string> QueriesToIgnore;
     bool CheckDFS;
     bool GlobalScope(0);
+    bool IgnoreWarnings(0);
+    bool IgnoreOptionalFiles(0);
 };
 
 ESPstruct ValidatePackageInfo

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -1165,10 +1165,18 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
         if (queryid && *queryid)
             queriesToVerify.appendUniq(queryid);
     }
-    map->validate(queriesToVerify, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles);
+    StringArray queriesToIgnore;
+    ForEachItemIn(i2, req.getQueriesToIgnore())
+    {
+        queryid = req.getQueriesToIgnore().item(i2);
+        if (queryid && *queryid)
+            queriesToIgnore.appendUniq(queryid);
+    }
+    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
 
     resp.setPMID(map->queryPackageId());
-    resp.setWarnings(warnings);
+    if (!req.getIgnoreWarnings())
+        resp.setWarnings(warnings);
     resp.setErrors(errors);
     resp.updateQueries().setUnmatched(unmatchedQueries);
     resp.updatePackages().setUnmatched(unusedPackages);

fs/dafsserver/dafsserver.cpp

@@ -3207,7 +3207,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
     CClientStatsTable clientStatsTable;
     atomic_t globallasttick;
     unsigned targetActiveThreads;
-    Owned<IPropertyTree> keyPairInfo;
+    Linked<IPropertyTree> keyPairInfo;
 
     int getNextHandle()
     {
@@ -4946,7 +4946,7 @@ public:
                     {
                         EXCLOG(e,"CRemoteFileServer");
                         e->Release();
-                        break;
+                        continue;
                     }
                 }
 

roxie/ccd/ccdstate.cpp

@@ -823,9 +823,9 @@ public:
     {
         return BASE::getPartIds();
     }
-    virtual bool validate(StringArray &queryids, StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
+    virtual bool validate(const StringArray &queryids, const StringArray &queriesToIgnore, StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles, bool ignoreOptionalFiles) const
     {
-        return BASE::validate(queryids, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles);
+        return BASE::validate(queryids, queriesToIgnore, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles, ignoreOptionalFiles);
     }
     virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const
     {