Merge pull request #14025 from wangkx/h24365

HPCC-24365 Support package map for ANY target cluster

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/ecl-package/ecl-package.cpp

@@ -918,7 +918,6 @@ public:
             outputLfnCategoryTree(querynode, "Optional");
         }
     }
-
     virtual int processCMD()
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
@@ -948,7 +947,36 @@ public:
         int ret = outputMultiExceptionsEx(resp->getExceptions());
         if (ret != 0)
             validateMessages = true;
-        StringArray &errors = resp->getErrors();
+        const char *pmid = resp->getPMID();
+        if (!isEmptyString(pmid))
+        { //server version < 1.04
+            processValidatePackageResponse(nullptr, nullptr, resp->getErrors(), resp->getWarnings(),
+                resp->getQueries().getUnmatched(), resp->getPackages().getUnmatched(), resp->getFiles(), validateMessages);
+        }
+        else
+        {
+            IArrayOf<IConstValidateResult> &results = resp->getResults();
+            ForEachItemIn(i, results)
+            {
+                IConstValidateResult &result = results.item(i);
+                processValidatePackageResponse(result.getTarget(), result.getPMID(), result.getErrors(), result.getWarnings(),
+                    result.getQueries().getUnmatched(), result.getPackages().getUnmatched(), result.getFiles(), validateMessages);
+            }
+        }
+
+        if (!validateMessages)
+            fputs("   Validation was successful\n", stdout);
+
+        return ret;
+    }
+
+    bool processValidatePackageResponse(const char *target, const char *pmid, StringArray &errors, StringArray &warnings,
+        StringArray &unmatchedQueries, StringArray &unusedPackages, IConstValidatePackageFiles &files,
+        bool &validateMessages)
+    {
+        if (!isEmptyString(target) && !isEmptyString(pmid))
+            fprintf(stderr, "   Target: %s, PMID: %s :\n", target, pmid);
+
         if (errors.ordinality()>0)
         {
             validateMessages = true;
@@ -957,7 +985,6 @@ public:
             ForEachItemIn(i, errors)
                 fprintf(stderr, "      %s\n", errors.item(i));
         }
-        StringArray &warnings = resp->getWarnings();
         if (warnings.ordinality()>0)
         {
             validateMessages = true;
@@ -965,7 +992,6 @@ public:
             ForEachItemIn(i, warnings)
                 fprintf(stderr, "      %s\n", warnings.item(i));
         }
-        StringArray &unmatchedQueries = resp->getQueries().getUnmatched();
         if (unmatchedQueries.ordinality()>0)
         {
             validateMessages = true;
@@ -973,7 +999,6 @@ public:
             ForEachItemIn(i, unmatchedQueries)
                 fprintf(stderr, "      %s\n", unmatchedQueries.item(i));
         }
-        StringArray &unusedPackages = resp->getPackages().getUnmatched();
         if (unusedPackages.ordinality()>0)
         {
             validateMessages = true;
@@ -981,7 +1006,7 @@ public:
             ForEachItemIn(i, unusedPackages)
                 fprintf(stderr, "      %s\n", unusedPackages.item(i));
         }
-        StringArray &unusedFiles = resp->getFiles().getUnmatched();
+        StringArray &unusedFiles = files.getUnmatched();
         if (unusedFiles.ordinality()>0)
         {
             validateMessages = true;
@@ -1006,7 +1031,7 @@ public:
             outputLfnTree(filetree);
         }
 
-        StringArray &notInDFS = resp->getFiles().getNotInDFS();
+        StringArray &notInDFS = files.getNotInDFS();
         if (notInDFS.ordinality()>0)
         {
             validateMessages = true;
@@ -1015,10 +1040,9 @@ public:
                 fprintf(stderr, "      %s\n", notInDFS.item(i));
         }
 
-        if (!validateMessages)
-            fputs("   Validation was successful\n", stdout);
-
-        return ret;
+        if (!isEmptyString(target) && !isEmptyString(pmid))
+            fprintf(stderr, "\n   Target: %s, PMID: %s done\n\n", target, pmid);
+        return validateMessages;
     }
 
     virtual void usage()

esp/scm/ws_packageprocess.ecm

@@ -212,10 +212,10 @@ ESPstruct ValidatePackageFiles
     ESParray<string, File> NotInDFS;
 };
 
-ESPresponse [exceptions_inline] ValidatePackageResponse
+ESPstruct ValidateResult
 {
+    string Target;
     string PMID;
-    ESPstruct BasePackageStatus status;
     ESParray<string> Warnings;
     ESParray<string> Errors;
     ESPstruct ValidatePackageInfo packages;
@@ -223,6 +223,18 @@ ESPresponse [exceptions_inline] ValidatePackageResponse
     ESPstruct ValidatePackageFiles files;
 };
 
+ESPresponse [exceptions_inline] ValidatePackageResponse
+{
+    [depr_ver("1.04")] string PMID;
+    [depr_ver("1.04")] ESPstruct BasePackageStatus status;
+    [depr_ver("1.04")] ESParray<string> Warnings;
+    [depr_ver("1.04")] ESParray<string> Errors;
+    [depr_ver("1.04")] ESPstruct ValidatePackageInfo packages;
+    [depr_ver("1.04")] ESPstruct ValidatePackageQueries queries;
+    [depr_ver("1.04")] ESPstruct ValidatePackageFiles files;
+    [min_ver("1.04")] ESParray<ESPstruct ValidateResult, Result> Results;
+};
+
 ESPrequest GetQueryFileMappingRequest
 {
     string Target;
@@ -316,7 +328,7 @@ ESPresponse [exceptions_inline] GetPartFromPackageMapResponse
     string Content;
 };
 
-ESPservice [auth_feature("NONE"), version("1.03"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
+ESPservice [auth_feature("NONE"), version("1.04"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod [auth_feature("PackageMapAccess:WRITE")] AddPackage(AddPackageRequest, AddPackageResponse);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -28,6 +28,7 @@
 #include "referencedfilelist.hpp"
 #include "package.h"
 #include "eclwatch_errorlist.hpp"
+#include "exception_util.hpp"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -287,7 +288,9 @@ public:
             target.set(_target);
             if (target.isEmpty())
                 throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "Target cluster parameter required");
-            ensureClusterInfo();
+
+            if (!streq(target.get(), "*"))
+                ensureClusterInfo();
             pmid.append(target).append("::");
         }
         pmid.append(name);
@@ -362,7 +365,19 @@ public:
     }
     void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound, IPropertyTree *pt)
     {
-        cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+        if (!streq(target.get(), "*"))
+            cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+        else
+        {
+            CConstWUClusterInfoArray clusters;
+            getEnvironmentClusterInfo(clusters);
+            ForEachItemIn(i, clusters)
+            {
+                IConstWUClusterInfo &cluster = clusters.item(i);
+                if (cluster.getPlatform() == RoxieCluster)
+                    cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, &cluster, srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+            }
+        }
     }
     void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound)
     {
@@ -1084,102 +1099,150 @@ bool CWsPackageProcessEx::onGetPackageMapById(IEspContext &context, IEspGetPacka
 
 bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp)
 {
-    StringArray warnings;
-    StringArray errors;
-    StringArray unmatchedQueries;
-    StringArray unusedPackages;
-    StringArray unmatchedFiles;
+    try
+    {
+        const char *target = req.getTarget();
+        if (isEmptyString(target))
+            throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
+
+        Owned<IPropertyTree> packageMapTree;
+        const char *pmID = req.getPMID();
+        const char *packageMapString = req.getInfo();
+        if (!req.getActive() && isEmptyString(pmID))
+        {
+            if (isEmptyString(packageMapString))
+                throw makeStringException(ECLWATCH_INVALID_INPUT, "Please specify PMID or Package Map content");
 
-    Owned<IHpccPackageSet> set;
-    Owned<IPropertyTree> mapTree;
+            packageMapTree.setown(createPTreeFromXMLString(packageMapString));
+            if (!packageMapTree)
+                throw makeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
+            fixPackageMapFileIds(packageMapTree, false);
+        }
 
-    const char *target = req.getTarget();
-    if (!target || !*target)
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
-    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
-    if (!clusterInfo)
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
+        StringArray &queriesToIgnore = req.getQueriesToIgnore();
+        StringArray &queriesToVerify = req.getQueriesToVerify();
+        const char *queryid = req.getQueryIdToVerify();
+        if (!isEmptyString(queryid))
+            queriesToVerify.appendUniq(queryid);
+
+        double version = context.getClientVersion();
+        IArrayOf<IEspValidateResult> results;
+        if (!streq(target, "*"))
+        {
+            Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+            if (!clusterInfo)
+                throw makeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
+            validatePackage(context, req, packageMapTree, clusterInfo, queriesToVerify, queriesToIgnore,
+                (version < 1.04) ? &resp : nullptr, results);
+        }
+        else
+        {
+            CConstWUClusterInfoArray clusters;
+            getEnvironmentClusterInfo(clusters);
+            ForEachItemIn(i, clusters)
+            {
+                IConstWUClusterInfo &cluster = clusters.item(i);
+                if (cluster.getPlatform() == RoxieCluster)
+                {
+                    validatePackage(context, req, packageMapTree, &cluster, queriesToVerify, queriesToIgnore,
+                        (version < 1.04) ? &resp : nullptr, results);
+                    if (version < 1.04)
+                        break;
+                }
+            }
+        }
+        if (version >= 1.04)
+            resp.setResults(results);
+    }
+    catch(IException *e)
+    {
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
+void CWsPackageProcessEx::validatePackage(IEspContext &context, IEspValidatePackageRequest &req, IPropertyTree *packageMapTree,
+    IConstWUClusterInfo *clusterInfo, StringArray &queriesToVerify, StringArray &queriesToIgnore,
+    IEspValidatePackageResponse *resp, IArrayOf<IEspValidateResult>& results)
+{
     SCMStringBuffer process;
     clusterInfo->getRoxieProcess(process);
     if (!process.length())
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
+        throw makeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
 
+    SCMStringBuffer clusterName;
+    clusterInfo->getName(clusterName);
+    const char *target = clusterName.str();
     const char* pmID = req.getPMID();
-    const char* info = req.getInfo();
 
+    Owned<IPropertyTree> mapTree;
     if (req.getActive()) //validate active map
     {
         mapTree.setown(resolveActivePackageMap(process.str(), target, true));
         if (!mapTree)
-            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
-    }
-    else if (pmID && *pmID)
-    {
-        mapTree.setown(getPackageMapById(req.getGlobalScope() ? NULL : target, pmID, true));
-        if (!mapTree)
-            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", pmID);
+            throw makeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
     }
-    else if (info && *info)
+    else if (!isEmptyString(pmID))
     {
-        mapTree.setown(createPTreeFromXMLString(info));
+        mapTree.setown(getPackageMapById(req.getGlobalScope() ? nullptr : target, pmID, true));
         if (!mapTree)
-            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
-        fixPackageMapFileIds(mapTree, false);
+            throw makeStringExceptionV(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", pmID);
     }
-    else
+    else if (packageMapTree)
     {
-        throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "package map not specified");
+        mapTree.setown(LINK(packageMapTree));
     }
 
+    StringArray notInDFS;
     if (req.getCheckDFS())
     {
         Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true, false);
         pmfiles->addFilesFromPackageMap(mapTree);
-        pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false, false);
+        pmfiles->resolveFiles(process.str(), nullptr, nullptr, nullptr, true, false, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();
-        StringArray notInDFS;
         ForEach(*files)
         {
             IReferencedFile &file = files->query();
             if (file.getFlags() & RefFileNotFound)
                 notInDFS.append(file.getLogicalName());
         }
-        resp.updateFiles().setNotInDFS(notInDFS);
     }
 
     const char *id = mapTree->queryProp("@id");
     Owned<IHpccPackageMap> map = createPackageMapFromPtree(mapTree, target, id);
     if (!map)
-        throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
+        throw makeStringExceptionV(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
 
-    StringArray queriesToVerify;
-    const char *queryid = req.getQueryIdToVerify();
-    if (queryid && *queryid)
-        queriesToVerify.append(queryid);
-    ForEachItemIn(i, req.getQueriesToVerify())
-    {
-        queryid = req.getQueriesToVerify().item(i);
-        if (queryid && *queryid)
-            queriesToVerify.appendUniq(queryid);
-    }
-    StringArray queriesToIgnore;
-    ForEachItemIn(i2, req.getQueriesToIgnore())
+    StringArray warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles;
+    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
+
+    if (resp)
     {
-        queryid = req.getQueriesToIgnore().item(i2);
-        if (queryid && *queryid)
-            queriesToIgnore.appendUniq(queryid);
+        resp->setPMID(map->queryPackageId());
+        if (!req.getIgnoreWarnings())
+            resp->setWarnings(warnings);
+        resp->setErrors(errors);
+        resp->updateQueries().setUnmatched(unmatchedQueries);
+        resp->updatePackages().setUnmatched(unusedPackages);
+        resp->updateFiles().setUnmatched(unmatchedFiles);
+        resp->updateStatus().setCode(0);
+        if (notInDFS.length())
+            resp->updateFiles().setNotInDFS(notInDFS);
+        return;
     }
-    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
 
-    resp.setPMID(map->queryPackageId());
+    Owned<IEspValidateResult> result = createValidateResult();
+    result->setTarget(target);
+    result->setPMID(map->queryPackageId());
     if (!req.getIgnoreWarnings())
-        resp.setWarnings(warnings);
-    resp.setErrors(errors);
-    resp.updateQueries().setUnmatched(unmatchedQueries);
-    resp.updatePackages().setUnmatched(unusedPackages);
-    resp.updateFiles().setUnmatched(unmatchedFiles);
-    resp.updateStatus().setCode(0);
-    return true;
+        result->setWarnings(warnings);
+    result->setErrors(errors);
+    result->updateQueries().setUnmatched(unmatchedQueries);
+    result->updatePackages().setUnmatched(unusedPackages);
+    result->updateFiles().setUnmatched(unmatchedFiles);
+    if (notInDFS.length())
+        result->updateFiles().setNotInDFS(notInDFS);
+    results.append(*result.getClear());
 }
 
 bool CWsPackageProcessEx::onGetPackageMapSelectOptions(IEspContext &context, IEspGetPackageMapSelectOptionsRequest &req, IEspGetPackageMapSelectOptionsResponse &resp)

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -20,6 +20,7 @@
 
 #include "ws_packageprocess_esp.ipp"
 #include "dasds.hpp"
+#include "environment.hpp"
 
 #define THORCLUSTER "thor"
 #define HTHORCLUSTER "hthor"
@@ -151,6 +152,8 @@ class CWsPackageProcessEx : public CWsPackageProcess
     void getPkgInfoById(const char *target, const char *packageMapId, IPropertyTree* tree);
     void getPkgInfoById(const char *packageMapId, IPropertyTree* tree);
     void deletePackage(const char *packageMap, const char *target, const char *process, bool globalScope, StringBuffer &returnMsg, int &returnCode);
+    void validatePackage(IEspContext &context, IEspValidatePackageRequest &req, IPropertyTree *packageMapTree, IConstWUClusterInfo *clusterInfo,
+        StringArray &queriesToVerify, StringArray &queriesToIgnore, IEspValidatePackageResponse *resp, IArrayOf<IEspValidateResult>& results);
 public:
     IMPLEMENT_IINTERFACE;
     virtual ~CWsPackageProcessEx()