HPCC-9869 Add support for explicitly using packagemaps shared globally

With Packagemaps now defaulting to be unique copies per target,
--global-scope now provides a way of sharing the same packagemap
globally across multiple targets.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/package.cpp

@@ -222,11 +222,14 @@ extern WORKUNIT_API IPropertyTree * getPackageMapById(const char * id, bool read
 
 extern WORKUNIT_API IPropertyTree * getPackageMapById(const char *target, const char * id, bool readonly)
 {
-    VStringBuffer xpath("/PackageMaps/PackageMap[@id='%s::%s']", target, id);
-    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
-    if (!conn)
-        return getPackageMapById(id, readonly);
-    return conn->getRoot();
+    if (target && *target)
+    {
+        VStringBuffer xpath("/PackageMaps/PackageMap[@id='%s::%s']", target, id);
+        Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+        if (conn)
+            return conn->getRoot();
+    }
+    return getPackageMapById(id, readonly);
 }
 
 extern WORKUNIT_API IPropertyTree * getPackageSetById(const char * id, bool readonly)

ecl/ecl-package/ecl-package.cpp

@@ -49,7 +49,7 @@ IClientWsPackageProcess *getWsPackageSoapService(const char *server, const char
 class EclCmdPackageActivate : public EclCmdCommon
 {
 public:
-    EclCmdPackageActivate()
+    EclCmdPackageActivate() : optGlobalScope(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -76,6 +76,8 @@ public:
                 }
                 continue;
             }
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -109,6 +111,7 @@ public:
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
         request->setProcess(optProcess);
+        request->setGlobalScope(optGlobalScope);
 
         Owned<IClientActivatePackageResponse> resp = packageProcessClient->ActivatePackage(request);
         if (resp->getExceptions().ordinality())
@@ -126,7 +129,8 @@ public:
                     "ecl packagemap activate <target> <packagemap>\n"
                     " Options:\n"
                     "   <target>               Name of target containing package map to activate\n"
-                    "   <packagemap>           Packagemap to activate\n",
+                    "   <packagemap>           Packagemap to activate\n"
+                    "   --global-scope         The specified packagemap can be shared across multiple targets\n",
                     stdout);
         EclCmdCommon::usage();
     }
@@ -135,12 +139,13 @@ private:
     StringAttr optTarget;
     StringAttr optPackageMap;
     StringAttr optProcess;
+    bool optGlobalScope;
 };
 
 class EclCmdPackageDeActivate : public EclCmdCommon
 {
 public:
-    EclCmdPackageDeActivate()
+    EclCmdPackageDeActivate() : optGlobalScope(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -167,6 +172,8 @@ public:
                 }
                 continue;
             }
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -200,6 +207,7 @@ public:
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
         request->setProcess(optProcess);
+        request->setGlobalScope(optGlobalScope);
 
         Owned<IClientDeActivatePackageResponse> resp = packageProcessClient->DeActivatePackage(request);
         if (resp->getExceptions().ordinality())
@@ -216,7 +224,8 @@ public:
                     "ecl packagemap deactivate <target> <packagemap>\n"
                     " Options:\n"
                     "   <target>               Name of target containing package map to activate\n"
-                    "   <packagemap>           Packagemap to activate\n",
+                    "   <packagemap>           Packagemap to activate\n"
+                    "   --global-scope         The specified packagemap can be shared across multiple targets\n",
                     stdout);
         EclCmdCommon::usage();
     }
@@ -225,6 +234,7 @@ private:
     StringAttr optTarget;
     StringAttr optPackageMap;
     StringAttr optProcess;
+    bool optGlobalScope;
 };
 
 class EclCmdPackageList : public EclCmdCommon
@@ -413,6 +423,8 @@ public:
                 }
                 continue;
             }
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -451,6 +463,7 @@ public:
         request->setTarget(optTarget);
         request->setPackageMap(optPackageMap);
         request->setProcess(optProcess);
+        request->setGlobalScope(optGlobalScope);
 
         Owned<IClientDeletePackageResponse> resp = packageProcessClient->DeletePackage(request);
         if (resp->getExceptions().ordinality())
@@ -470,7 +483,8 @@ public:
                     "ecl packagemap delete <target> <packagemap>\n"
                     " Options:\n"
                     "   <target>               Name of the target to use \n"
-                    "   <packagemap>           Name of the package map to delete\n",
+                    "   <packagemap>           Name of the package map to delete\n"
+                    "   --global-scope         The specified packagemap is sharable across multiple targets\n",
                     stdout);
         EclCmdCommon::usage();
     }
@@ -478,12 +492,13 @@ private:
     StringAttr optPackageMap;
     StringAttr optTarget;
     StringAttr optProcess;
+    bool optGlobalScope;
 };
 
 class EclCmdPackageAdd : public EclCmdCommon
 {
 public:
-    EclCmdPackageAdd() : optActivate(false), optOverWrite(false)
+    EclCmdPackageAdd() : optActivate(false), optOverWrite(false), optGlobalScope(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -518,6 +533,8 @@ public:
                 continue;
             if (iter.matchFlag(optOverWrite, ECLOPT_OVERWRITE)||iter.matchFlag(optOverWrite, ECLOPT_OVERWRITE_S))
                 continue;
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -571,6 +588,7 @@ public:
         request->setProcess(optProcess);
         request->setDaliIp(optDaliIP);
         request->setOverWrite(optOverWrite);
+        request->setGlobalScope(optGlobalScope);
 
         Owned<IClientAddPackageResponse> resp = packageProcessClient->AddPackage(request);
         if (resp->getExceptions().ordinality())
@@ -599,7 +617,8 @@ public:
                     "   -O, --overwrite             Overwrite existing information\n"
                     "   -A, --activate              Activate the package information\n"
                     "   --daliip=<ip>               IP of the remote dali to use for logical file lookups\n"
-                   "   --pmid                       Identifier of package map - defaults to filename if not specified."
+                    "   --pmid                      Identifier of package map - defaults to filename if not specified\n"
+                    "   --global-scope              The specified packagemap can be shared across multiple targets\n"
 // NOT-YET          "  --packageprocessname         if not set use this package process name for all clusters"
                     "   <target>                    Name of target to use when adding package map information\n"
                     "   <filename>                  Name of file containing package map information\n",
@@ -608,20 +627,21 @@ public:
         EclCmdCommon::usage();
     }
 private:
+    StringBuffer pkgInfo;
     StringAttr optFileName;
     StringAttr optTarget;
     StringAttr optProcess;
-    bool optActivate;
-    bool optOverWrite;
-    StringBuffer pkgInfo;
     StringAttr optDaliIP;
     StringAttr optPackageMapId;
+    bool optActivate;
+    bool optOverWrite;
+    bool optGlobalScope;
 };
 
 class EclCmdPackageValidate : public EclCmdCommon
 {
 public:
-    EclCmdPackageValidate() : optValidateActive(false), optCheckDFS(false)
+    EclCmdPackageValidate() : optValidateActive(false), optCheckDFS(false), optGlobalScope(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -656,6 +676,8 @@ public:
                 continue;
             if (iter.matchOption(optQueryId, ECLOPT_QUERYID))
                 continue;
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -709,6 +731,7 @@ public:
         request->setTarget(optTarget);
         request->setQueryIdToVerify(optQueryId);
         request->setCheckDFS(optCheckDFS);
+        request->setGlobalScope(optGlobalScope);
 
         bool validateMessages = false;
         Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
@@ -783,7 +806,8 @@ public:
                     "   <filename>                  Name of file containing package map information\n"
                     "   --active                    Validate the active packagemap\n"
                     "   --check-dfs                 Verify that subfiles exist in DFS\n"
-                    "   -pm, --pmid                 Identifier of packagemap to validate\n",
+                    "   -pm, --pmid                 Identifier of packagemap to validate\n"
+                    "   --global-scope              The specified packagemap can be shared across multiple targets\n",
                     stdout);
 
         EclCmdCommon::usage();
@@ -795,12 +819,13 @@ private:
     StringAttr optQueryId;
     bool optValidateActive;
     bool optCheckDFS;
+    bool optGlobalScope;
 };
 
 class EclCmdPackageQueryFiles : public EclCmdCommon
 {
 public:
-    EclCmdPackageQueryFiles()
+    EclCmdPackageQueryFiles() : optGlobalScope(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -829,6 +854,8 @@ public:
             }
             if (iter.matchOption(optPMID, ECLOPT_PMID) || iter.matchOption(optPMID, ECLOPT_PMID_S))
                 continue;
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -863,6 +890,7 @@ public:
         request->setTarget(optTarget);
         request->setQueryName(optQueryId);
         request->setPMID(optPMID);
+        request->setGlobalScope(optGlobalScope);
 
         Owned<IClientGetQueryFileMappingResponse> resp = packageProcessClient->GetQueryFileMapping(request);
         if (resp->getExceptions().ordinality()>0)
@@ -911,7 +939,8 @@ public:
                     " Options:\n"
                     "   <target>                    Name of target to use when validating package map information\n"
                     "   <queryid>                   Name of query to get file mappings for\n"
-                    "   -pm, --pmid                 Optional id of packagemap to validate, defaults to active\n",
+                    "   -pm, --pmid                 Optional id of packagemap to validate, defaults to active\n"
+                    "   --global-scope              The specified packagemap can be shared across multiple targets\n",
                     stdout);
 
         EclCmdCommon::usage();
@@ -920,6 +949,7 @@ private:
     StringAttr optTarget;
     StringAttr optQueryId;
     StringAttr optPMID;
+    bool optGlobalScope;
 };
 
 IEclCommand *createPackageSubCommand(const char *cmdname)

ecl/eclcmd/eclcmd_common.hpp

@@ -83,6 +83,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_DELETE_PREVIOUS_INI "deletePrevDefault"
 #define ECLOPT_DELETE_PREVIOUS_ENV "ACTIVATE_DELETE_PREVIOUS"
 #define ECLOPT_CHECK_DFS "--check-dfs"
+#define ECLOPT_GLOBAL_SCOPE "--global-scope"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

esp/scm/ws_packageprocess.ecm

@@ -28,6 +28,7 @@ ESPrequest AddPackageRequest
     string PackageMap;
     string Process;
     string DaliIp;
+    bool GlobalScope(0);
 };
 
 
@@ -42,6 +43,7 @@ ESPrequest DeletePackageRequest
     string Target;
     string PackageMap;
     string Process;
+    bool GlobalScope(0);
 };
 
 ESPresponse [exceptions_inline] DeletePackageResponse
@@ -54,6 +56,7 @@ ESPrequest ActivatePackageRequest
     string Target;
     string PackageMap;
     string Process;
+    bool GlobalScope(0);
 };
 
 ESPresponse [exceptions_inline] ActivatePackageResponse
@@ -66,6 +69,7 @@ ESPrequest DeActivatePackageRequest
     string Target;
     string PackageMap;
     string Process;
+    bool GlobalScope(0);
 };
 
 ESPresponse [exceptions_inline] DeActivatePackageResponse
@@ -119,6 +123,7 @@ ESPrequest ValidatePackageRequest
     string PMID;
     string QueryIdToVerify;
     bool CheckDFS;
+    bool GlobalScope(0);
 };
 
 ESPstruct ValidatePackageInfo
@@ -153,6 +158,7 @@ ESPrequest GetQueryFileMappingRequest
     string Target;
     string PMID;
     string QueryName;
+    bool GlobalScope(0);
 };
 
 ESPstruct SuperFile

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -363,7 +363,7 @@ void getPkgInfo(const char *target, const char *process, StringBuffer &info)
     toXML(tree, info);
 }
 
-bool deletePkgInfo(const char *name, const char *target, const char *process)
+bool deletePkgInfo(const char *name, const char *target, const char *process, bool globalScope)
 {
     Owned<IRemoteConnection> pkgSetsConn = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
     if (!pkgSetsConn)
@@ -384,8 +384,13 @@ bool deletePkgInfo(const char *name, const char *target, const char *process)
     StringBuffer lcName(name);
     name = lcName.toLowerCase().str();
 
-    VStringBuffer xpath("PackageMap[@id='%s::%s'][@querySet='%s']", target, name, target);
-    IPropertyTree *mapEntry = pkgSetRegistry->getPropTree(xpath.str());
+    IPropertyTree *mapEntry = NULL;
+    StringBuffer xpath;
+    if (!globalScope)
+    {
+        xpath.appendf("PackageMap[@id='%s::%s'][@querySet='%s']", target, name, target);
+        mapEntry = pkgSetRegistry->getPropTree(xpath.str());
+    }
     if (!mapEntry)
     {
         xpath.clear().appendf("PackageMap[@id='%s'][@querySet='%s']", name, target);
@@ -412,7 +417,7 @@ bool deletePkgInfo(const char *name, const char *target, const char *process)
     return true;
 }
 
-void activatePackageMapInfo(const char *target, const char *name, const char *process, bool activate)
+void activatePackageMapInfo(const char *target, const char *name, const char *process, bool globalScope, bool activate)
 {
     if (!target || !*target)
         throw MakeStringExceptionDirect(PKG_TARGET_NOT_DEFINED, "No target defined");
@@ -440,8 +445,12 @@ void activatePackageMapInfo(const char *target, const char *name, const char *pr
     IPropertyTree *pkgSetTree = root->queryPropTree(xpath);
     if (pkgSetTree)
     {
-        xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s::%s']", target, target, name);
-        IPropertyTree *mapTree = pkgSetTree->queryPropTree(xpath);
+        IPropertyTree *mapTree = NULL;
+        if (!globalScope)
+        {
+            xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s::%s']", target, target, name);
+            mapTree = pkgSetTree->queryPropTree(xpath);
+        }
         if (!mapTree)
         {
             xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s']", target, name);
@@ -465,7 +474,10 @@ bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageReque
     if (name.isEmpty())
         throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "PackageMap name parameter required");
 
-    VStringBuffer pmid("%s::%s", target.get(), name.get());
+    StringBuffer pmid;
+    if (!req.getGlobalScope())
+        pmid.append(target).append("::");
+    pmid.append(name.get());
 
     StringBuffer info(req.getInfo());
     bool activate = req.getActivate();
@@ -503,7 +515,7 @@ bool CWsPackageProcessEx::onDeletePackage(IEspContext &context, IEspDeletePackag
     if (processName.length()==0)
         processName.set("*");
 
-    bool ret = deletePkgInfo(pkgMap.get(), req.getTarget(), processName.get());
+    bool ret = deletePkgInfo(pkgMap.get(), req.getTarget(), processName.get(), req.getGlobalScope());
     StringBuffer msg;
     (ret) ? msg.append("Successfully ") : msg.append("Unsuccessfully ");
     msg.append("deleted ").append(pkgMap.get()).append(" from ").append(req.getTarget());
@@ -515,14 +527,14 @@ bool CWsPackageProcessEx::onDeletePackage(IEspContext &context, IEspDeletePackag
 bool CWsPackageProcessEx::onActivatePackage(IEspContext &context, IEspActivatePackageRequest &req, IEspActivatePackageResponse &resp)
 {
     resp.updateStatus().setCode(0);
-    activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), true);
+    activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), req.getGlobalScope(), true);
     return true;
 }
 
 bool CWsPackageProcessEx::onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp)
 {
     resp.updateStatus().setCode(0);
-    activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), false);
+    activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), req.getGlobalScope(), false);
     return true;
 }
 
@@ -577,7 +589,7 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     }
     else if (pmid && *pmid)
     {
-        mapTree.setown(getPackageMapById(target, pmid, true));
+        mapTree.setown(getPackageMapById(req.getGlobalScope() ? NULL : target, pmid, true));
         if (!mapTree)
             throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", req.getPMID());
     }
@@ -642,7 +654,7 @@ bool CWsPackageProcessEx::onGetQueryFileMapping(IEspContext &context, IEspGetQue
     const char *pmid = req.getPMID();
     if (pmid && *pmid)
     {
-        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(target, pmid, true), target, pmid));
+        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(req.getGlobalScope() ? NULL : target, pmid, true), target, pmid));
         if (!ownedmap)
             throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
         map = ownedmap;