HPCC-14071 Supporting maintaining PackageMaps in multiple files

Add the ability to add and remove additional files from/to existing
packagemaps.  This will make it easier to support sub collections of
files and queries, and will make it easier for individual ECL
developers to deploy their own packagemaps in shared environments.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/pkgimpl.hpp

@@ -366,29 +366,38 @@ public:
     {
         return (IFACE *) matchResolvedPackage(name);
     }
-
+    void loadPackage(IPropertyTree &packageTree)
+    {
+        const char *id = packageTree.queryProp("@id");
+        if (!id || !*id)
+            throw MakeStringException(PACKAGE_MISSING_ID, "Invalid package map - Package element missing id attribute");
+        Owned<packageType> package = new packageType(&packageTree);
+        packages.setValue(id, package.get());
+        const char *queries = packageTree.queryProp("@queries");
+        if (queries && *queries)
+        {
+            wildMatches.append(queries);
+            wildIds.append(id);
+        }
+    }
+    void loadPart(IPropertyTree &part)
+    {
+        Owned<IPropertyTreeIterator> partPackages = part.getElements("Package");
+        ForEach(*partPackages)
+            loadPackage(partPackages->query());
+    }
     void load(IPropertyTree *xml)
     {
         if (!xml)
             return;
         compulsory = xml->getPropBool("@compulsory");
-        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
+        Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package"); //old style non-part packages first
         ForEach(*allpackages)
-        {
-            IPropertyTree &packageTree = allpackages->query();
-            const char *id = packageTree.queryProp("@id");
-            if (!id || !*id)
-                throw MakeStringException(PACKAGE_MISSING_ID, "Invalid package map - Package element missing id attribute");
-            Owned<packageType> package = new packageType(&packageTree);
-            packages.setValue(id, package.get());
-            const char *queries = packageTree.queryProp("@queries");
-            if (queries && *queries)
-            {
-                wildMatches.append(queries);
-                wildIds.append(id);
-            }
-        }
-        HashIterator it(packages);
+            loadPackage(allpackages->query());
+        Owned<IPropertyTreeIterator> parts = xml->getElements("Part"); //new multipart packagemap
+        ForEach(*parts)
+            loadPart(parts->query());
+        HashIterator it(packages); //package bases can be across parts
         ForEach (it)
         {
             packageType *pkg = packages.getValue((const char *)it.query().getKey());

ecl/ecl-package/ecl-package.cpp

@@ -968,6 +968,371 @@ private:
     bool optGlobalScope;
 };
 
+
+class EclCmdPackageAddPart : public EclCmdCommon
+{
+public:
+    EclCmdPackageAddPart() : optDeletePrevious(false), optGlobalScope(false), optAllowForeign(false), optPreloadAll(false), optUpdateSuperfiles(false), optUpdateCloneFrom(false), optDontAppendCluster(false)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+            return false;
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else if (optPMID.isEmpty())
+                    optPMID.set(arg);
+                else if (optFileName.isEmpty())
+                    optFileName.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (iter.matchOption(optPartName, ECLOPT_PART_NAME))
+                continue;
+            if (iter.matchOption(optDaliIP, ECLOPT_DALIIP))
+                continue;
+            if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
+                continue;
+            if (iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS))
+                continue;
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
+            if (iter.matchFlag(optAllowForeign, ECLOPT_ALLOW_FOREIGN))
+                continue;
+            if (iter.matchFlag(optPreloadAll, ECLOPT_PRELOAD_ALL_PACKAGES))
+                continue;
+            if (iter.matchFlag(optUpdateSuperfiles, ECLOPT_UPDATE_SUPER_FILES))
+                continue;
+            if (iter.matchFlag(optUpdateCloneFrom, ECLOPT_UPDATE_CLONE_FROM))
+                continue;
+            if (iter.matchFlag(optDontAppendCluster, ECLOPT_DONT_APPEND_CLUSTER))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n");
+        else if (optTarget.isEmpty())
+            err.append("\n ... Specify a cluster name\n");
+        else if (optPMID.isEmpty())
+            err.append("\n ... Specify a packagemap name\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            return false;
+        }
+
+        if (optPartName.isEmpty())
+        {
+            StringBuffer name;
+            splitFilename(optFileName.get(), NULL, NULL, &name, &name);
+            optPartName.set(name.str());
+        }
+        optPMID.toLowerCase();
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
+        StringBuffer content;
+        content.loadFile(optFileName);
+
+        fprintf(stdout, "\n ... adding packagemap %s part %s from file %s\n\n", optPMID.get(), optPartName.get(), optFileName.get());
+
+        Owned<IClientAddPartToPackageMapRequest> request = packageProcessClient->createAddPartToPackageMapRequest();
+        request->setTarget(optTarget);
+        request->setPackageMap(optPMID);
+        request->setPartName(optPartName);
+        request->setContent(content);
+        request->setDeletePrevious(optDeletePrevious);
+        request->setDaliIp(optDaliIP);
+        request->setGlobalScope(optGlobalScope);
+        request->setSourceProcess(optSourceProcess);
+        request->setAllowForeignFiles(optAllowForeign);
+        request->setPreloadAllPackages(optPreloadAll);
+        request->setUpdateSuperFiles(optUpdateSuperfiles);
+        request->setUpdateCloneFrom(optUpdateCloneFrom);
+        request->setAppendCluster(!optDontAppendCluster);
+
+        Owned<IClientAddPartToPackageMapResponse> resp = packageProcessClient->AddPartToPackageMap(request);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+
+        StringArray &notFound = resp->getFilesNotFound();
+        if (notFound.length())
+        {
+            fputs("\nFiles defined in packagemap part but not found in DFS:\n", stderr);
+            ForEachItemIn(i, notFound)
+                fprintf(stderr, "  %s\n", notFound.item(i));
+            fputs("\n", stderr);
+        }
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'add-part' command will add the packagemap part to an existing packagemap\n"
+                    "\n"
+                    "ecl packagemap add-part [options] <target> <pmid> <filename>\n"
+                    "   <target>                    Name of target to use when adding packagemap part\n"
+                    "   <pmid>                      Identifier of packagemap to add the part to\n"
+                    "   <filename>                  Name of file containing packagemap part content\n"
+                    " Options:\n"
+                    "   --part-name                 Name of part being added (defaults to filename)\n"
+                    "   --delete-previous           Replace an existing part with matching name\n"
+                    "   --daliip=<ip>               IP of the remote dali to use for logical file lookups\n"
+                    "   --global-scope              The specified packagemap is shared across multiple targets\n"
+                    "   --source-process=<value>    Process cluster to copy files from\n"
+                    "   --allow-foreign             Do not fail if foreign files are used in packagemap\n"
+                    "   --preload-all               Set preload files option for all packages\n"
+                    "   --update-super-files        Update local DFS super-files if remote DALI has changed\n"
+                    "   --update-clone-from         Update local clone from location if remote DALI has changed\n"
+                    "   --dont-append-cluster       Only use to avoid locking issues due to adding cluster to file\n",
+                    stdout);
+
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optPMID;
+    StringAttr optTarget;
+    StringAttr optDaliIP;
+    StringAttr optSourceProcess;
+    StringAttr optPartName;
+    StringAttr optFileName;
+    bool optDeletePrevious;
+    bool optGlobalScope;
+    bool optAllowForeign;
+    bool optPreloadAll;
+    bool optUpdateSuperfiles;
+    bool optUpdateCloneFrom;
+    bool optDontAppendCluster;
+};
+
+class EclCmdPackageRemovePart : public EclCmdCommon
+{
+public:
+    EclCmdPackageRemovePart()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+            return false;
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (!optTarget.length())
+                    optTarget.set(arg);
+                else if (!optPMID.length())
+                    optPMID.set(arg);
+                else if (!optPartName.length())
+                    optPartName.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optPMID.isEmpty())
+            err.append("\n ... Missing package map name\n");
+        else if (optPartName.isEmpty())
+            err.append("\n ... Missing part name\n");
+        else if (optTarget.isEmpty())
+            err.append("\n ... Specify a target cluster name\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            return false;
+        }
+
+        return true;
+    }
+    virtual int processCMD()
+    {
+        fprintf(stdout, "\n ... removing part %s from packagemap %s\n\n", optPartName.get(), optPMID.get());
+
+        Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
+        Owned<IClientRemovePartFromPackageMapRequest> request = packageProcessClient->createRemovePartFromPackageMapRequest();
+        request->setTarget(optTarget);
+        request->setPackageMap(optPMID);
+        request->setGlobalScope(optGlobalScope);
+        request->setPartName(optPartName);
+
+        Owned<IClientRemovePartFromPackageMapResponse> resp = packageProcessClient->RemovePartFromPackageMap(request);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+        else
+            printf("Successfully removed part %s from package %s\n", optPartName.get(), optPMID.get());
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'remove-part' command will remove the given part from the given packagemap\n"
+                    "\n"
+                    "ecl packagemap remove-part <target> <packagemap> <partname>\n"
+                    "   <target>               Name of the target to use \n"
+                    "   <packagemap>           Name of the package map containing the part\n"
+                    "   <partname>             Name of the part to remove\n"
+                    " Options:\n"
+                    "   --global-scope         The specified packagemap is sharable across multiple targets\n",
+                    stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optTarget;
+    StringAttr optPMID;
+    StringAttr optPartName;
+    bool optGlobalScope;
+};
+
+class EclCmdPackageGetPart: public EclCmdCommon
+{
+public:
+    EclCmdPackageGetPart()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+            return false;
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (!optTarget.length())
+                    optTarget.set(arg);
+                else if (!optPMID.length())
+                    optPMID.set(arg);
+                else if (!optPartName.length())
+                    optPartName.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (iter.matchFlag(optGlobalScope, ECLOPT_GLOBAL_SCOPE))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optPMID.isEmpty())
+            err.append("\n ... Missing package map name\n");
+        else if (optPartName.isEmpty())
+            err.append("\n ... Missing part name\n");
+        else if (optTarget.isEmpty())
+            err.append("\n ... Specify a target cluster name\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            return false;
+        }
+
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = createCmdClient(WsPackageProcess, *this);
+        Owned<IClientGetPartFromPackageMapRequest> request = packageProcessClient->createGetPartFromPackageMapRequest();
+        request->setTarget(optTarget);
+        request->setPackageMap(optPMID);
+        request->setGlobalScope(optGlobalScope);
+        request->setPartName(optPartName);
+
+        Owned<IClientGetPartFromPackageMapResponse> resp = packageProcessClient->GetPartFromPackageMap(request);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+        else
+            printf("%s", resp->getContent());
+
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+                    "\n"
+                    "The 'get-part' command will fetch the given part from the given packagemap\n"
+                    "\n"
+                    "ecl packagemap get-part <target> <packagemap> <partname>\n"
+                    "   <target>               Name of the target to use \n"
+                    "   <packagemap>           Name of the package map containing the part\n"
+                    "   <partname>             Name of the part to get\n"
+                    " Options:\n"
+                    "   --global-scope         The specified packagemap is sharable across multiple targets\n",
+                    stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optTarget;
+    StringAttr optPMID;
+    StringAttr optPartName;
+    bool optGlobalScope;
+};
+
 IEclCommand *createPackageSubCommand(const char *cmdname)
 {
     if (!cmdname || !*cmdname)
@@ -988,7 +1353,13 @@ IEclCommand *createPackageSubCommand(const char *cmdname)
         return new EclCmdPackageValidate();
     if (strieq(cmdname, "query-files"))
         return new EclCmdPackageQueryFiles();
-return NULL;
+    if (strieq(cmdname, "add-part"))
+        return new EclCmdPackageAddPart();
+    if (strieq(cmdname, "remove-part"))
+        return new EclCmdPackageRemovePart();
+    if (strieq(cmdname, "get-part"))
+        return new EclCmdPackageGetPart();
+    return NULL;
 }
 
 //=========================================================================================
@@ -1014,6 +1385,9 @@ public:
             "      info         Return active package map information\n"
             "      validate     Validate information in the package map file \n"
             "      query-files  Show files used by a query and if/how they are mapped\n"
+            "      add-part     Add additional packagemap content to an existing packagemap\n"
+            "      get-part     Get the content of a packagemap part from a packagemap\n"
+            "      remove-part  Remove a packagemap part from a packagemap\n"
         );
     }
 };

ecl/eclcmd/eclcmd_common.hpp

@@ -99,6 +99,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_UPDATE_SUPER_FILES "--update-super-files"
 #define ECLOPT_UPDATE_CLONE_FROM "--update-clone-from"
 #define ECLOPT_DONT_APPEND_CLUSTER "--dont-append-cluster"
+#define ECLOPT_PART_NAME "--part-name"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

esp/scm/ws_packageprocess.ecm

@@ -237,6 +237,57 @@ ESPresponse [exceptions_inline] GetPackageMapSelectOptionsResponse
     ESParray<string> ProcessFilters;
 };
 
+ESPrequest AddPartToPackageMapRequest
+{
+    string Target;
+    string Process;
+    string PackageMap;
+    bool GlobalScope(0);
+    string PartName;
+    string Content;
+    boolean DeletePrevious;
+    string DaliIp;
+    string SourceProcess;
+    bool AllowForeignFiles(true);
+    bool PreloadAllPackages(false);
+    bool UpdateSuperFiles(false); //usually wouldn't be needed, packagemap referencing superfiles?
+    bool UpdateCloneFrom(false); //explicitly wan't to change where roxie will grab from
+    bool AppendCluster(true); //file exists on other local cluster, add new one, make optional in case of locking issues, but should be made to work
+};
+
+ESPresponse [exceptions_inline] AddPartToPackageMapResponse
+{
+    ESPstruct BasePackageStatus status;
+    ESParray<string, File> FilesNotFound;
+};
+
+ESPrequest RemovePartFromPackageMapRequest
+{
+    string Target;
+    string PackageMap;
+    bool GlobalScope(0);
+    string PartName;
+};
+
+ESPresponse [exceptions_inline] RemovePartFromPackageMapResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPrequest GetPartFromPackageMapRequest
+{
+    string Target;
+    string PackageMap;
+    bool GlobalScope(0);
+    string PartName;
+};
+
+ESPresponse [exceptions_inline] GetPartFromPackageMapResponse
+{
+    ESPstruct BasePackageStatus status;
+    string Content;
+};
+
 ESPservice [version("1.02"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
@@ -251,6 +302,9 @@ ESPservice [version("1.02"), exceptions_inline("./smc_xslt/exceptions.xslt")] Ws
     ESPmethod ValidatePackage(ValidatePackageRequest, ValidatePackageResponse);
     ESPmethod GetQueryFileMapping(GetQueryFileMappingRequest, GetQueryFileMappingResponse);
     ESPmethod GetPackageMapSelectOptions(GetPackageMapSelectOptionsRequest, GetPackageMapSelectOptionsResponse);
+    ESPmethod AddPartToPackageMap(AddPartToPackageMapRequest, AddPartToPackageMapResponse);
+    ESPmethod RemovePartFromPackageMap(RemovePartFromPackageMapRequest, RemovePartFromPackageMapResponse);
+    ESPmethod GetPartFromPackageMap(GetPartFromPackageMapRequest, GetPartFromPackageMapResponse);
 };
 
 SCMexportdef(WsPackageProcess);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -184,85 +184,224 @@ void makePackageActive(IPropertyTree *pkgSet, IPropertyTree *psEntryNew, const c
 
 //////////////////////////////////////////////////////////
 
-void addPackageMapInfo(unsigned updateFlags, const char *xml, StringArray &filesNotFound, const char *process, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, bool activate, IUserDescriptor* userdesc, bool allowForeignFiles, bool preloadAll)
+#define PKGADD_DFS_OVERWRITE    0x0001
+#define PKGADD_ALLOW_FOREIGN    0x0002
+#define PKGADD_PRELOAD_ALL      0x0004
+#define PKGADD_MAP_ACTIVATE     0x0100
+#define PKGADD_MAP_CREATE       0x0200
+#define PKGADD_MAP_REPLACE      0x0400
+#define PKGADD_SEG_ADD          0x1000
+#define PKGADD_SEG_REPLACE      0x2000
+
+class PackageMapUpdater
 {
-    if (!xml || !*xml)
-        throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "PackageMap info not provided");
+public:
+    Owned<IRemoteConnection> globalLock;
+    Owned<IUserDescriptor> userdesc;
+    Owned<IConstWUClusterInfo> clusterInfo;
+    Owned<IPropertyTree> pmPart;
+    IPropertyTree *packageMaps;
+    IPropertyTree *pmExisting;
+
+    StringBuffer daliIP;
+    StringBuffer srcCluster;
+    StringBuffer prefix;
+    StringBuffer pmid;
+    StringBuffer pkgSetId;
+
+    StringAttr process;
+    StringAttr target;
+    unsigned flags;
 
-    if (srcCluster && *srcCluster)
+    PackageMapUpdater() : flags(0), packageMaps(NULL), pmExisting(NULL){}
+
+    inline bool checkFlag(unsigned check)
     {
-        if (!isProcessCluster(lookupDaliIp, srcCluster))
-            throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Process cluster %s not found on %s DALI", srcCluster, lookupDaliIp ? lookupDaliIp : "local");
+        return flags & check;
     }
 
-    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
-    if (!clusterInfo)
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", target);
+    inline void setFlag(unsigned flag, bool on=true)
+    {
+        if (on)
+            flags |= flag;
+    }
+    IConstWUClusterInfo *ensureClusterInfo()
+    {
+        clusterInfo.setown(getTargetClusterInfo(target));
+        if (!clusterInfo)
+            throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", target.str());
+        return clusterInfo;
+    }
+    void setPMID(const char *_target, const char *name, bool globalScope)
+    {
+        if (!name || !*name)
+            throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "PackageMap name parameter required");
+        if (!globalScope)
+        {
+            target.set(_target);
+            if (target.isEmpty())
+                throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "Target cluster parameter required");
+            ensureClusterInfo();
+            pmid.append(target).append("::");
+        }
+        pmid.append(name);
+    }
+    void setProcess(const char *name)
+    {
+        process.set(name);
+        buildPkgSetId(pkgSetId, process);
+    }
+    void setUser(const char *user, const char *password)
+    {
+        if (user && *user && *password && *password)
+        {
+            userdesc.setown(createUserDescriptor());
+            userdesc->set(user, password);
+        }
+    }
+    void setDerivedDfsLocation(const char *dfsLocation, const char *srcProcess)
+    {
+        splitDerivedDfsLocation(dfsLocation, srcCluster, daliIP, prefix, srcProcess, srcProcess, NULL, NULL);
 
-    Owned<IPropertyTree> pmTree = createPTreeFromXMLString(xml);
-    if (!pmTree)
-        throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "Invalid PackageMap info");
+        if (srcCluster.length())
+        {
+            if (!isProcessCluster(daliIP, srcCluster))
+                throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Process cluster %s not found on %s DALI", srcCluster.str(), daliIP.length() ? daliIP.str() : "local");
+        }
+    }
+    void init()
+    {
+        VStringBuffer xpath("PackageMap[@id='%s']", pmid.str());
+        globalLock.setown(querySDS().connect("/PackageMaps", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
+        packageMaps = globalLock->queryRoot();
+        pmExisting = packageMaps->queryPropTree(xpath);
+    }
+    void createPart(const char *partname, const char *xml)
+    {
+        if (!partname || !*partname)
+            throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "No PackageMap Part name provided");
+        if (!xml || !*xml)
+            throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "No PackageMap content provided");
 
-    StringBuffer lcPmid(pmid);
-    pmid = lcPmid.toLowerCase().str();
-    pmTree->setProp("@id", pmid);
+        pmPart.setown(createPTreeFromXMLString(xml));
+        if (!pmPart)
+            throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "Invalid PackageMap content");
+        pmPart->addProp("@id", partname);
 
-    Owned<IPropertyTreeIterator> iter = pmTree->getElements("Package");
-    ForEach(*iter)
-    {
-        IPropertyTree &item = iter->query();
-        if (preloadAll)
-            item.setPropBool("@preload", true);
-        Owned<IPropertyTreeIterator> superFiles = item.getElements("SuperFile");
-        ForEach(*superFiles)
+        StringBuffer lcPmid(pmid);
+        pmid = lcPmid.toLowerCase().str();
+
+        Owned<IPropertyTreeIterator> iter = pmPart->getElements("Package");
+        ForEach(*iter)
         {
-            IPropertyTree &superFile = superFiles->query();
-            StringBuffer lc(superFile.queryProp("@id"));
-            const char *id = lc.toLowerCase().str();
-            if (*id == '~')
-                id++;
-            superFile.setProp("@id", id);
-
-            Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile");
-            ForEach(*subFiles)
+            IPropertyTree &item = iter->query();
+            if (checkFlag(PKGADD_PRELOAD_ALL))
+                item.setPropBool("@preload", true);
+            Owned<IPropertyTreeIterator> superFiles = item.getElements("SuperFile");
+            ForEach(*superFiles)
             {
-                IPropertyTree &subFile = subFiles->query();
-                id = subFile.queryProp("@value");
-                if (id && *id == '~')
+                IPropertyTree &superFile = superFiles->query();
+                StringBuffer lc(superFile.queryProp("@id"));
+                const char *id = lc.toLowerCase().str();
+                if (*id == '~')
+                    id++;
+                superFile.setProp("@id", id);
+
+                Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile");
+                ForEach(*subFiles)
                 {
-                    StringAttr value(id+1);
-                    subFile.setProp("@value", value.get());
+                    IPropertyTree &subFile = subFiles->query();
+                    id = subFile.queryProp("@value");
+                    if (id && *id == '~')
+                    {
+                        StringAttr value(id+1);
+                        subFile.setProp("@value", value.get());
+                    }
                 }
             }
         }
     }
+    void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound)
+    {
+        cloneFileInfoToDali(updateFlags, filesNotFound, pmPart, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+    }
+    void create(const char *partname, const char *xml, unsigned updateFlags, StringArray &filesNotFound)
+    {
+        init();
+        createPart(partname, xml);
+
+        if (pmExisting)
+        {
+            if (!checkFlag(PKGADD_MAP_REPLACE))
+                throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s already exists, either delete it or specify overwrite", pmid.str());
+        }
 
-    VStringBuffer xpath("PackageMap[@id='%s']", pmid);
-    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
-    IPropertyTree *packageMaps = globalLock->queryRoot();
-    IPropertyTree *pmExisting = packageMaps->queryPropTree(xpath);
+        cloneDfsInfo(updateFlags, filesNotFound);
 
-    xpath.appendf("[@querySet='%s']", target);
-    Owned<IPropertyTree> pkgSet = getPkgSetRegistry(process, false);
-    IPropertyTree *psEntry = pkgSet->queryPropTree(xpath);
+        if (pmExisting)
+            packageMaps->removeTree(pmExisting);
 
-    if (!(updateFlags & DALI_UPDATEF_PACKAGEMAP) && (psEntry || pmExisting))
-        throw MakeStringException(PKG_NAME_EXISTS, "Package name %s already exists, either delete it or specify replace packagemap", pmid);
+        Owned<IPropertyTree> pmTree = createPTree("PackageMap", ipt_ordered);
+        pmTree->setProp("@id", pmid);
+        pmTree->addPropTree("Part", pmPart.getClear());
+        packageMaps->addPropTree("PackageMap", pmTree.getClear());
 
-    cloneFileInfoToDali(updateFlags, filesNotFound, pmTree, lookupDaliIp, clusterInfo, srcCluster, prefix, userdesc, allowForeignFiles);
+        VStringBuffer xpath("PackageMap[@id='%s'][@querySet='%s']", pmid.str(), target.get());
+        Owned<IPropertyTree> pkgSet = getPkgSetRegistry(process, false);
+        IPropertyTree *psEntry = pkgSet->queryPropTree(xpath);
 
-    if (pmExisting)
-        packageMaps->removeTree(pmExisting);
-    packageMaps->addPropTree("PackageMap", pmTree.getClear());
+        if (!psEntry)
+        {
+            psEntry = pkgSet->addPropTree("PackageMap", createPTree("PackageMap"));
+            psEntry->setProp("@id", pmid);
+            psEntry->setProp("@querySet", target);
+        }
+        makePackageActive(pkgSet, psEntry, target, checkFlag(PKGADD_MAP_ACTIVATE));
+    }
 
-    if (!psEntry)
+    void addPart(const char *partname, const char *xml, unsigned updateFlags, StringArray &filesNotFound)
     {
-        psEntry = pkgSet->addPropTree("PackageMap", createPTree("PackageMap"));
-        psEntry->setProp("@id", pmid);
-        psEntry->setProp("@querySet", target);
+        init();
+        createPart(partname, xml);
+
+        if (!pmExisting)
+            throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s not found, create it before adding additional parts/files", pmid.str());
+
+        VStringBuffer xpath("Part[@id='%s']", partname);
+        IPropertyTree *existingPart = pmExisting->queryPropTree(xpath);
+        if (existingPart && !checkFlag(PKGADD_SEG_REPLACE))
+            throw MakeStringException(PKG_NAME_EXISTS, "Package Part %s already exists, remove, or specify 'delete previous'", partname);
+
+        cloneDfsInfo(updateFlags, filesNotFound);
+
+        if (existingPart)
+            pmExisting->removeTree(existingPart);
+
+        pmExisting->addPropTree("Part", pmPart.getClear());
     }
-    makePackageActive(pkgSet, psEntry, target, activate);
-}
+    IPropertyTree *ensurePart(const char *partname)
+    {
+        if (!pmExisting)
+            throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s not found", pmid.str());
+
+        VStringBuffer xpath("Part[@id='%s']", partname);
+        IPropertyTree *existingPart = pmExisting->queryPropTree(xpath);
+        if (!existingPart)
+            throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s Part %s not found", pmid.str(), partname);
+        return existingPart;
+    }
+    void removePart(const char *partname)
+    {
+        init();
+        pmExisting->removeTree(ensurePart(partname));
+    }
+    StringBuffer &getPartContent(const char *partname, StringBuffer &content)
+    {
+        init();
+        return toXML(ensurePart(partname), content);
+    }
+};
+
 
 void getPackageListInfo(IPropertyTree *mapTree, IEspPackageListMapData *pkgList)
 {
@@ -528,24 +667,18 @@ void CWsPackageProcessEx::getPkgInfoById(const char *packageMapId, IPropertyTree
 
 bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageRequest &req, IEspAddPackageResponse &resp)
 {
-    resp.updateStatus().setCode(0);
-
-    StringAttr target(req.getTarget());
-    StringAttr name(req.getPackageMap());
-
-    if (target.isEmpty())
-        throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "Target cluster parameter required");
-    if (name.isEmpty())
-        throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "PackageMap name parameter required");
-
-    DBGLOG("%s adding packagemap %s to target %s", context.queryUserId(), name.str(), target.str());
+    PackageMapUpdater updater;
+    updater.setFlag(PKGADD_MAP_CREATE);
+    updater.setFlag(PKGADD_MAP_ACTIVATE, req.getActivate());
+    updater.setFlag(PKGADD_MAP_REPLACE, req.getOverWrite());
+    updater.setFlag(PKGADD_ALLOW_FOREIGN, req.getAllowForeignFiles());
+    updater.setFlag(PKGADD_PRELOAD_ALL, req.getPreloadAllPackages());
+
+    updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
+    updater.setProcess(req.getProcess());
+    updater.setUser(context.queryUserId(), context.queryPassword());
+    updater.setDerivedDfsLocation(req.getDaliIp(), req.getSourceProcess());
 
-    StringBuffer pmid;
-    if (!req.getGlobalScope())
-        pmid.append(target).append("::");
-    pmid.append(name.get());
-
-    bool activate = req.getActivate();
     unsigned updateFlags = 0;
     if (req.getOverWrite())
         updateFlags |= (DALI_UPDATEF_PACKAGEMAP | DALI_UPDATEF_REPLACE_FILE | DALI_UPDATEF_CLONE_FROM | DALI_UPDATEF_SUPERFILES);
@@ -558,32 +691,12 @@ bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageReque
     if (req.getAppendCluster())
         updateFlags |= DALI_UPDATEF_APPEND_CLUSTER;
 
-    StringAttr processName(req.getProcess());
-
-    Owned<IUserDescriptor> userdesc;
-    const char *user = context.queryUserId();
-    const char *password = context.queryPassword();
-    if (user && *user && *password && *password)
-    {
-        userdesc.setown(createUserDescriptor());
-        userdesc->set(user, password);
-    }
-
-    StringBuffer srcCluster;
-    StringBuffer daliip;
-    StringBuffer prefix;
-    splitDerivedDfsLocation(req.getDaliIp(), srcCluster, daliip, prefix, req.getSourceProcess(), req.getSourceProcess(), NULL, NULL);
-
-    StringBuffer pkgSetId;
-    buildPkgSetId(pkgSetId, processName.get());
-
     StringArray filesNotFound;
-    addPackageMapInfo(updateFlags, req.getInfo(), filesNotFound, processName, target, pmid, pkgSetId, daliip, srcCluster, prefix, activate, userdesc, req.getAllowForeignFiles(), req.getPreloadAllPackages());
+    updater.create(req.getPackageMap(), req.getInfo(), updateFlags, filesNotFound);
     resp.setFilesNotFound(filesNotFound);
 
-    StringBuffer msg;
-    msg.append("Successfully loaded ").append(name.get());
-    resp.updateStatus().setDescription(msg.str());
+    resp.updateStatus().setCode(0);
+    resp.updateStatus().setDescription(StringBuffer("Successfully loaded ").append(req.getPackageMap()));
     return true;
 }
 
@@ -1028,6 +1141,59 @@ bool CWsPackageProcessEx::onGetQueryFileMapping(IEspContext &context, IEspGetQue
     return true;
 }
 
+bool CWsPackageProcessEx::onAddPartToPackageMap(IEspContext &context, IEspAddPartToPackageMapRequest &req, IEspAddPartToPackageMapResponse &resp)
+{
+    PackageMapUpdater updater;
+    updater.setFlag(PKGADD_SEG_ADD);
+    updater.setFlag(PKGADD_SEG_REPLACE, req.getDeletePrevious());
+    updater.setFlag(PKGADD_ALLOW_FOREIGN, req.getAllowForeignFiles());
+    updater.setFlag(PKGADD_PRELOAD_ALL, req.getPreloadAllPackages());
+
+    updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
+    updater.setProcess(req.getProcess());
+    updater.setUser(context.queryUserId(), context.queryPassword());
+    updater.setDerivedDfsLocation(req.getDaliIp(), req.getSourceProcess());
+
+    unsigned updateFlags = 0;
+    if (req.getDeletePrevious())
+        updateFlags |= DALI_UPDATEF_PACKAGEMAP;
+    if (req.getUpdateCloneFrom())
+        updateFlags |= DALI_UPDATEF_CLONE_FROM;
+    if (req.getUpdateSuperFiles())
+        updateFlags |= DALI_UPDATEF_SUPERFILES;
+    if (req.getAppendCluster())
+        updateFlags |= DALI_UPDATEF_APPEND_CLUSTER;
+
+    StringArray filesNotFound;
+    updater.addPart(req.getPartName(), req.getContent(), updateFlags, filesNotFound);
+    resp.setFilesNotFound(filesNotFound);
+
+    resp.updateStatus().setCode(0);
+    resp.updateStatus().setDescription(VStringBuffer("Successfully loaded Part %s to PackageMap %s", req.getPartName(), updater.pmid.str()));
+    return true;
+}
+
+bool CWsPackageProcessEx::onGetPartFromPackageMap(IEspContext &context, IEspGetPartFromPackageMapRequest &req, IEspGetPartFromPackageMapResponse &resp)
+{
+    PackageMapUpdater updater;
+    updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
+
+    StringBuffer content;
+    resp.setContent(updater.getPartContent(req.getPartName(), content));
+    return true;
+}
+
+bool CWsPackageProcessEx::onRemovePartFromPackageMap(IEspContext &context, IEspRemovePartFromPackageMapRequest &req, IEspRemovePartFromPackageMapResponse &resp)
+{
+    PackageMapUpdater updater;
+    updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
+    updater.removePart(req.getPartName());
+
+    resp.updateStatus().setCode(0);
+    resp.updateStatus().setDescription(VStringBuffer("Successfully removed Part %s from PackageMap %s", req.getPartName(), updater.pmid.str()));
+    return true;
+}
+
 int CWsPackageProcessSoapBindingEx::onFinishUpload(IEspContext &ctx, CHttpRequest* request, CHttpResponse* response,
     const char *service, const char *method, StringArray& fileNames, StringArray& files, IMultiException *meIn)
 {

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -159,6 +159,10 @@ public:
     virtual bool onListPackages(IEspContext &context, IEspListPackagesRequest &req, IEspListPackagesResponse &resp);
     virtual bool onGetPackageMapSelectOptions(IEspContext &context, IEspGetPackageMapSelectOptionsRequest &req, IEspGetPackageMapSelectOptionsResponse &resp);
     virtual bool onGetPackageMapById(IEspContext &context, IEspGetPackageMapByIdRequest &req, IEspGetPackageMapByIdResponse &resp);
+    virtual bool onAddPartToPackageMap(IEspContext &context, IEspAddPartToPackageMapRequest &req, IEspAddPartToPackageMapResponse &resp);
+    virtual bool onGetPartFromPackageMap(IEspContext &context, IEspGetPartFromPackageMapRequest &req, IEspGetPartFromPackageMapResponse &resp);
+    virtual bool onRemovePartFromPackageMap(IEspContext &context, IEspRemovePartFromPackageMapRequest &req, IEspRemovePartFromPackageMapResponse &resp);
+
     PackageMapAndSet packageMapAndSet;
 };