Merge pull request #1854 from stuartort/CE-pkg

Move package code to community release

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/CMakeLists.txt

@@ -23,6 +23,7 @@ add_subdirectory (eclccserver)
 add_subdirectory (eclcmd)
 add_subdirectory (eclscheduler)
 add_subdirectory (eclplus)
+add_subdirectory (ecl-package)
 add_subdirectory (hql)
 add_subdirectory (hqlcpp)
 add_subdirectory (hthor)

ecl/ecl-package/CMakeLists.txt

@@ -0,0 +1,54 @@
+##############################################################################
+# Copyright (C) 2011 HPCC Systems.
+##############################################################################
+
+
+# Component: ecl-package
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for ecl-package
+#####################################################
+
+
+project( ecl-package )
+
+include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
+
+set (    SRCS
+         ${ESPSCM_GENERATED_DIR}/ws_packageprocess_esp.cpp
+         ecl-package.cpp
+         ${HPCC_SOURCE_DIR}/ecl/eclcmd/eclcmd_shell.cpp
+         ${HPCC_SOURCE_DIR}/ecl/eclcmd/eclcmd_common.hpp
+         ${HPCC_SOURCE_DIR}/ecl/eclcmd/eclcmd_common.cpp
+    )
+
+include_directories (
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/jlib
+         ${HPCC_SOURCE_DIR}/common/workunit
+         ${HPCC_SOURCE_DIR}/esp/clients
+         ${HPCC_SOURCE_DIR}/esp/bindings
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+         ${HPCC_SOURCE_DIR}/esp/platform
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/ecl/eclcmd
+    )
+
+ADD_DEFINITIONS( -D_CONSOLE )
+
+add_executable ( ecl-package ${SRCS} )
+add_dependencies ( ecl-package espscm ws_packageprocess )
+install ( TARGETS ecl-package DESTINATION ${DIR_NAME}/bin )
+target_link_libraries ( ecl-package
+        jlib
+        esphttp
+        workunit
+    )
+
+install ( PROGRAMS ecl-package.install DESTINATION ${DIR_NAME}/etc/init.d/install COMPONENT Runtime )
+

ecl/ecl-package/ecl-package.cpp

@@ -0,0 +1,676 @@
+/*##############################################################################
+Copyright (C) 2011 HPCC Systems.
+############################################################################## */
+
+#include <stdio.h>
+#include "jlog.hpp"
+#include "jfile.hpp"
+#include "jargv.hpp"
+
+#include "build-config.h"
+
+#include "ws_packageprocess_esp.ipp"
+
+#include "eclcmd.hpp"
+#include "eclcmd_common.hpp"
+#include "eclcmd_core.hpp"
+
+#define ECLOPT_PACKAGEMAP "--packagemap"
+#define ECLOPT_OVERWRITE "--overwrite"
+#define ECLOPT_PACKAGE "--packagename"
+#define ECLOPT_DALIIP "--daliip"
+#define ECLOPT_PROCESS "--process"
+
+//=========================================================================================
+
+IClientWsPackageProcess *getWsPackageSoapService(const char *server, const char *port, const char *username, const char *password)
+{
+    if(server == NULL)
+        throw MakeStringException(-1, "Server url not specified");
+
+    VStringBuffer url("http://%s:%s/WsPackageProcess", server, port);
+
+    IClientWsPackageProcess *packageProcessClient = createWsPackageProcessClient();
+    packageProcessClient->addServiceUrl(url.str());
+    packageProcessClient->setUsernameToken(username, password, NULL);
+
+    return packageProcessClient;
+}
+
+
+class EclCmdPackageActivate : public EclCmdCommon
+{
+public:
+    EclCmdPackageActivate()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optQuerySetName.set(arg);
+                return true;
+            }
+            if (iter.matchOption(optPackageMap, ECLOPT_PACKAGEMAP))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        if (optQuerySetName.isEmpty())
+        {
+            fprintf(stdout, "\n ... Missing package name\n\n");
+            usage();
+            return false;
+        }
+
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+
+        Owned<IClientActivatePackageRequest> request = packageProcessClient->createActivatePackageRequest();
+        request->setPackageName(optQuerySetName);
+        request->setPackageMapName(optPackageMap);
+
+        Owned<IClientActivatePackageResponse> resp = packageProcessClient->ActivatePackage(request);
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package activate [options] [<querySetName>]\n\n"
+            "   Options:\n"
+            "      --packagemap=<packagemap>        name of packagemap to update\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+
+    StringAttr optQuerySetName;
+    StringAttr optPackageMap;
+};
+
+class EclCmdPackageDeActivate : public EclCmdCommon
+{
+public:
+    EclCmdPackageDeActivate()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optQuerySetName.set(arg);
+                break;
+            }
+            switch (EclCmdCommon::matchCommandLineOption(iter))
+            {
+                case EclCmdOptionNoMatch:
+                    fprintf(stderr, "\n%s option not recognized\n", arg);
+                    return false;
+                case EclCmdOptionCompletion:
+                    return false;
+                case EclCmdOptionMatch:
+                    break;
+            }
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        if (optQuerySetName.isEmpty())
+        {
+            fprintf(stdout, "\n ... Missing package name\n\n");
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+
+        Owned<IClientDeActivatePackageRequest> request = packageProcessClient->createDeActivatePackageRequest();
+        request->setPackageName(optQuerySetName);
+        request->setPackageMapName(optPackageMap);
+
+        Owned<IClientDeActivatePackageResponse> resp = packageProcessClient->DeActivatePackage(request);
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package deactivate [options] [<querySetName>]\n\n"
+            "   Options:\n"
+            "      --packagemap=<packagemap>        name of packagemap to update\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+
+    StringAttr optQuerySetName;
+    StringAttr optPackageMap;
+};
+
+class EclCmdPackageList : public EclCmdCommon
+{
+public:
+    EclCmdPackageList()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optCluster.set(arg);
+                return true;
+            }
+            if (iter.matchOption(optCluster, ECLOPT_CLUSTER))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        if (optCluster.isEmpty())
+        {
+            fprintf(stderr, "\nCluster must be specified\n");
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+
+        Owned<IClientListPackageRequest> request = packageProcessClient->createListPackageRequest();
+        request->setCluster(optCluster);
+
+        Owned<IClientListPackageResponse> resp = packageProcessClient->ListPackage(request);
+
+        IArrayOf<IConstPackageListMapData> &pkgMapInfo = resp->getPkgListMapData();
+        unsigned int num = pkgMapInfo.ordinality();
+
+        for (unsigned i=0; i<num; i++)
+        {
+            IConstPackageListMapData& req = pkgMapInfo.item(i);
+            const char *id = req.getId();
+            printf("\nPackage Name = %s\n", id);
+            IArrayOf<IConstPackageListData> &pkgInfo = req.getPkgListData();
+
+            unsigned int numPkgs = pkgInfo.ordinality();
+            for (unsigned int j = 0; j <numPkgs; j++)
+            {
+                IConstPackageListData& req = pkgInfo.item(j);
+                const char *id = req.getId();
+                const char *queries = req.getQueries();
+                if (queries && *queries)
+                    printf("\t\tid = %s  queries = %s\n", id, queries);
+                else
+                    printf("\t\tid = %s\n", id);
+            }
+        }
+
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package list [options] \n\n"
+            "   Options:\n"
+            "      [--cluster<cluster>   name of cluster to retrieve package information.  Defaults to all package information stored in dali\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+
+    StringAttr optCluster;
+};
+
+class EclCmdPackageInfo: public EclCmdCommon
+{
+public:
+    EclCmdPackageInfo()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optPkgName.set(arg);
+                return true;
+            }
+            else if (iter.matchOption(optCluster, ECLOPT_CLUSTER))
+                continue;
+            else if (iter.matchOption(optPkgName, ECLOPT_PACKAGE))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (!optPkgName.isEmpty() && !optCluster.isEmpty())
+            err.append("\n ... Specify either a cluster name of a package name, but NOT both\n\n");
+        else if (optPkgName.isEmpty() && optCluster.isEmpty())
+            err.append("\n ... Specify either a cluster name of a package name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+
+        Owned<IClientGetPackageRequest> request = packageProcessClient->createGetPackageRequest();
+        request->setPackageName(optPkgName);
+        request->setCluster(optCluster);
+
+        Owned<IClientGetPackageResponse> resp = packageProcessClient->GetPackage(request);
+        printf("%s", resp->getInfo());
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package info [options] [<packageName>]\n\n"
+            "   Options:\n"
+            "      [--cluster=<cluster> | --packageName=<packageName>]  specify either a cluster name or a pacakge name to retrieve information\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+
+    StringAttr optPkgName;
+    StringAttr optCluster;
+};
+
+class EclCmdPackageDelete : public EclCmdCommon
+{
+public:
+    EclCmdPackageDelete()
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optFileName.set(arg);
+                return true;
+            }
+            if (iter.matchOption(optQuerySet, ECLOPT_QUERYSET))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n\n");
+        else if (optQuerySet.isEmpty())
+            err.append("\n ... Specify either a cluster name of a package name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        StringBuffer pkgInfo;
+        pkgInfo.loadFile(optFileName);
+
+        fprintf(stdout, "\n ... deleting package %s now\n\n", optFileName.sget());
+
+        Owned<IClientDeletePackageRequest> request = packageProcessClient->createDeletePackageRequest();
+        request->setQuerySet(optQuerySet);
+        request->setPackageName(optFileName);
+
+        Owned<IClientDeletePackageResponse> resp = packageProcessClient->DeletePackage(request);
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package delete [options] [<filename>]\n\n"
+            "   Options:\n"
+            "      --queryset=<queryset>        name of queryset to associate the information\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optFileName;
+    StringAttr optQuerySet;
+};
+
+class EclCmdPackageAdd : public EclCmdCommon
+{
+public:
+    EclCmdPackageAdd() : optActivate(true), optOverWrite(true)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optFileName.set(arg);
+                return true;
+            }
+            if (iter.matchOption(optQuerySet, ECLOPT_QUERYSET))
+                continue;
+            if (iter.matchFlag(optActivate, ECLOPT_ACTIVATE))
+                continue;
+            if (iter.matchFlag(optOverWrite, ECLOPT_OVERWRITE))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n\n");
+        else if (optQuerySet.isEmpty())
+            err.append("\n ... Specify either a cluster name of a package name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        StringBuffer pkgInfo;
+        pkgInfo.loadFile(optFileName);
+
+        fprintf(stdout, "\n ... adding package %s now\n\n", optFileName.sget());
+
+        Owned<IClientAddPackageRequest> request = packageProcessClient->createAddPackageRequest();
+        request->setActivate(optActivate);
+        request->setInfo(pkgInfo);
+        request->setQuerySet(optQuerySet);
+        request->setPackageName(optFileName);
+        request->setOverWrite(optOverWrite);
+
+        Owned<IClientAddPackageResponse> resp = packageProcessClient->AddPackage(request);
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package add [options] [<filename>]\n\n"
+            "   Options:\n"
+            "      --queryset=<queryset>        name of queryset to associate the information\n"
+            "      --overwritename=<true/false> overwrite existing information - defaults to true\n"
+            "      --activate=<true/false>      activate the package information - defaults to true\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optFileName;
+    StringAttr optQuerySet;
+    bool optActivate;
+    bool optOverWrite;
+    StringBuffer pkgInfo;
+
+};
+
+
+
+class EclCmdPackageCopyFiles : public EclCmdCommon
+{
+public:
+    EclCmdPackageCopyFiles() :optOverWrite (false)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optFileName.set(arg);
+                return true;
+            }
+            if (iter.matchOption(optProcess, ECLOPT_PROCESS))
+                continue;
+            if (iter.matchFlag(optDaliIp, ECLOPT_DALIIP))
+                continue;
+            if (iter.matchFlag(optOverWrite, ECLOPT_OVERWRITE))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+        {
+            usage();
+            return false;
+        }
+        StringBuffer err;
+        if (optFileName.isEmpty())
+            err.append("\n ... Missing package file name\n\n");
+        else if (optProcess.isEmpty())
+            err.append("\n ... Specify a process name\n\n");
+
+        if (err.length())
+        {
+            fprintf(stdout, "%s", err.str());
+            usage();
+            return false;
+        }
+        return true;
+    }
+    virtual int processCMD()
+    {
+        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
+        StringBuffer pkgInfo;
+        pkgInfo.loadFile(optFileName);
+
+        fprintf(stdout, "\n ... looking up files in package to see what needs copying\n\n");
+
+        Owned<IClientCopyFilesRequest> request = packageProcessClient->createCopyFilesRequest();
+        request->setInfo(pkgInfo);
+        request->setProcess(optProcess);
+        request->setPackageName(optFileName);
+        request->setOverWrite(optOverWrite);
+
+        Owned<IClientCopyFilesResponse> resp = packageProcessClient->CopyFiles(request);
+        return 0;
+    }
+
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package copyFiles [options] [<filename>]\n\n"
+            "   Options:\n"
+            "      --process=<processcluster>    name of the process cluster to copy files\n"
+            "      --overwrite=<true/false>      overwrite data file if it already exists on the target cluster. defaults to false\n"
+            "      --daliip=<daliip>            ip of the source dali to use for file lookups\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optFileName;
+    StringAttr optProcess;
+    StringAttr optDaliIp;
+    StringBuffer pkgInfo;
+    bool optOverWrite;
+};
+
+IEclCommand *createPackageSubCommand(const char *cmdname)
+{
+    if (!cmdname || !*cmdname)
+        return NULL;
+    if (strieq(cmdname, "add"))
+        return new EclCmdPackageAdd();
+    if (strieq(cmdname, "delete"))
+        return new EclCmdPackageDelete();
+    if (strieq(cmdname, "activate"))
+        return new EclCmdPackageActivate();
+    if (strieq(cmdname, "deactivate"))
+        return new EclCmdPackageDeActivate();
+    if (strieq(cmdname, "info"))
+        return new EclCmdPackageInfo();
+    if (strieq(cmdname, "list"))
+        return new EclCmdPackageList();
+    if (strieq(cmdname, "copyFiles"))
+        return new EclCmdPackageCopyFiles();
+    return NULL;
+}
+
+//=========================================================================================
+
+class PackageCMDShell : public EclCMDShell
+{
+public:
+    PackageCMDShell(int argc, const char *argv[], EclCommandFactory _factory, const char *_version)
+        : EclCMDShell(argc, argv, _factory, _version)
+    {
+    }
+
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl package <command> [command options]\n\n"
+            "   Package Commands:\n"
+            "      add          add a package to the environment\n"
+            "      copyFiles    copy missing data files to the appropriate cluster\n"
+            "      delete       delete a package\n"
+            "      activate     activate a package\n"
+            "      deactivate   deactivate a package (package will not get loaded)\n"
+            "      list         list loaded package names\n"
+            "      info         return active package information for a cluster\n"
+        );
+    }
+};
+
+static int doMain(int argc, const char *argv[])
+{
+    PackageCMDShell processor(argc, argv, createPackageSubCommand, BUILD_TAG);
+    return processor.run();
+}
+
+int main(int argc, const char *argv[])
+{
+    InitModuleObjects();
+    queryStderrLogMsgHandler()->setMessageFields(0);
+    unsigned exitCode = doMain(argc, argv);
+    releaseAtoms();
+    exit(exitCode);
+}

ecl/ecl-package/ecl-package.install

@@ -0,0 +1 @@
+installFile "$binPath/ecl-package" "/usr/bin/ecl-package" 1 || exit 1

ecl/ecl-package/sourcedoc.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>ecl/ecl</title>
+
+    <para>
+        The ecl/ecl directory contains the sources for the ecl command line tool.
+    </para>
+</section>

esp/scm/espscm.cmake

@@ -42,6 +42,7 @@ set ( ESPSCM_SRCS
       ws_smc.ecm
       ws_topology.ecm
       ws_workunits.ecm
+      ws_packageprocess.ecm
     )
 
 foreach ( loop_var ${ESPSCM_SRCS} )

esp/scm/smcscm.cmake

@@ -36,6 +36,7 @@ set ( ESPSCM_SRCS
       ws_smc.ecm
       ws_topology.ecm
       ws_workunits.ecm
+      ws_packageprocess.ecm
     )
 
 foreach ( loop_var ${ESPSCM_SRCS} )

esp/scm/ws_packageprocess.ecm

@@ -0,0 +1,134 @@
+/*##############################################################################
+Copyright (C) 2011 HPCC Systems.
+############################################################################## */
+
+ESPrequest EchoRequest
+{
+    string Request;
+};
+
+ESPresponse EchoResponse
+{
+    string Response;
+};
+
+ESPstruct BasePackageStatus
+{
+    int Code;
+    string Description;
+};
+
+
+ESPrequest AddPackageRequest
+{
+    string Info;
+    boolean Activate;
+    boolean OverWrite;
+    string QuerySet;
+    string PackageName;
+};
+
+
+ESPresponse AddPackageResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPrequest DeletePackageRequest
+{
+    string QuerySet;
+    string PackageName;
+};
+
+ESPresponse DeletePackageResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPrequest ActivatePackageRequest
+{
+    string PackageName;
+    string PackageMapName;
+};
+
+ESPresponse ActivatePackageResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPrequest DeActivatePackageRequest
+{
+    string PackageName;
+    string PackageMapName;
+};
+
+ESPresponse DeActivatePackageResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPrequest GetPackageRequest
+{
+    string Cluster;
+    string PackageName;
+};
+
+ESPresponse GetPackageResponse
+{
+    ESPstruct BasePackageStatus status;
+    string Info;
+};
+
+ESPrequest ListPackageRequest
+{
+    string Cluster;
+};
+
+ESPstruct PackageListData
+{
+    string Id;
+    string Queries;
+};
+
+ESPstruct PackageListMapData
+{
+    string Id;
+    ESParray<ESPstruct PackageListData> PkgListData;
+};
+
+ESPresponse ListPackageResponse
+{
+    ESPstruct BasePackageStatus status;
+    ESParray<ESPstruct PackageListMapData> PkgListMapData;
+};
+
+ESPrequest CopyFilesRequest
+{
+    string Process;
+    string PackageName;
+    string DaliIp;
+    string Info;
+    bool OverWrite;
+};
+
+
+ESPresponse CopyFilesResponse
+{
+    ESPstruct BasePackageStatus status;
+};
+
+ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
+{
+    ESPmethod Echo(EchoRequest, EchoResponse);
+    ESPmethod AddPackage(AddPackageRequest, AddPackageResponse);
+    ESPmethod DeletePackage(DeletePackageRequest, DeletePackageResponse);
+    ESPmethod ActivatePackage(ActivatePackageRequest, ActivatePackageResponse);
+    ESPmethod DeActivatePackage(DeActivatePackageRequest, DeActivatePackageResponse);
+    ESPmethod ListPackage(ListPackageRequest, ListPackageResponse);
+    ESPmethod GetPackage(GetPackageRequest, GetPackageResponse);
+    ESPmethod CopyFiles(CopyFilesRequest, CopyFilesResponse);
+};
+
+SCMexportdef(WsPackageProcess);
+
+SCMapi(WsPackageProcess) IClientWsPackageProcess *createWsPackageProcessClient();

esp/services/CMakeLists.txt

@@ -29,4 +29,5 @@ add_subdirectory (ws_roxiequery)
 add_subdirectory (ws_smc)
 add_subdirectory (ws_topology)
 add_subdirectory (ws_workunits)
-add_subdirectory (WsDeploy)
+add_subdirectory (WsDeploy)
+add_subdirectory (ws_packageprocess)

esp/services/ws_packageprocess/CMakeLists.txt

@@ -0,0 +1,58 @@
+##############################################################################
+# Copyright (C) 2011 HPCC Systems.
+##############################################################################
+
+# Component: ws_packageprocess
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for ws_packageprocess
+#####################################################
+
+project( ws_packageprocess )
+
+include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
+
+set (    SRCS
+         ${HPCC_SOURCE_DIR}/dali/dfu/dfuutil.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_fs_esp.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_packageprocess_esp.cpp
+         ${HPCC_SOURCE_DIR}/esp/scm/ws_packageprocess.ecm
+         ws_packageprocessPlugin.cpp
+         ws_packageprocessService.cpp
+         ws_packageprocessService.hpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/esp/platform
+         ${HPCC_SOURCE_DIR}/system/jlib
+         ${HPCC_SOURCE_DIR}/esp/services
+         ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/mp
+         ${HPCC_SOURCE_DIR}/esp/clients
+         ${HPCC_SOURCE_DIR}/esp/bindings
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${HPCC_SOURCE_DIR}/common/environment
+         ${HPCC_SOURCE_DIR}/dali/dfu
+         ${HPCC_SOURCE_DIR}/dali/dfutil
+         ${HPCC_SOURCE_DIR}/common/remote
+    )
+
+ADD_DEFINITIONS( -D_USRDLL )
+
+HPCC_ADD_LIBRARY( ws_packageprocess SHARED ${SRCS} )
+install ( TARGETS ws_packageprocess DESTINATION ${DIR_NAME}/lib )
+add_dependencies (ws_packageprocess espscm)
+target_link_libraries ( ws_packageprocess
+         jlib
+         xmllib
+         esphttp
+         securesocket
+         dalibase
+         ws_fs
+    )
+

esp/services/ws_packageprocess/ws_packageprocessPlugin.cpp

@@ -0,0 +1,80 @@
+/*##############################################################################
+
+    Copyright (C) <2010>  <LexisNexis Risk Data Management Inc.>
+
+    All rights reserved. This program is NOT PRESENTLY free software: you can NOT redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+#include "ws_packageprocess_esp.ipp"
+
+//ESP Bindings
+#include "http/platform/httpprot.hpp"
+
+//ESP Service
+#include "ws_packageprocessService.hpp"
+
+#include "espplugin.hpp"
+
+extern "C"
+{
+
+//when we aren't loading dynamically
+// Change the function names when we stick with dynamic loading.
+ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "WsPackageProcess")==0)
+    {
+        CWsPackageProcessEx* service = new CWsPackageProcessEx;
+        service->init(cfg, process, name);
+        return service;
+    }
+    return NULL;
+}
+
+
+ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "WsPackageProcessSoapBinding")==0)
+    {
+        CWsPackageProcessSoapBindingEx* binding = new CWsPackageProcessSoapBindingEx(cfg, name, process);
+        return binding;
+    }
+
+    return NULL;
+}
+
+
+
+ESP_FACTORY IEspProtocol * esp_protocol_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "http_protocol")==0)
+    {
+        return new CHttpProtocol;
+    }
+    else if(strcmp(type, "secure_http_protocol") == 0)
+    {
+        IPropertyTree *sslSettings;
+        sslSettings = cfg->getPropTree(StringBuffer("Software/EspProcess[@name=\"").append(process).append("\"]").append("/EspProtocol[@name=\"").append(name).append("\"]").str());
+        if(sslSettings != NULL)
+        {
+            return new CSecureHttpProtocol(sslSettings);
+        }
+    }
+
+    return NULL;
+}
+
+};

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -0,0 +1,542 @@
+/*##############################################################################
+Copyright (C) 2011 HPCC Systems.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+#include "ws_packageprocessService.hpp"
+#include "daclient.hpp"
+#include "dalienv.hpp"
+#include "dadfs.hpp"
+#include "dfuutil.hpp"
+#include "ws_fs.hpp"
+#include "ws_workunits.hpp"
+
+#define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
+
+void CWsPackageProcessEx::init(IPropertyTree *cfg, const char *process, const char *service)
+{
+}
+
+bool CWsPackageProcessEx::onEcho(IEspContext &context, IEspEchoRequest &req, IEspEchoResponse &resp)
+{
+    StringBuffer respMsg;
+    ISecUser* user = context.queryUser();
+    if(user != NULL)
+    {
+        const char* name = user->getName();
+        if (name && *name)
+            respMsg.appendf("%s: ", name);
+    }
+
+    const char* reqMsg = req.getRequest();
+    if (reqMsg && *reqMsg)
+        respMsg.append(reqMsg);
+    else
+        respMsg.append("??");
+
+    resp.setResponse(respMsg.str());
+    return true;
+}
+
+IPropertyTree *getPkgSetRegistry(const char *setName, bool readonly)
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+
+    //Only lock the branch for the target we're interested in.
+    StringBuffer xpath;
+    xpath.append("/PackageSets/PackageSet[@id=\"").append(setName).append("\"]");
+    Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+    {
+        if (readonly)
+            return NULL;
+        Owned<IPropertyTree> querySet = createPTree();
+        querySet->setProp("@id", setName);
+        globalLock->queryRoot()->addPropTree("PackageSet", querySet.getClear());
+        globalLock->commit();
+
+        conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
+        if (!conn)
+            throwUnexpected();
+    }
+
+    return conn->getRoot();
+}
+
+////////////////////////////////////////////////////////////////////////////////////////
+const unsigned roxieQueryRoxieTimeOut = 60000;
+
+#define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
+
+bool isRoxieProcess(const char *process)
+{
+    if (!process)
+        return false;
+    Owned<IRemoteConnection> conn = querySDS().connect("Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return false;
+    VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
+    return conn->queryRoot()->hasProp(xpath.str());
+}
+
+bool isFileKnownOnCluster(const char *logicalname, const char *lookupDaliIp, const char *process, IUserDescriptor* userdesc)
+{
+    Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true);
+    if (dst)
+    {
+        if (dst->findCluster(process) != NotFound)
+            return true; // file already known for this cluster
+    }
+    return false;
+}
+
+bool addFileInfoToDali(const char *logicalname, const char *lookupDaliIp, const char *process, bool overwrite, IUserDescriptor* userdesc, StringBuffer &host, short port, StringBuffer &msg)
+{
+    bool retval = true;
+    try
+    {
+        if (!overwrite)
+        {
+            if (isFileKnownOnCluster(logicalname, lookupDaliIp, process, userdesc))
+                return true;
+        }
+
+        StringBuffer user;
+        userdesc->getUserName(user);
+
+        StringBuffer password;
+        userdesc->getPassword(password);
+
+        Owned<IClientFileSpray> fs;
+        fs.setown(createFileSprayClient());
+        fs->setUsernameToken(user.str(), password.str(), NULL);
+
+        VStringBuffer url("http://%s:%d/FileSpray", host.str(), port);
+        fs->addServiceUrl(url.str());
+
+        bool isRoxie = isRoxieProcess(process);
+
+        Owned<IClientCopy> req = fs->createCopyRequest();
+        req->setSourceLogicalName(logicalname);
+        req->setDestLogicalName(logicalname);
+        req->setDestGroup(process);
+        req->setSuperCopy(false);
+        if (isRoxie)
+            req->setDestGroupRoxie("Yes");
+
+        req->setSourceDali(lookupDaliIp);
+
+        req->setSrcusername(user);
+        req->setSrcpassword(password);
+        req->setOverwrite(overwrite);
+
+        Owned<IClientCopyResponse> resp = fs->Copy(req);
+    }
+    catch(IException *e)
+    {
+        e->errorMessage(msg);
+        DBGLOG("ERROR = %s", msg.str());
+        e->Release();  // report the error later if needed
+        retval = false;
+    }
+    catch(...)
+    {
+        retval = false;
+    }
+
+    return retval;
+}
+//////////////////////////////////////////////////////////
+
+void addPackageMapInfo(IPropertyTree *pkgSetRegistry, const char *setName, const char *packageSetName, IPropertyTree *packageInfo, bool active, bool overWrite)
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+
+    StringBuffer lcName(packageSetName);
+    lcName.toLowerCase();
+    StringBuffer xpath;
+    xpath.append("PackageMap[@id='").append(lcName).append("']");
+
+    IPropertyTree *pkgRegTree = pkgSetRegistry->queryPropTree(xpath.str());
+
+    IPropertyTree *root = globalLock->queryRoot();
+    IPropertyTree *mapTree = root->queryPropTree(xpath);
+
+    if (!overWrite && (pkgRegTree || mapTree))
+    {
+        throw MakeStringException(0, "Package name %s already exists, either delete it or specify overwrite", lcName.str());
+    }
+
+    if (mapTree)
+        root->removeTree(mapTree);
+
+    if (pkgRegTree)
+        pkgSetRegistry->removeTree(pkgRegTree);
+
+
+    mapTree = root->addPropTree("PackageMap", createPTree());
+    mapTree->addProp("@id", packageSetName);
+
+    IPropertyTree *baseInfo = createPTree();
+    Owned<IPropertyTreeIterator> iter = packageInfo->getElements("Package");
+    ForEach(*iter)
+    {
+        IPropertyTree &item = iter->query();
+        Owned<IPropertyTreeIterator> super_iter = item.getElements("SuperFile");
+        if (super_iter->first())
+        {
+            ForEach(*super_iter)
+            {
+                IPropertyTree &supertree = super_iter->query();
+                StringAttr id(supertree.queryProp("@id"));
+                if (id.length() && id[0] == '~')
+                    supertree.setProp("@id", id+1);
+
+                Owned<IPropertyTreeIterator> sub_iter = supertree.getElements("SubFile");
+                ForEach(*sub_iter)
+                {
+                    IPropertyTree &subtree = sub_iter->query();
+                    StringAttr subid = subtree.queryProp("@value");
+                    if (subid.length())
+                    {
+                        if (subid[0] == '~')
+                            subtree.setProp("@value", subid+1);
+                    }
+                }
+                mapTree->addPropTree("Package", LINK(&item));
+            }
+        }
+        else
+        {
+            baseInfo->addPropTree("Package", LINK(&item));
+        }
+    }
+
+    mergePTree(mapTree, baseInfo);
+    globalLock->commit();
+
+    IPropertyTree *pkgSetTree = pkgSetRegistry->addPropTree("PackageMap", createPTree("PackageMap"));
+    pkgSetTree->setProp("@id", lcName);
+    pkgSetTree->setProp("@querySet", setName);
+    pkgSetTree->setPropBool("@active", active);
+}
+
+void copyPackageSubFiles(IPropertyTree *packageInfo, const char *process, const char *defaultLookupDaliIp, bool overwrite, IUserDescriptor* userdesc, StringBuffer &host, short port)
+{
+    Owned<IPropertyTreeIterator> iter = packageInfo->getElements("Package");
+    ForEach(*iter)
+    {
+        IPropertyTree &item = iter->query();
+        StringBuffer lookupDaliIp;
+        lookupDaliIp.append(item.queryProp("@daliip"));
+        if (lookupDaliIp.length() == 0)
+            lookupDaliIp.append(defaultLookupDaliIp);
+        if (lookupDaliIp.length() == 0)
+        {
+            StringAttr superfile(item.queryProp("@id"));
+            DBGLOG("Could not lookup SubFiles in package %s because no remote dali ip was specified", superfile.get());
+            return;
+        }
+        Owned<IPropertyTreeIterator> super_iter = item.getElements("SuperFile");
+        ForEach(*super_iter)
+        {
+            IPropertyTree &supertree = super_iter->query();
+            Owned<IPropertyTreeIterator> sub_iter = supertree.getElements("SubFile");
+            ForEach(*sub_iter)
+            {
+                IPropertyTree &subtree = sub_iter->query();
+                StringAttr subid = subtree.queryProp("@value");
+                if (subid.length())
+                {
+                    StringBuffer msg;
+                    addFileInfoToDali(subid.get(), lookupDaliIp, process, overwrite, userdesc, host, port, msg);
+                }
+            }
+        }
+    }
+}
+
+void getPackageListInfo(IPropertyTree *mapTree, IEspPackageListMapData *pkgList)
+{
+    pkgList->setId(mapTree->queryProp("@id"));
+
+    Owned<IPropertyTreeIterator> iter = mapTree->getElements("Package");
+    IArrayOf<IConstPackageListData> results;
+    ForEach(*iter)
+    {
+        IPropertyTree &item = iter->query();
+
+        Owned<IEspPackageListData> res = createPackageListData("", "");
+        res->setId(item.queryProp("@id"));
+        if (item.hasProp("@queries"))
+            res->setQueries(item.queryProp("@queries"));
+        results.append(*res.getClear());
+    }
+    pkgList->setPkgListData(results);
+}
+void getAllPackageListInfo(IPropertyTree *mapTree, StringBuffer &info)
+{
+    info.append("<PackageMap id='").append(mapTree->queryProp("@id")).append("'");
+
+    Owned<IPropertyTreeIterator> iter = mapTree->getElements("Package");
+    ForEach(*iter)
+    {
+        IPropertyTree &item = iter->query();
+        info.append("<Package id='").append(item.queryProp("@id")).append("'");
+        if (item.hasProp("@queries"))
+            info.append(" queries='").append(item.queryProp("@queries")).append("'");
+        info.append("></Package>");
+    }
+    info.append("</PackageMap>");
+}
+void listPkgInfo(const char *cluster, IArrayOf<IConstPackageListMapData>* results)
+{
+    StringBuffer info;
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!globalLock)
+        return;
+    IPropertyTree *root = globalLock->queryRoot();
+    if (!cluster || !*cluster)
+    {
+        info.append("<PackageMaps>");
+        Owned<IPropertyTreeIterator> iter = root->getElements("PackageMap");
+        ForEach(*iter)
+        {
+            Owned<IEspPackageListMapData> res = createPackageListMapData("", "");
+            IPropertyTree &item = iter->query();
+            getPackageListInfo(&item, res);
+            results->append(*res.getClear());
+        }
+        info.append("</PackageMaps>");
+    }
+    else
+    {
+        Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(cluster, true);
+        Owned<IPropertyTreeIterator> iter = pkgSetRegistry->getElements("PackageMap");
+        info.append("<PackageMaps>");
+        ForEach(*iter)
+        {
+            IPropertyTree &item = iter->query();
+            const char *id = item.queryProp("@id");
+            if (id)
+            {
+                StringBuffer xpath;
+                xpath.append("PackageMap[@id='").append(id).append("']");
+                IPropertyTree *mapTree = root->queryPropTree(xpath);
+                Owned<IEspPackageListMapData> res = createPackageListMapData("", "");
+                getPackageListInfo(mapTree, res);
+                results->append(*res.getClear());
+            }
+        }
+        info.append("</PackageMaps>");
+    }
+}
+void getPkgInfo(const char *cluster, const char *package, StringBuffer &info)
+{
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!globalLock)
+        return;
+    IPropertyTree *root = globalLock->queryRoot();
+    Owned<IPropertyTree> tree = createPTree("PackageMaps");
+    if (cluster)
+    {
+        Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(cluster, true);
+        Owned<IPropertyTreeIterator> iter = pkgSetRegistry->getElements("PackageMap[@active='1']");
+        ForEach(*iter)
+        {
+            IPropertyTree &item = iter->query();
+            const char *id = item.queryProp("@id");
+            if (id)
+            {
+                StringBuffer xpath;
+                xpath.append("PackageMap[@id='").append(id).append("']");
+                IPropertyTree *mapTree = root->queryPropTree(xpath);
+                if (mapTree)
+                    mergePTree(tree, mapTree);
+            }
+        }
+    }
+    else
+    {
+        StringBuffer xpath;
+        xpath.append("PackageMap[@id='").append(package).append("']");
+        Owned<IPropertyTreeIterator> iter = root->getElements(xpath.str());
+        ForEach(*iter)
+        {
+            IPropertyTree &item = iter->query();
+            mergePTree(tree, &item);
+        }
+    }
+    toXML(tree, info);
+}
+
+bool deletePkgInfo(const char *packageSetName, const char *queryset)
+{
+    Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(queryset, false);
+    Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+
+    StringBuffer lcName(packageSetName);
+    lcName.toLowerCase();
+    StringBuffer xpath;
+    xpath.append("PackageMap[@id='").append(lcName).append("']");
+
+    bool ret = true;
+    IPropertyTree *root = globalLock->queryRoot();
+    IPropertyTree *mapTree = root->queryPropTree(xpath);
+    if (mapTree)
+        ret = root->removeTree(mapTree);
+
+    if (ret)
+    {
+        IPropertyTree *pkgTree = pkgSetRegistry->queryPropTree(xpath.str());
+        if (pkgTree)
+            ret = pkgSetRegistry->removeTree(pkgTree);
+    }
+    return ret;
+}
+
+void activatePackageMapInfo(const char *packageSetName, const char *packageMap, bool activate)
+{
+    if (!packageSetName || !*packageSetName)
+        return;
+
+    Owned<IRemoteConnection> globalLock = querySDS().connect("PackageSets", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!globalLock)
+        return;
+
+    StringBuffer lcName(packageSetName);
+    lcName.toLowerCase();
+    VStringBuffer xpath("PackageSet[@id=\"%s\"]", lcName.str());
+
+    IPropertyTree *root = globalLock->queryRoot();
+    if (!root)
+        return;
+
+    IPropertyTree *pkgSetTree = root->queryPropTree(xpath);
+    if (pkgSetTree)
+    {
+        if (packageMap && *packageMap)
+        {
+            StringBuffer lcMapName(packageMap);
+            lcMapName.toLowerCase();
+            VStringBuffer xpath_map("PackageMap[@id=\"%s\"]", lcMapName.str());
+            IPropertyTree *mapTree = pkgSetTree->queryPropTree(xpath_map);
+            mapTree->setPropBool("@active", activate);
+        }
+        else
+        {
+            Owned<IPropertyTreeIterator> iter = pkgSetTree->getElements("PackageMap");
+            ForEach(*iter)
+            {
+                IPropertyTree &item = iter->query();
+                item.setPropBool("@active", activate);
+            }
+        }
+    }
+}
+
+bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageRequest &req, IEspAddPackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringBuffer info(req.getInfo());
+    bool activate = req.getActivate();
+    bool overWrite = req.getOverWrite();
+    StringAttr querySet(req.getQuerySet());
+    StringAttr pkgName(req.getPackageName());
+
+    Owned<IPropertyTree> packageTree = createPTreeFromXMLString(info.str());
+    Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(querySet.get(), false);
+    addPackageMapInfo(pkgSetRegistry, querySet.get(), pkgName.get(), LINK(packageTree), activate, overWrite);
+
+    StringBuffer msg;
+    msg.append("Successfully loaded ").append(pkgName.get());
+    resp.updateStatus().setDescription(msg.str());
+    return true;
+}
+
+bool CWsPackageProcessEx::onDeletePackage(IEspContext &context, IEspDeletePackageRequest &req, IEspDeletePackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringAttr pkgName(req.getPackageName());
+    bool ret = deletePkgInfo(pkgName.get(), req.getQuerySet());
+    StringBuffer msg;
+    (ret) ? msg.append("Successfully ") : msg.append("Unsuccessfully ");
+    msg.append("deleted").append(pkgName.get());
+
+    resp.updateStatus().setDescription(msg.str());
+    return true;
+}
+
+bool CWsPackageProcessEx::onActivatePackage(IEspContext &context, IEspActivatePackageRequest &req, IEspActivatePackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringBuffer pkgName(req.getPackageName());
+    StringBuffer pkgMapName(req.getPackageMapName());
+
+    activatePackageMapInfo(pkgName.str(), pkgMapName.str(), true);
+    return true;
+}
+
+bool CWsPackageProcessEx::onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringBuffer pkgName(req.getPackageName());
+    StringBuffer pkgMapName(req.getPackageMapName());
+
+    activatePackageMapInfo(pkgName.str(), pkgMapName.str(), false);
+    return true;
+}
+
+bool CWsPackageProcessEx::onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    IArrayOf<IConstPackageListMapData> results;
+    listPkgInfo(req.getCluster(), &results);
+    resp.setPkgListMapData(results);
+    return true;
+}
+
+bool CWsPackageProcessEx::onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringAttr cluster(req.getCluster());
+    StringAttr pkgName(req.getPackageName());
+    StringBuffer info;
+    getPkgInfo(cluster.length() ? cluster.get() : NULL, pkgName.length() ? pkgName.get() : NULL, info);
+    resp.setInfo(info);
+    return true;
+}
+
+bool CWsPackageProcessEx::onCopyFiles(IEspContext &context, IEspCopyFilesRequest &req, IEspCopyFilesResponse &resp)
+{
+    resp.updateStatus().setCode(0);
+    StringBuffer info(req.getInfo());
+    StringAttr process(req.getProcess());
+    StringAttr pkgName(req.getPackageName());
+    StringAttr lookupDaliIp(req.getDaliIp());
+
+    if (process.length() == 0)
+        throw MakeStringException(0, "CWsPackageProcessEx::onCopyFiles process parameter not set.");
+
+    Owned<IUserDescriptor> userdesc;
+    const char *user = context.queryUserId();
+    const char *password = context.queryPassword();
+    if (user && *user && *password && *password)
+    {
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(user, password);
+    }
+
+    StringBuffer host;
+    short port;
+    context.getServAddress(host, port);
+
+    Owned<IPropertyTree> packageTree = createPTreeFromXMLString(info.str());
+    copyPackageSubFiles(LINK(packageTree), process, lookupDaliIp, req.getOverWrite(), userdesc, host, port);
+
+    StringBuffer msg;
+    msg.append("Successfully loaded ").append(pkgName.get());
+    resp.updateStatus().setDescription(msg.str());
+    return true;
+}

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    Copyright (C) <2010>  <LexisNexis Risk Data Management Inc.>
+
+    All rights reserved. This program is NOT PRESENTLY free software: you can NOT redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _ESPWIZ_ws_packageprocess_HPP__
+#define _ESPWIZ_ws_packageprocess_HPP__
+
+#include "ws_packageprocess_esp.ipp"
+
+
+
+class CWsPackageProcessSoapBindingEx : public CWsPackageProcessSoapBinding
+{
+public:
+    CWsPackageProcessSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel=hsl_none) : CWsPackageProcessSoapBinding(cfg, name, process, llevel)
+    {
+    }
+
+    virtual void getNavigationData(IEspContext &context, IPropertyTree & data)
+    {
+        //Add navigation link here
+    }
+};
+
+
+class CWsPackageProcessEx : public CWsPackageProcess
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    virtual ~CWsPackageProcessEx(){};
+
+    virtual void init(IPropertyTree *cfg, const char *process, const char *service);
+
+    virtual bool onEcho(IEspContext &context, IEspEchoRequest &req, IEspEchoResponse &resp);
+    virtual bool onAddPackage(IEspContext &context, IEspAddPackageRequest &req, IEspAddPackageResponse &resp);
+    virtual bool onDeletePackage(IEspContext &context, IEspDeletePackageRequest &req, IEspDeletePackageResponse &resp);
+    virtual bool onActivatePackage(IEspContext &context, IEspActivatePackageRequest &req, IEspActivatePackageResponse &resp);
+    virtual bool onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp);
+    virtual bool onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp);
+    virtual bool onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp);
+    virtual bool onCopyFiles(IEspContext &context, IEspCopyFilesRequest &req, IEspCopyFilesResponse &resp);
+};
+
+#endif //_ESPWIZ_ws_packageprocess_HPP__
+