Merge pull request #703 from afishbeck/ecl_command_line

New ECL Command Line tool

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -169,6 +169,7 @@ add_subdirectory (ecl/agentexec)
 add_subdirectory (ecl/eclagent)
 add_subdirectory (ecl/eclcc)
 add_subdirectory (ecl/eclccserver)
+add_subdirectory (ecl/eclcmd)
 add_subdirectory (ecl/eclscheduler)
 add_subdirectory (ecl/eclplus)
 add_subdirectory (ecl/hql)

ecl/eclcmd/CMakeLists.txt

@@ -0,0 +1,73 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: ecl
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for ecl
+#####################################################
+
+
+project( ecl )
+
+include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
+
+set (    SRCS
+         ${ESPSCM_GENERATED_DIR}/ws_workunits_esp.cpp
+         eclcmd.hpp
+         ecl.cpp
+         eclcmd_common.hpp
+         eclcmd_shell.cpp
+         eclcmd_common.cpp
+         eclcmd_core.hpp
+         eclcmd_core.cpp
+    )
+
+include_directories (
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+         ./../../system/include
+         ./../../system/jlib
+         ./../../esp/clients
+         ./../../system/security/shared
+         ./../../system/mp
+         ./../../common/dllserver
+         ./../../common/fileview2
+         ./../../common/workunit
+         ./../../esp/clients
+         ./../../common/environment
+         ./../../esp/bindings/SOAP/xpp
+         ./../../system/include
+         ./../../esp/bindings
+         ./../../ecl/hql
+         ./../../dali/base
+         ./../../esp/platform
+         ./../../system/jlib
+         ./../../system/xmllib
+    )
+
+ADD_DEFINITIONS( -D_CONSOLE )
+
+add_executable ( ecl ${SRCS} )
+add_dependencies ( ecl espscm ws_workunits )
+install ( TARGETS ecl DESTINATION ${OSSDIR}/bin )
+target_link_libraries ( ecl
+        jlib
+        esphttp
+    )

ecl/eclcmd/ecl.cpp

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#include <stdio.h>
+#include "jlog.hpp"
+#include "jfile.hpp"
+#include "jargv.hpp"
+
+#include "build-config.h"
+
+#include "ws_workunits.hpp"
+
+#include "eclcmd.hpp"
+#include "eclcmd_common.hpp"
+#include "eclcmd_core.hpp"
+
+#define INIFILE "ecl.ini"
+#define SYSTEMCONFDIR CONFIG_DIR
+#define DEFAULTINIFILE "ecl.ini"
+#define SYSTEMCONFFILE ENV_CONF_FILE
+
+
+//=========================================================================================
+
+static int doMain(int argc, const char *argv[])
+{
+    EclCMDShell processor(argc, argv, createCoreEclCommand, BUILD_TAG);
+    return processor.run();
+}
+
+int main(int argc, const char *argv[])
+{
+    InitModuleObjects();
+    queryStderrLogMsgHandler()->setMessageFields(0);
+    unsigned exitCode = doMain(argc, argv);
+    releaseAtoms();
+    return exitCode;
+}

ecl/eclcmd/eclcmd.hpp

@@ -0,0 +1,53 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef ECLCMD_HPP
+#define ECLCMD_HPP
+
+#include "eclcmd_common.hpp"
+
+class EclCMDShell
+{
+public:
+    EclCMDShell(int argc, const char *argv[], EclCommandFactory _factory, const char *_version)
+        : args(argc, argv), factory(_factory), version(_version), optHelp(false)
+    {
+        splitFilename(argv[0], NULL, NULL, &name, NULL);
+    }
+
+    bool parseCommandLineOptions(ArgvIterator &iter);
+    void finalizeOptions(IProperties *globals);
+    int processCMD(ArgvIterator &iter);
+    int run();
+
+    virtual void usage();
+
+protected:
+    ArgvIterator args;
+    Owned<IProperties> globals;
+    EclCommandFactory factory;
+    StringBuffer name;
+    StringAttr cmd;
+
+    StringAttr version;
+
+    bool optHelp;
+    StringAttr optIniFilename;
+};
+
+#endif

ecl/eclcmd/eclcmd_common.cpp

@@ -0,0 +1,208 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include <stdio.h>
+#include "jlog.hpp"
+#include "jfile.hpp"
+#include "jargv.hpp"
+#include "build-config.h"
+
+#include "ws_workunits.hpp"
+#include "eclcmd_common.hpp"
+
+bool extractOption(StringBuffer & option, IProperties * globals, const char * envName, const char * propertyName, const char * defaultPrefix, const char * defaultSuffix)
+{
+    if (option.length())        // check if already specified via a command line option
+        return true;
+    if (globals->getProp(propertyName, option))
+        return true;
+    const char * env = getenv(envName);
+    if (env)
+    {
+        option.append(env);
+        return true;
+    }
+    if (defaultPrefix)
+        option.append(defaultPrefix);
+    if (defaultSuffix)
+        option.append(defaultSuffix);
+    return false;
+}
+
+bool extractOption(StringAttr & option, IProperties * globals, const char * envName, const char * propertyName, const char * defaultPrefix, const char * defaultSuffix)
+{
+    if (option)
+        return true;
+    StringBuffer temp;
+    bool ret = extractOption(temp, globals, envName, propertyName, defaultPrefix, defaultSuffix);
+    option.set(temp.str());
+    return ret;
+}
+
+bool extractOption(bool & option, IProperties * globals, const char * envName, const char * propertyName, bool defval)
+{
+    StringBuffer temp;
+    bool ret = extractOption(temp, globals, envName, propertyName, defval ? "1" : "0", NULL);
+    option=(streq(temp.str(),"1")||strieq(temp.str(),"true"));
+    return ret;
+}
+
+static bool looksLikeOnlyAWuid(const char * wuid)
+{
+    if (!wuid)
+        return false;
+    if (wuid[0] != 'W')
+        return false;
+    if (!isdigit(wuid[1]) || !isdigit(wuid[2]) || !isdigit(wuid[3]) || !isdigit(wuid[4]))
+        return false;
+    if (!isdigit(wuid[5]) || !isdigit(wuid[6]) || !isdigit(wuid[7]) || !isdigit(wuid[8]))
+        return false;
+    if (wuid[9]!='-')
+        return false;
+    for (int i=10; wuid[i]; i++)
+        if (wuid[i]!='-' && !isdigit(wuid[i]))
+            return false;
+    return true;
+}
+
+//=========================================================================================
+
+static const char * skipWhitespace(const char * s)
+{
+    while (*s && strchr(" \t\r\n", *s))
+        s++;
+    return s;
+}
+
+static const char * skipXmlDeclaration(const char *s)
+{
+    while (*s)
+    {
+        s = skipWhitespace(s);
+        if (*s!='<' || (s[1]!='?'))
+            return s;
+        while (*s && *s++!='>');
+    }
+    return s;
+}
+
+#define PE_OFFSET_LOCATION_IN_DOS_SECTION 0x3C
+
+class determineEclObjParameterType
+{
+public:
+    determineEclObjParameterType(const char *path, unsigned _accept=0x00) : accept(_accept)
+    {
+        file.setown(createIFile(path));
+        io.setown(file->open(IFOread));
+        read(io, 0, (size32_t) 1024, signature);
+        signature.append((char)0);
+    }
+    bool isArchive()
+    {
+        const char *s = skipXmlDeclaration(signature.toByteArray());
+        return (strnicmp("<Archive", s, 8)==0);
+    }
+    bool isElfFile()
+    {
+        const char *s = signature.toByteArray();
+        return (s[0]==0x7F && s[1]=='E' && s[2]=='L' && s[3]=='F');
+    }
+    bool isPEFile()
+    {
+        if (file->size()<=PE_OFFSET_LOCATION_IN_DOS_SECTION)
+            return false;
+        const char *buffer = signature.toByteArray();
+        if (memcmp("MZ", buffer, 2)!=0)
+            return false;
+        unsigned long PESectStart = *((unsigned long *)(buffer + PE_OFFSET_LOCATION_IN_DOS_SECTION));
+        if (file->size()<=PESectStart)
+            return false;
+        unsigned len = signature.length()-1;
+        if (len<PESectStart+2)
+        {
+            signature.setLength(len);
+            read(io, len, (size32_t) PESectStart+2-len, signature);
+            signature.append((char)0);
+        }
+        if (signature.length()<PESectStart+3)
+            return false;
+        buffer = signature.toByteArray()+PESectStart;
+        if (memcmp("PE", buffer, 2)!=0)
+            return false;
+        return true;
+    }
+    eclObjParameterType checkType()
+    {
+        if ((!accept || accept & eclObjSharedObject) && isElfFile() || isPEFile())
+            return eclObjSharedObject;
+        if ((!accept || accept & eclObjArchive) && isArchive())
+            return eclObjArchive;
+        else if (!accept || accept & eclObjSource)
+            return eclObjSource;
+        else if (accept & eclObjQueryId)
+            return eclObjQueryId;
+        return eclObjTypeUnknown;
+    }
+public:
+    Owned<IFile> file;
+    Owned<IFileIO> io;
+    MemoryBuffer signature;
+    unsigned accept;
+};
+
+
+EclObjectParameter::EclObjectParameter(unsigned _accept) : type(eclObjTypeUnknown), accept(_accept)
+{
+}
+
+eclObjParameterType EclObjectParameter::set(const char *_value)
+{
+    value.set(_value);
+    if (checkFileExists(_value))
+    {
+        determineEclObjParameterType fileType(_value, accept);
+        type = fileType.checkType();
+    }
+    else if (looksLikeOnlyAWuid(_value))
+        type = eclObjWuid;
+    return type;
+}
+
+const char *EclObjectParameter::queryTypeName()
+{
+    switch (type)
+    {
+    case eclObjWuid:
+        return "WUID";
+    case eclObjSource:
+        return "ECL File";
+    case eclObjArchive:
+        return "ECL Archive";
+    case eclObjSharedObject:
+        return "ECL Shared Object";
+    case eclObjTypeUnknown:
+    default:
+        return "Unknown";
+    }
+}
+
+StringBuffer &EclObjectParameter::getDescription(StringBuffer &s)
+{
+    return s.append(queryTypeName()).append(' ').append(value.get());
+}

ecl/eclcmd/eclcmd_common.hpp

@@ -0,0 +1,112 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef ECLCMD_COMMON_HPP
+#define ECLCMD_COMMON_HPP
+
+//=========================================================================================
+
+interface IEclCommand : extends IInterface
+{
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)=0;
+    virtual void finalizeOptions(IProperties *globals)=0;
+    virtual int processCMD()=0;
+    virtual void usage()=0;
+};
+
+typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
+
+#define ECLOPT_SERVER "--server"
+#define ECLOPT_SERVER_INI "eclWatchIP"
+#define ECLOPT_SERVER_ENV "ECL_WATCH_IP"
+
+#define ECLOPT_PORT "--port"
+#define ECLOPT_PORT_INI "eclWatchPort"
+#define ECLOPT_PORT_ENV "ECL_WATCH_PORT"
+
+#define ECLOPT_ACTIVATE "--activate"
+#define ECLOPT_ACTIVATE_INI "activateDefault"
+#define ECLOPT_ACTIVATE_ENV NULL
+
+#define ECLOPT_WUID "--wuid"
+#define ECLOPT_CLUSTER "--cluster"
+#define ECLOPT_NAME "--name"
+#define ECLOPT_ACTIVATE "--activate"
+#define ECLOPT_VERSION "--version"
+
+bool extractOption(StringBuffer & option, IProperties * globals, const char * envName, const char * propertyName, const char * defaultPrefix, const char * defaultSuffix);
+bool extractOption(StringAttr & option, IProperties * globals, const char * envName, const char * propertyName, const char * defaultPrefix, const char * defaultSuffix);
+bool extractOption(bool & option, IProperties * globals, const char * envName, const char * propertyName, bool defval);
+
+enum eclObjParameterType
+{
+    eclObjTypeUnknown = 0x00,
+    eclObjSource = 0x01,
+    eclObjArchive = 0x02,
+    eclObjSharedObject = 0x04,
+    eclObjWuid = 0x08,
+    eclObjQueryId = 0x10
+};
+
+class EclObjectParameter
+{
+public:
+    EclObjectParameter(unsigned _accept=0x00);
+    eclObjParameterType set(const char *_value);
+    const char *queryTypeName();
+    StringBuffer &getDescription(StringBuffer &s);
+
+public:
+    eclObjParameterType type;
+    StringAttr value;
+    unsigned accept;
+};
+
+class EclCmdCommon : public CInterface, implements IEclCommand
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    EclCmdCommon()
+    {
+    }
+    virtual bool matchCommandLineOption(ArgvIterator &iter)
+    {
+        if (iter.matchOption(optServer, ECLOPT_SERVER))
+            return true;
+        else if (iter.matchOption(optPort, ECLOPT_PORT))
+            return true;
+        return false;
+    }
+    virtual void finalizeOptions(IProperties *globals)
+    {
+        extractOption(optServer, globals, ECLOPT_SERVER_ENV, ECLOPT_SERVER_INI, ".", NULL);
+        extractOption(optPort, globals, ECLOPT_PORT_ENV, ECLOPT_PORT_INI, "8010", NULL);
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,
+            "      --server=<ip>        ip of server running ecl services (eclwatch)\n"
+            "      --port=<port>        ecl services port\n"
+        );
+    }
+public:
+    StringAttr optServer;
+    StringAttr optPort;
+};
+
+#endif

ecl/eclcmd/eclcmd_core.cpp

@@ -0,0 +1,262 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include <stdio.h>
+#include "jlog.hpp"
+#include "jfile.hpp"
+#include "jargv.hpp"
+#include "build-config.h"
+
+#include "ws_workunits.hpp"
+#include "eclcmd_common.hpp"
+#include "eclcmd_core.hpp"
+
+void outputMultiExceptions(const IMultiException &me)
+{
+    fprintf(stderr, "\nException(s):\n");
+    aindex_t count = me.ordinality();
+    for (aindex_t i=0; i<count; i++)
+    {
+        IException& e = me.item(i);
+        StringBuffer msg;
+        fprintf(stderr, "%d: %s\n", e.errorCode(), e.errorMessage(msg).str());
+    }
+    fprintf(stderr, "\n");
+}
+
+class EclCmdDeploy : public EclCmdCommon
+{
+public:
+    EclCmdDeploy() : optObj(eclObjSource | eclObjArchive | eclObjSharedObject)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        bool boolValue;
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optObj.set(arg);
+                break;
+            }
+            else if (EclCmdCommon::matchCommandLineOption(iter))
+                continue;
+            else if (iter.matchOption(optCluster, ECLOPT_CLUSTER))
+                continue;
+            else if (iter.matchOption(optName, ECLOPT_NAME))
+                continue;
+            else if (iter.matchFlag(boolValue, ECLOPT_VERSION))
+            {
+                fprintf(stdout, "\necl delploy version %s\n\n", BUILD_TAG);
+                return false;
+            }
+        }
+        return true;
+    }
+    virtual void finalizeOptions(IProperties *globals)
+    {
+        EclCmdCommon::finalizeOptions(globals);
+    }
+    virtual int processCMD()
+    {
+        StringBuffer s;
+        if (optObj.type==eclObjTypeUnknown)
+            fprintf(stderr, "\nCan't determine content type of argument %s\n", optObj.value.sget());
+        else if (optObj.type==eclObjSource)
+            fprintf(stderr, "\nDirect deployment of ECL source is not yet supported\n");
+        else if (optObj.type==eclObjWuid || optObj.type==eclObjQueryId)
+            fprintf(stderr, "\nRemote objects already deployed %s\n", optObj.getDescription(s).str());
+        else
+        {
+            fprintf(stdout, "\nDeploying %s\n", optObj.getDescription(s).str());
+            Owned<IClientWsWorkunits> client = createWsWorkunitsClient();
+            VStringBuffer url("http://%s:%s/WsWorkunits", optServer.sget(), optPort.sget());
+            client->addServiceUrl(url.str());
+            Owned<IClientWUDeployWorkunitRequest> req = client->createWUDeployWorkunitRequest();
+            switch (optObj.type)
+            {
+                case eclObjArchive:
+                {
+                    req->setObjType("archive");
+                    break;
+                }
+                case eclObjSharedObject:
+                {
+                    req->setObjType("shared_object");
+                    break;
+                }
+            }
+            MemoryBuffer mb;
+            Owned<IFile> file = createIFile(optObj.value.sget());
+            Owned<IFileIO> io = file->open(IFOread);
+            read(io, 0, (size32_t)file->size(), mb);
+            if (optName.length())
+                req->setName(optName.get());
+            if (optCluster.length())
+                req->setCluster(optCluster.get());
+            req->setFileName(optObj.value.sget());
+            req->setObject(mb);
+            Owned<IClientWUDeployWorkunitResponse> resp = client->WUDeployWorkunit(req);
+            if (resp->getExceptions().ordinality())
+                outputMultiExceptions(resp->getExceptions());
+            const char *wuid = resp->getWorkunit().getWuid();
+            if (wuid && *wuid)
+            {
+                fprintf(stdout, "\nDeployed\nwuid: %s\nstate: %s\n", wuid, resp->getWorkunit().getState());
+            }
+        }
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl deploy [options][<archive>|<eclfile>|<so>|<dll>]\n\n"
+            "   Options:\n"
+            "      --cluster=<cluster>  cluster to associate workunit with\n"
+            "      --name=<name>        workunit job name\n\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+    EclObjectParameter optObj;
+    StringAttr optCluster;
+    StringAttr optName;
+};
+
+class EclCmdPublish : public EclCmdCommon
+{
+public:
+    EclCmdPublish() : optActivate(false), activateSet(false)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        bool boolValue;
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                optWuid.set(arg);
+                break;
+            }
+            else if (EclCmdCommon::matchCommandLineOption(iter))
+                continue;
+            else if (iter.matchOption(optWuid, ECLOPT_WUID))
+                continue;
+            else if (iter.matchOption(optName, ECLOPT_NAME))
+                continue;
+            else if (iter.matchOption(optCluster, ECLOPT_CLUSTER))
+                continue;
+            else if (iter.matchFlag(optActivate, ECLOPT_ACTIVATE))
+            {
+                activateSet=true;
+                continue;
+            }
+            else if (iter.matchFlag(boolValue, ECLOPT_VERSION))
+            {
+                fprintf(stdout, "\necl publish version %s\n\n", BUILD_TAG);
+                return false;
+            }
+        }
+        return true;
+    }
+    virtual void finalizeOptions(IProperties *globals)
+    {
+        EclCmdCommon::finalizeOptions(globals);
+        if (!activateSet)
+            extractOption(optActivate, globals, ECLOPT_ACTIVATE_ENV, ECLOPT_ACTIVATE_INI, false);
+    }
+    virtual int processCMD()
+    {
+        StringBuffer s;
+        if (!optWuid.length())
+        {
+            fprintf(stderr, "\nError: wuid parameter required\n");
+            return 1;
+        }
+
+        fprintf(stdout, "\nPublishing %s\n", optWuid.get());
+        Owned<IClientWsWorkunits> client = createWsWorkunitsClient();
+        VStringBuffer url("http://%s:%s/WsWorkunits", optServer.sget(), optPort.sget());
+        client->addServiceUrl(url.str());
+
+        Owned<IClientWUPublishWorkunitRequest> req = client->createWUPublishWorkunitRequest();
+        req->setWuid(optWuid.get());
+        req->setActivate(optActivate);
+        if (optName.length())
+            req->setJobName(optName.get());
+        if (optCluster.length())
+            req->setCluster(optCluster.get());
+
+        Owned<IClientWUPublishWorkunitResponse> resp = client->WUPublishWorkunit(req);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+        const char *id = resp->getQueryId();
+        if (id && *id)
+            fprintf(stdout, "\nPublished\nQuerySet: %s\nQueryName: %s\nQueryId: %s\n", resp->getQuerySet(), resp->getQueryName(), resp->getQueryId());
+
+        return 0;
+    }
+    virtual void usage()
+    {
+        fprintf(stdout,"\nUsage:\n\n"
+            "ecl publish [options][<wuid>]\n\n"
+            "   Options:\n"
+            "      --cluster=<cluster>  cluster to publish workunit to\n"
+            "                           (defaults to cluster defined inside workunit)\n"
+            "      --name=<name>        query name to use for published workunit\n"
+            "      --wuid=<wuid>        workunit id to publish\n"
+            "      --activate           activates query when published\n\n"
+        );
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optCluster;
+    StringAttr optWuid;
+    StringAttr optName;
+    bool optActivate;
+    bool activateSet;
+};
+
+//=========================================================================================
+
+IEclCommand *createCoreEclCommand(const char *cmdname)
+{
+    if (!cmdname || !*cmdname)
+        return NULL;
+    if (strieq(cmdname, "deploy"))
+        return new EclCmdDeploy();
+    if (strieq(cmdname, "publish"))
+        return new EclCmdPublish();
+    return NULL;
+}

ecl/eclcmd/eclcmd_core.hpp

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef ECLCMD_CORE_HPP
+#define ECLCMD_CORE_HPP
+
+#include "eclcmd_common.hpp"
+
+IEclCommand *createCoreEclCommand(const char *cmdname);
+
+#endif

ecl/eclcmd/eclcmd_shell.cpp

@@ -0,0 +1,153 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#include <stdio.h>
+#include "jlog.hpp"
+#include "jfile.hpp"
+#include "jargv.hpp"
+#include "jprop.hpp"
+
+#include "build-config.h"
+
+#include "eclcmd.hpp"
+#include "eclcmd_core.hpp"
+
+#define INIFILE "ecl.ini"
+#define SYSTEMCONFDIR CONFIG_DIR
+#define DEFAULTINIFILE "ecl.ini"
+#define SYSTEMCONFFILE ENV_CONF_FILE
+
+//=========================================================================================
+
+int EclCMDShell::processCMD(ArgvIterator &iter)
+{
+    Owned<IEclCommand> c = factory(cmd.get());
+    if (!c)
+    {
+        if (cmd.length())
+            fprintf(stdout, "ecl '%s' command not found\n", cmd.sget());
+        usage();
+        return 1;
+    }
+    if (optHelp)
+    {
+        c->usage();
+        return 1;
+    }
+    if (!c->parseCommandLineOptions(iter))
+        return 1;
+
+    c->finalizeOptions(globals);
+
+    return c->processCMD();
+}
+
+void EclCMDShell::finalizeOptions(IProperties *globals)
+{
+}
+
+int EclCMDShell::run()
+{
+    if (!parseCommandLineOptions(args))
+        return 1;
+
+    if (!optIniFilename)
+    {
+        if (checkFileExists(INIFILE))
+            optIniFilename.set(INIFILE);
+        else
+        {
+            StringBuffer fn(SYSTEMCONFDIR);
+            fn.append(PATHSEPSTR).append(DEFAULTINIFILE);
+            if (checkFileExists(fn))
+                optIniFilename.set(fn);
+        }
+    }
+
+    globals.setown(createProperties(optIniFilename, true));
+    finalizeOptions(globals);
+
+    try
+    {
+        return processCMD(args);
+    }
+    catch (IException *E)
+    {
+        StringBuffer m("Error: ");
+        fputs(E->errorMessage(m).newline().str(), stderr);
+        E->Release();
+        return 2;
+    }
+#ifndef _DEBUG
+    catch (...)
+    {
+        ERRLOG("Unexpected exception\n");
+        return 4;
+    }
+#endif
+    return 0;
+}
+
+//=========================================================================================
+
+bool EclCMDShell::parseCommandLineOptions(ArgvIterator &iter)
+{
+    const char *dash = strchr(name.str(), '-');
+    if (dash)
+        cmd.set(dash+1);
+    else
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        bool boolValue;
+        for (; !iter.done(); iter.next())
+        {
+            const char * arg = iter.query();
+            if (iter.matchFlag(optHelp, "help"))
+                continue;
+            else if (*arg!='-')
+            {
+                cmd.set(arg);
+                iter.next();
+                break;
+            }
+            else if (iter.matchFlag(boolValue, "--version"))
+            {
+                fprintf(stdout, "\necl command line version %s\n\n", BUILD_TAG);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+//=========================================================================================
+
+void EclCMDShell::usage()
+{
+    fprintf(stdout,"\nUsage:\n"
+        "    ecl [--version] <command> [<args>]\n\n"
+           "Commonly used commands:\n"
+           "   deploy    create an HPCC workunit from a local archive or shared object\n"
+           "   publish   add an HPCC workunit to a query set\n"
+           "\nRun 'ecl help <command>' for more information on a specific command\n\n"
+    );
+}

ecl/eclcmd/sourcedoc.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>ecl/ecl</title>
+
+    <para>
+        The ecl/ecl directory contains the sources for the ecl command line tool.
+    </para>
+</section>

esp/eclwatch/ws_XSLT/wuid.xslt

@@ -568,17 +568,17 @@
                 alert('The workunit must have a jobname to be published. \r\nEnter a jobname, then publish.');
                 return;
               }
-              var cObj = YAHOO.util.Connect.asyncRequest('POST', '/WsWorkunits/WUDeployWorkunit?Wuid=' + wuid + '&JobName=' + Jobname + '&Activate=1', publishCallback);
+              var cObj = YAHOO.util.Connect.asyncRequest('POST', '/WsWorkunits/WUPublishWorkunit?Wuid=' + wuid + '&JobName=' + Jobname + '&Activate=1', publishCallback);
             }
 
         /*
         <?xml version="1.0" encoding="utf-8"?>
 <soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope" xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding" xmlns="http://webservices.seisint.com/WsWorkunits">
  <soap:Body>
-  <WUDeployWorkunitRequest>
+  <WUPublishWorkunitRequest>
    <Wuid>W20110125-150953</Wuid>
    <Activate>1</Activate>
-  </WUDeployWorkunitRequest>
+  </WUPublishWorkunitRequest>
  </soap:Body>
 </soap:Envelope>
         */

esp/scm/ws_workunits.ecm

@@ -284,6 +284,21 @@ ESPresponse [exceptions_inline] WUCreateResponse
     ESPstruct ECLWorkunit Workunit;
 };
 
+ESPrequest WUDeployWorkunitRequest
+{
+    string Cluster;
+    string Name;
+    int Wait(-1);
+    string ObjType;
+    string FileName;
+    binary Object;
+};
+
+ESPresponse [exceptions_inline] WUDeployWorkunitResponse
+{
+    ESPstruct ECLWorkunit Workunit;
+};
+
 ESPrequest [nil_remove] WUQueryRequest
 {
     string Wuid;
@@ -993,9 +1008,10 @@ ESPresponse [exceptions_inline] WUCopyLogicalFilesResponse
 };
 
 
-ESPrequest WUDeployWorkunitRequest
+ESPrequest WUPublishWorkunitRequest
 {
     string Wuid;
+    string Cluster;
     string JobName;
     int Activate;
     bool NotifyCluster(false);
@@ -1003,10 +1019,13 @@ ESPrequest WUDeployWorkunitRequest
     bool CopyLocal(0);
 };
 
-ESPresponse [exceptions_inline] WUDeployWorkunitResponse
+ESPresponse [exceptions_inline] WUPublishWorkunitResponse
 {
     string Wuid;
     string Result;
+    string QuerySet;
+    string QueryName;
+    string QueryId;
     ESParray<ESPStruct WUCopyLogicalClusterFileSections, Cluster> ClusterFiles;
 };
 
@@ -1130,6 +1149,7 @@ ESPservice [
     ESPmethod WUSubmit(WUSubmitRequest, WUSubmitResponse);
     ESPmethod WUSchedule(WUScheduleRequest, WUScheduleResponse);
     ESPmethod WUPushEvent(WUPushEventRequest, WUPushEventResponse);
+    ESPmethod WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
 
     ESPmethod WUAbort(WUAbortRequest, WUAbortResponse);
     ESPmethod WUProtect(WUProtectRequest, WUProtectResponse);
@@ -1154,7 +1174,7 @@ ESPservice [
 
     ESPmethod WUCDebug(WUDebugRequest, WUDebugResponse);
     
-    ESPmethod [resp_xsl_default("/esp/xslt/WUDeployWorkunit.xslt")] WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
+    ESPmethod [resp_xsl_default("/esp/xslt/WUPublishWorkunit.xslt")] WUPublishWorkunit(WUPublishWorkunitRequest, WUPublishWorkunitResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUQuerysets.xslt")] WUQuerysets(WUQuerysetsRequest, WUQuerysetsResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUQuerysetQueries.xslt")] WUQuerysetDetails(WUQuerySetDetailsRequest, WUQuerySetDetailsResponse);
     ESPmethod WUQuerysetActionQueries(WUQuerySetActionQueriesRequest, WUQuerySetActionQueriesResponse);

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -199,7 +199,7 @@ bool CWsWorkunitsEx::onWUCopyLogicalFiles(IEspContext &context, IEspWUCopyLogica
     return true;
 }
 
-bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWorkunitRequest & req, IEspWUPublishWorkunitResponse & resp)
 {
     if (isEmpty(req.getWuid()))
         throw MakeStringException(ECLWATCH_NO_WUID_SPECIFIED,"No Workunit ID has been specified.");
@@ -218,7 +218,10 @@ bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkun
         cw->getJobName(queryName).str();
 
     SCMStringBuffer cluster;
-    cw->getClusterName(cluster);
+    if (notEmpty(req.getCluster()))
+        cluster.set(req.getCluster());
+    else
+        cw->getClusterName(cluster);
 
     Owned <IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster.str());
 
@@ -234,6 +237,11 @@ bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkun
     wu->commit();
     wu.clear();
 
+    if (queryId.length())
+        resp.setQueryId(queryId.str());
+    resp.setQueryName(queryName.str());
+    resp.setQuerySet(queryset.str());
+
     if (req.getCopyLocal() || req.getShowFiles())
     {
         IArrayOf<IConstWUCopyLogicalClusterFileSections> clusterfiles;

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -38,11 +38,14 @@
 #include "roxiemanager.hpp"
 #include "dadfs.hpp"
 #include "dfuwu.hpp"
+#include "thorplugin.hpp"
 
 #ifdef _USE_ZLIB
 #include "zcrypt.hpp"
 #endif
 
+#define ESP_WORKUNIT_DIR "workunits/"
+
 void submitWsWorkunit(IEspContext& context, IConstWorkUnit* cw, const char* cluster, const char* snapshot, int maxruntime, bool compile, bool resetWorkflow)
 {
     ensureWsWorkunitAccess(context, *cw, SecAccess_Write);
@@ -377,6 +380,8 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
     if(!tmpdir->exists())
         tmpdir->createDirectory();
 
+    recursiveCreateDirectory(ESP_WORKUNIT_DIR);
+
     m_sched.start();
 }
 
@@ -3078,3 +3083,137 @@ int CWsWorkunitsSoapBindingEx::onGetForm(IEspContext &context, CHttpRequest* req
     }
     return onGetNotFound(context, request, response, service);
 }
+
+
+void deployArchive(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+{
+    const MemoryBuffer &obj = req.getObject();
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+    WorkunitUpdate wu(factory->createWorkUnit(NULL, "ws_workunits", context.queryUserId()));
+
+    SCMStringBuffer wuid;
+    wu->getWuid(wuid);
+
+    wu->setAction(WUActionCompile);
+
+    if (notEmpty(req.getName()))
+        wu->setJobName(req.getName());
+
+    Owned<IWUQuery> query=wu->updateQuery();
+    StringBuffer text(obj.length(), obj.toByteArray());
+    query->setQueryText(text.str());
+    query.clear();
+
+    wu->commit();
+    wu.clear();
+
+    submitWsWorkunit(context, wuid.str(), req.getCluster(), NULL, 0, true, false);
+    waitForWorkUnitToCompile(wuid.str(), req.getWait());
+
+    WsWuInfo winfo(context, wuid.str());
+    winfo.getCommon(resp.updateWorkunit(), WUINFO_All);
+
+    AuditSystemAccess(context.queryUserId(), true, "Updated %s", wuid.str());
+}
+
+
+StringBuffer &sharedObjectFileName(StringBuffer &filename, const char *name, const char *ext, unsigned copy)
+{
+    filename.append(name);
+    if (copy)
+        filename.append('-').append(copy);
+    if (notEmpty(ext))
+        filename.append(ext);
+    return filename;
+}
+
+void writeSharedObject(const char *srcpath, const MemoryBuffer &obj, StringBuffer &dllpath, StringBuffer &dllname)
+{
+    StringBuffer name, ext;
+    splitFilename(srcpath, NULL, NULL, &name, &ext);
+
+    unsigned copy=0;
+    dllpath.clear().append(ESP_WORKUNIT_DIR).append(sharedObjectFileName(dllname.clear(), name.str(), ext.str(), copy++));
+    while(checkFileExists(dllpath.str()))
+    {
+        dllpath.clear().append(ESP_WORKUNIT_DIR).append(sharedObjectFileName(dllname.clear(), name.str(), ext.str(), copy++));
+    }
+
+    Owned<IFile> f = createIFile(dllpath.str());
+    Owned<IFileIO> io = f->open(IFOcreate);
+    io->write(0, obj.length(), obj.toByteArray());
+}
+
+void deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+{
+    if (isEmpty(req.getFileName()))
+       throw MakeStringException(ECLWATCH_INVALID_INPUT, "File name required when deploying a shared object.");
+
+    if (isEmpty(req.getCluster()))
+       throw MakeStringException(ECLWATCH_INVALID_INPUT, "Cluster name required when deploying a shared object.");
+
+    const MemoryBuffer &obj = req.getObject();
+    StringBuffer dllpath, dllname;
+    writeSharedObject(req.getFileName(), obj, dllpath, dllname);
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+    WorkunitUpdate wu(factory->createWorkUnit(NULL, "ws_workunits", context.queryUserId()));
+
+    SCMStringBuffer wuid;
+    wu->getWuid(wuid);
+    wu->setClusterName(req.getCluster());
+    wu->commit();
+
+    StringBuffer wuXML;
+    if (getWorkunitXMLFromFile(dllpath.str(), wuXML))
+    {
+        Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
+        embeddedWU->loadXML(wuXML);
+        queryExtendedWU(wu)->copyWorkUnit(embeddedWU);
+        //Owned<IWUQuery> query = workunit->updateQuery();
+        //query->setQueryText(eclQuery.s.str());
+    }
+
+    StringBuffer dllurl;
+    createUNCFilename(dllpath.str(), dllurl);
+    unsigned crc = crc_file(dllpath.str());
+
+    Owned<IWUQuery> query = wu->updateQuery();
+    associateLocalFile(query, FileTypeDll, dllpath, "Workunit DLL", crc);
+    queryDllServer().registerDll(dllname.str(), "Workunit DLL", dllurl.str());
+
+    if (notEmpty(req.getName()))
+        wu->setJobName(req.getName());
+    wu->setState(WUStateCompiled);
+    wu->commit();
+    wu.clear();
+
+    WsWuInfo winfo(context, wuid.str());
+    winfo.getCommon(resp.updateWorkunit(), WUINFO_All);
+
+    AuditSystemAccess(context.queryUserId(), true, "Updated %s", wuid.str());
+}
+
+bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+{
+    const char *type = req.getObjType();
+
+    try
+    {
+        if (!context.validateFeatureAccess(OWN_WU_ACCESS, SecAccess_Write, false))
+            throw MakeStringException(ECLWATCH_ECL_WU_ACCESS_DENIED, "Failed to create workunit. Permission denied.");
+
+        if (strieq(type, "archive"))
+            deployArchive(context, req, resp);
+        else if (strieq(type, "shared_object"))
+            deploySharedObject(context, req, resp);
+        else
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "WUDeployWorkunit '%s' unkown object type.", type);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -39,7 +39,7 @@ public:
     }
 
     bool onWUQuery(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp);
-    bool onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp);
+    bool onWUPublishWorkunit(IEspContext &context, IEspWUPublishWorkunitRequest & req, IEspWUPublishWorkunitResponse & resp);
     bool onWUQuerysets(IEspContext &context, IEspWUQuerysetsRequest & req, IEspWUQuerysetsResponse & resp);
     bool onWUQuerysetDetails(IEspContext &context, IEspWUQuerySetDetailsRequest & req, IEspWUQuerySetDetailsResponse & resp);
     bool onWUQuerysetActionQueries(IEspContext &context, IEspWUQuerySetActionQueriesRequest & req, IEspWUQuerySetActionQueriesResponse & resp);
@@ -93,6 +93,7 @@ public:
     bool onWUClusterJobSummaryXLS(IEspContext &context, IEspWUClusterJobSummaryXLSRequest &req, IEspWUClusterJobSummaryXLSResponse &resp);
 
     bool onWUCDebug(IEspContext &context, IEspWUDebugRequest &req, IEspWUDebugResponse &resp);
+    bool onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp);
 
     void setPort(unsigned short _port){port=_port;}