gh-1376 ecl command line support for using eclserver

ecl deploy, publish, and run should be able to refer to a repository
definition or send ECL text without calling eclcc to create an archive.

2 new options have been added:

--main  specify definition in MySQL repository, allowing the
	direct publishing and running of repository definitions.

--ecl-only  send ecl text to hpcc without generating archive, supporting
            running ecl text on eclserver, as well as eclccserver if the
            ecl is self contained.

Fixes gh-1376

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

ecl/eclcmd/eclcmd_common.cpp

@@ -274,6 +274,118 @@ bool EclCmdCommon::finalizeOptions(IProperties *globals)
     return true;
 }
 
+class ConvertEclParameterToArchive
+{
+public:
+    ConvertEclParameterToArchive(EclCmdWithEclTarget &_cmd) : cmd(_cmd)
+    {
+    }
+
+    void appendOptPath(StringBuffer &cmdLine, const char opt, const char *path)
+    {
+        if (!path || !*path)
+            return;
+        if (*path==';')
+            path++;
+        cmdLine.append(" -").append(opt).append(path);
+    }
+
+    void buildCmd(StringBuffer &cmdLine)
+    {
+        cmdLine.set("eclcc -E");
+        appendOptPath(cmdLine, 'I', cmd.optImpPath.str());
+        appendOptPath(cmdLine, 'L', cmd.optLibPath.str());
+        if (cmd.optAttributePath.length())
+            cmdLine.append(" -main ").append(cmd.optAttributePath.get());
+        if (cmd.optManifest.length())
+            cmdLine.append(" -manifest ").append(cmd.optManifest.get());
+        if (cmd.optObj.value.get())
+            cmdLine.append(" ").append(streq(cmd.optObj.value.get(), "stdin") ? "- " : cmd.optObj.value.get());
+    }
+
+    bool eclcc(StringBuffer &out)
+    {
+        StringBuffer cmdLine;
+        buildCmd(cmdLine);
+
+        Owned<IPipeProcess> pipe = createPipeProcess();
+        bool hasInput = streq(cmd.optObj.value.sget(), "stdin");
+        pipe->run(cmd.optVerbose ? "EXEC" : NULL, cmdLine.str(), NULL, hasInput, true, true);
+
+        StringBuffer errors;
+        Owned<EclCmdErrorReader> errorReader = new EclCmdErrorReader(pipe, errors);
+        errorReader->start();
+
+        if (pipe->hasInput())
+        {
+            pipe->write(cmd.optObj.mb.length(), cmd.optObj.mb.toByteArray());
+            pipe->closeInput();
+        }
+        if (pipe->hasOutput())
+        {
+           byte buf[4096];
+           loop
+           {
+                size32_t read = pipe->read(sizeof(buf),buf);
+                if (!read)
+                    break;
+                out.append(read, (const char *) buf);
+            }
+        }
+        int retcode = pipe->wait();
+        errorReader->join();
+
+        if (errors.length())
+            fprintf(stderr, "%s\n", errors.str());
+
+        return (retcode == 0);
+    }
+
+    bool process()
+    {
+        if (cmd.optObj.type!=eclObjSource || cmd.optObj.value.isEmpty())
+            return false;
+
+        StringBuffer output;
+        if (eclcc(output) && output.length() && isArchiveQuery(output.str()))
+        {
+            cmd.optObj.type = eclObjArchive;
+            cmd.optObj.mb.clear().append(output.str());
+            return true;
+        }
+        fprintf(stderr,"\nError creating archive\n");
+        return false;
+    }
+
+private:
+    EclCmdWithEclTarget &cmd;
+
+    class EclCmdErrorReader : public Thread
+    {
+    public:
+        EclCmdErrorReader(IPipeProcess *_pipe, StringBuffer &_errs)
+            : Thread("EclToArchive::ErrorReader"), pipe(_pipe), errs(_errs)
+        {
+        }
+
+        virtual int run()
+        {
+           byte buf[4096];
+           loop
+           {
+                size32_t read = pipe->readError(sizeof(buf), buf);
+                if (!read)
+                    break;
+                errs.append(read, (const char *) buf);
+            }
+            return 0;
+        }
+    private:
+        IPipeProcess *pipe;
+        StringBuffer &errs;
+    };
+};
+
 eclCmdOptionMatchIndicator EclCmdWithEclTarget::matchCommandLineOption(ArgvIterator &iter, bool finalAttempt)
 {
     const char *arg = iter.query();
@@ -305,12 +417,36 @@ eclCmdOptionMatchIndicator EclCmdWithEclTarget::matchCommandLineOption(ArgvItera
         return EclCmdOptionMatch;
     if (iter.matchOption(optManifest, ECLOPT_MANIFEST) || iter.matchOption(optManifest, ECLOPT_MANIFEST_DASH))
         return EclCmdOptionMatch;
+    if (iter.matchOption(optAttributePath, ECLOPT_MAIN) || iter.matchOption(optAttributePath, ECLOPT_MAIN_S))
+        return EclCmdOptionMatch;
+    if (iter.matchFlag(optNoArchive, ECLOPT_ECL_ONLY))
+        return EclCmdOptionMatch;
+
     return EclCmdCommon::matchCommandLineOption(iter, finalAttempt);
 }
 
 bool EclCmdWithEclTarget::finalizeOptions(IProperties *globals)
 {
-    return EclCmdCommon::finalizeOptions(globals);
+    if (!EclCmdCommon::finalizeOptions(globals))
+        return false;
+
+    if (optObj.type == eclObjTypeUnknown && optAttributePath.length())
+    {
+        optNoArchive=true;
+        optObj.type = eclObjSource;
+        optObj.value.set(optAttributePath.get());
+        StringBuffer text(optAttributePath.get());
+        text.append("();");
+        optObj.mb.append(text.str());
+    }
+    else if (optObj.type == eclObjSource && !optNoArchive)
+    {
+        ConvertEclParameterToArchive conversion(*this);
+        if (!conversion.process())
+            return false;
+    }
+    return true;
+
 }
 
 eclCmdOptionMatchIndicator EclCmdWithQueryTarget::matchCommandLineOption(ArgvIterator &iter, bool finalAttempt)

ecl/eclcmd/eclcmd_common.hpp

@@ -57,6 +57,10 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_ACTIVATE_INI "activateDefault"
 #define ECLOPT_ACTIVATE_ENV NULL
 
+#define ECLOPT_MAIN "--main"
+#define ECLOPT_MAIN_S "-main"  //eclcc compatible format
+#define ECLOPT_ECL_ONLY "--ecl-only"
+
 #define ECLOPT_WAIT "--wait"
 #define ECLOPT_WAIT_INI "waitTimeout"
 #define ECLOPT_WAIT_ENV "ECL_WAIT_TIMEOUT"
@@ -161,7 +165,7 @@ public:
 class EclCmdWithEclTarget : public EclCmdCommon
 {
 public:
-    EclCmdWithEclTarget()
+    EclCmdWithEclTarget() : optNoArchive(false)
     {
     }
     virtual eclCmdOptionMatchIndicator matchCommandLineOption(ArgvIterator &iter, bool finalAttempt=false);
@@ -171,6 +175,8 @@ public:
     {
         EclCmdCommon::usage();
         fprintf(stdout,
+            "   --main=<definition>    definition to use from legacy ECL repository\n"
+            "   --ecl-only             send ecl text to hpcc without generating archive\n"
             " eclcc options:\n"
             "   -Ipath                 Add path to locations to search for ecl imports\n"
             "   -Lpath                 Add path to locations to search for system libraries\n"
@@ -182,6 +188,8 @@ public:
     StringBuffer optLibPath;
     StringBuffer optImpPath;
     StringAttr optManifest;
+    StringAttr optAttributePath;
+    bool optNoArchive;
 };
 
 class EclCmdWithQueryTarget : public EclCmdCommon

ecl/eclcmd/eclcmd_core.cpp

@@ -40,133 +40,12 @@ void outputMultiExceptions(const IMultiException &me)
     fprintf(stderr, "\n");
 }
 
-class ConvertEclParameterToArchive
-{
-public:
-    ConvertEclParameterToArchive(EclCmdWithEclTarget &_cmd) : cmd(_cmd)
-    {
-    }
-
-    void appendOptPath(StringBuffer &cmdLine, const char opt, const char *path)
-    {
-        if (!path || !*path)
-            return;
-        if (*path==';')
-            path++;
-        cmdLine.append(" -").append(opt).append(path);
-    }
-
-    void buildCmd(StringBuffer &cmdLine)
-    {
-        cmdLine.set("eclcc -E");
-        appendOptPath(cmdLine, 'I', cmd.optImpPath.str());
-        appendOptPath(cmdLine, 'L', cmd.optLibPath.str());
-        if (cmd.optManifest.length())
-            cmdLine.append(" -manifest ").append(cmd.optManifest.get());
-        if (streq(cmd.optObj.value.sget(), "stdin"))
-            cmdLine.append(" - ");
-        else
-            cmdLine.append(" ").append(cmd.optObj.value.get());
-    }
-
-    bool eclcc(StringBuffer &out)
-    {
-        StringBuffer cmdLine;
-        buildCmd(cmdLine);
-
-        Owned<IPipeProcess> pipe = createPipeProcess();
-        bool hasInput = streq(cmd.optObj.value.sget(), "stdin");
-        pipe->run(cmd.optVerbose ? "EXEC" : NULL, cmdLine.str(), NULL, hasInput, true, true);
-
-        StringBuffer errors;
-        Owned<EclCmdErrorReader> errorReader = new EclCmdErrorReader(pipe, errors);
-        errorReader->start();
-
-        if (pipe->hasInput())
-        {
-            pipe->write(cmd.optObj.mb.length(), cmd.optObj.mb.toByteArray());
-            pipe->closeInput();
-        }
-        if (pipe->hasOutput())
-        {
-           byte buf[4096];
-           loop
-           {
-                size32_t read = pipe->read(sizeof(buf),buf);
-                if (!read)
-                    break;
-                out.append(read, (const char *) buf);
-            }
-        }
-        int retcode = pipe->wait();
-        errorReader->join();
-
-        if (errors.length())
-            fprintf(stderr, "%s\n", errors.str());
-
-        return (retcode == 0);
-    }
-
-    bool process()
-    {
-        if (cmd.optObj.type!=eclObjSource || cmd.optObj.value.isEmpty())
-            return false;
-
-        StringBuffer output;
-        if (eclcc(output) && output.length() && isArchiveQuery(output.str()))
-        {
-            cmd.optObj.type = eclObjArchive;
-            cmd.optObj.mb.clear().append(output.str());
-            return true;
-        }
-        fprintf(stderr,"\nError creating archive\n");
-        return false;
-    }
-
-private:
-    EclCmdWithEclTarget &cmd;
-
-    class EclCmdErrorReader : public Thread
-    {
-    public:
-        EclCmdErrorReader(IPipeProcess *_pipe, StringBuffer &_errs)
-            : Thread("EclToArchive::ErrorReader"), pipe(_pipe), errs(_errs)
-        {
-        }
-
-        virtual int run()
-        {
-           byte buf[4096];
-           loop
-           {
-                size32_t read = pipe->readError(sizeof(buf), buf);
-                if (!read)
-                    break;
-                errs.append(read, (const char *) buf);
-            }
-            return 0;
-        }
-    private:
-        IPipeProcess *pipe;
-        StringBuffer &errs;
-    };
-};
-
-
-
-bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, const char *cluster, const char *name, StringBuffer *wuid, bool displayWuid=true)
+bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, const char *cluster, const char *name, StringBuffer *wuid, bool noarchive, bool displayWuid=true)
 {
     StringBuffer s;
     if (cmd.optVerbose)
         fprintf(stdout, "\nDeploying %s\n", cmd.optObj.getDescription(s).str());
 
-    if (cmd.optObj.type==eclObjSource)
-    {
-        ConvertEclParameterToArchive conversion(cmd);
-        if (!conversion.process())
-            return false;
-    }
-
     Owned<IClientWUDeployWorkunitRequest> req = client->createWUDeployWorkunitRequest();
     switch (cmd.optObj.type)
     {
@@ -178,8 +57,14 @@ bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, const char *
             break;
         case eclObjSource:
         {
-            fprintf(stderr, "Failed to create archive from ECL Text\n");
-            return false;
+            if (noarchive)
+                req->setObjType("ecl_text");
+            else
+            {
+                fprintf(stderr, "Failed to create archive from ECL Text\n");
+                return false;
+            }
+            break;
         }
         default:
             fprintf(stderr, "Cannot deploy %s\n", cmd.optObj.queryTypeName());
@@ -192,6 +77,7 @@ bool doDeploy(EclCmdWithEclTarget &cmd, IClientWsWorkunits *client, const char *
         req->setCluster(cluster);
     req->setObject(cmd.optObj.mb);
     req->setFileName(cmd.optObj.value.sget());
+
     Owned<IClientWUDeployWorkunitResponse> resp = client->WUDeployWorkunit(req);
     if (resp->getExceptions().ordinality())
         outputMultiExceptions(resp->getExceptions());
@@ -275,7 +161,7 @@ public:
         client->addServiceUrl(url.str());
         if (optUsername.length())
             client->setUsernameToken(optUsername.get(), optPassword.sget(), NULL);
-        return doDeploy(*this, client, optCluster.get(), optName.get(), NULL) ? 0 : 1;
+        return doDeploy(*this, client, optCluster.get(), optName.get(), NULL, optNoArchive) ? 0 : 1;
     }
     virtual void usage()
     {
@@ -368,7 +254,7 @@ public:
         StringBuffer wuid;
         if (optObj.type==eclObjWuid)
             wuid.set(optObj.value.get());
-        else if (!doDeploy(*this, client, optCluster.get(), optName.get(), &wuid))
+        else if (!doDeploy(*this, client, optCluster.get(), optName.get(), &wuid, optNoArchive))
             return 1;
 
         StringBuffer descr;
@@ -520,7 +406,7 @@ public:
         else
         {
             req->setCloneWorkunit(false);
-            if (!doDeploy(*this, client, optCluster.get(), optName.get(), &wuid, optVerbose))
+            if (!doDeploy(*this, client, optCluster.get(), optName.get(), &wuid, optNoArchive, optVerbose))
                 return 1;
             req->setWuid(wuid.str());
             if (optVerbose)

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3241,7 +3241,7 @@ int CWsWorkunitsSoapBindingEx::onGetForm(IEspContext &context, CHttpRequest* req
 }
 
 
-void deployArchive(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+void deployEclOrArchive(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
 {
     const MemoryBuffer &obj = req.getObject();
 
@@ -3364,8 +3364,8 @@ bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkun
         if (!context.validateFeatureAccess(OWN_WU_ACCESS, SecAccess_Write, false))
             throw MakeStringException(ECLWATCH_ECL_WU_ACCESS_DENIED, "Failed to create workunit. Permission denied.");
 
-        if (strieq(type, "archive"))
-            deployArchive(context, req, resp);
+        if (strieq(type, "archive")|| strieq(type, "ecl_text"))
+            deployEclOrArchive(context, req, resp);
         else if (strieq(type, "shared_object"))
             deploySharedObject(context, req, resp);
         else