HPCC-9984 Compile direct from a version control system

Add code to eclccserver to support installation of hooks in place of eclcc,
and to pass values to those hooks via the environment.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclccserver/eclccserver.cpp

@@ -196,6 +196,44 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         }
     }
 
+    void processOption(const char *option, const char *value, StringBuffer &eclccCmd, StringBuffer &eclccProgName, IPipeProcess &pipe)
+    {
+        if (memicmp(option, "eclcc-", 6) == 0 || *option=='-')
+        {
+            //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
+            const char * start = option + (*option=='-' ? 1 : 6);
+            const char * dash = strchr(start, '-');     // position of second dash, if present
+            StringAttr optName;
+            if (dash)
+                optName.set(start, dash-start);
+            else
+                optName.set(start);
+
+            if (stricmp(optName, "hook") == 0)
+                eclccProgName.set(value);
+            else if (stricmp(optName, "compileOption") == 0)
+                eclccCmd.appendf(" -Wc,%s", value);
+            else if (stricmp(optName, "includeLibraryPath") == 0)
+                eclccCmd.appendf(" -I%s", value);
+            else if (stricmp(optName, "libraryPath") == 0)
+                eclccCmd.appendf(" -L%s", value);
+            else if (stricmp(start, "-allow")==0)
+                ; // Don't allow people to grant themselves permissions
+            else
+                eclccCmd.appendf(" -%s=%s", start, value);
+        }
+        else if (strchr(option, '-'))
+        {
+            StringBuffer envVar(option);
+            envVar.toUpperCase();
+            envVar.replace('-','_');
+            DBGLOG("Setting environment %s=%s", envVar.str(), value);
+            pipe.setenv(envVar, value);
+        }
+        else
+            eclccCmd.appendf(" -f%s=%s", option, value);
+    }
+
     bool compile(const char *wuid, const char *target, const char *targetCluster)
     {
         Owned<IConstWUQuery> query = workunit->getQuery();
@@ -210,7 +248,8 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         query->getQueryText(eclQuery);
         query->getQueryMainDefinition(mainDefinition);
 
-        StringBuffer eclccCmd("eclcc -shared");
+        StringBuffer eclccProgName("eclcc");
+        StringBuffer eclccCmd(" -shared");
         if (eclQuery.length())
             eclccCmd.append(" -");
         if (mainDefinition.length())
@@ -218,25 +257,16 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         if (workunit->getDebugValueBool("addTimingToWorkunit", true))
             eclccCmd.append(" --timings");
 
+        Owned<IPipeProcess> pipe = createPipeProcess();
         Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
         ForEach(*options)
         {
             IPropertyTree &option = options->query();
             const char *name = option.queryProp("@name");
             const char *value = option.queryProp("@value");
-            const char *cluster = option.queryProp("@cluster");
+            const char *cluster = option.queryProp("@cluster");                // if cluster is set it's specific to a particular target
             if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
-            {
-                // options starting '-' are simply passed through to eclcc as name=value
-                // others are passed as -foption=value
-                // if cluster is set it's specific to a particular target
-                eclccCmd.append(" ");
-                if (name[0]!='-')
-                    eclccCmd.append("-f");
-                eclccCmd.append(name);
-                if (value)
-                    eclccCmd.append('=').append(value);
-            }
+                processOption(name, value, eclccCmd, eclccProgName, *pipe);
         }
         eclccCmd.appendf(" -o%s", wuid);
         eclccCmd.appendf(" -platform=%s", target);
@@ -247,30 +277,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
             SCMStringBuffer debugStr, valueStr;
             debugValues->str(debugStr);
             workunit->getDebugValue(debugStr.str(), valueStr);
-            if (memicmp(debugStr.str(), "eclcc-", 6) == 0)
-            {
-                //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
-                const char * start = debugStr.str() + 6;
-                const char * dash = strchr(start, '-');
-                StringAttr optName;
-                if (dash)
-                    optName.set(start, dash-start);
-                else
-                    optName.set(start);
-
-                if (stricmp(optName, "compileOption") == 0)
-                    eclccCmd.appendf(" -Wc,%s", valueStr.str());
-                else if (stricmp(optName, "includeLibraryPath") == 0)
-                    eclccCmd.appendf(" -I%s", valueStr.str());
-                else if (stricmp(optName, "libraryPath") == 0)
-                    eclccCmd.appendf(" -L%s", valueStr.str());
-                else if (stricmp(start, "-allow")==0)
-                    ; // Don't allow people to grant themselves permissions
-                else
-                    eclccCmd.appendf(" -%s=%s", start, valueStr.str());
-            }
-            else
-                eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str());
+            processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe);
         }
         if (workunit->getResultLimit())
         {
@@ -279,9 +286,9 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         try
         {
             unsigned time = msTick();
-            Owned<IPipeProcess> pipe = createPipeProcess();
             Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
-            pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
+            eclccCmd.insert(0, eclccProgName);
+            pipe->run("eclccProgName", eclccCmd, ".", true, false, true, 0);
             errorReader->start();
             try
             {

system/jlib/jthread.cpp

@@ -1258,6 +1258,9 @@ class CWindowsPipeProcess: public CInterface, implements IPipeProcess
     CriticalSection sect;
     bool aborted;
     StringAttr allowedprogs;
+    StringArray envVars;
+    StringArray envValues;
+
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -1334,6 +1337,9 @@ public:
         StartupInfo.hStdInput  = hasinput?hProgInput:GetStdHandle(STD_INPUT_HANDLE);
         
         PROCESS_INFORMATION ProcessInformation;
+
+        // MORE - should create a new environment block that is copy of parent's, then set all the values in envVars/envValues, and pass it
+
         if (!CreateProcess(NULL, (char *)prog, NULL,NULL,TRUE,0,NULL, dir&&*dir?dir:NULL, &StartupInfo,&ProcessInformation)) {
             if (_title) {
                 StringBuffer errstr;
@@ -1352,6 +1358,16 @@ public:
             CloseHandle(hProgError);
         return true;
     }
+
+    virtual void setenv(const char *var, const char *value)
+    {
+        assertex(var);
+        if (!value)
+            value = "";
+        envVars.append(var);
+        envValues.append(value);
+    }
+
     size32_t read(size32_t sz, void *buf)
     {
         DWORD sizeRead;
@@ -1752,6 +1768,8 @@ protected: friend class PipeWriterThread;
     MemoryBuffer stderrbuf;
     size32_t stderrbufsize;
     StringAttr allowedprogs;
+    StringArray envVars;
+    StringArray envValues;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -1876,6 +1894,10 @@ public:
                 if (chdir(dir) == -1)
                     throw MakeStringException(-1, "CLinuxPipeProcess::run: could not change dir to %s", dir.get());
             }
+            ForEachItemIn(idx, envVars)
+            {
+                ::setenv(envVars.item(idx), envValues.item(idx), 1);
+            }
             execvp(argv[0],argv);
             _exit(START_FAILURE);    // must be _exit!!     
         }
@@ -1941,6 +1963,15 @@ public:
         }
         return true;
     }
+
+    virtual void setenv(const char *var, const char *value)
+    {
+        assertex(var);
+        if (!value)
+            value = "";
+        envVars.append(var);
+        envValues.append(value);
+    }
     
     size32_t read(size32_t sz, void *buf)
     {

system/jlib/jthread.hpp

@@ -266,6 +266,7 @@ interface IPipeProcess: extends IInterface
     virtual void abort() = 0;
     virtual void notifyTerminated(HANDLE pid,unsigned retcode) = 0; // internal
     virtual HANDLE getProcessHandle() = 0;                          // used to auto kill
+    virtual void setenv(const char *var, const char *value) = 0;  // Set a value to be passed in the called process environment
 };
 
 extern jlib_decl IPipeProcess *createPipeProcess(const char *allowedprograms=NULL);