Merge pull request #4825 from richardkchapman/eclcc-repo

HPCC-9984 Compile direct from a version control system

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/remote/hooks/git/gitfile.cpp

@@ -28,6 +28,7 @@
 /*
  * Direct access to files in git repositories, by revision, without needing to check them out first
  * Installs hooks into createIFile, spotting filenames of the form /my/directory/.git/{revision}/path/within/git
+ * Bare repositories of the form  /my/directory.git/{revision}/path/within/git also supported
  */
 
 IDirectoryIterator *createGitRepositoryDirectoryIterator(const char *gitFileName, const char *mask=NULL, bool sub=false,bool includedirs=false);
@@ -35,25 +36,21 @@ IDirectoryIterator *createGitRepositoryDirectoryIterator(const char *gitFileName
 static void splitGitFileName(const char *fullName, StringAttr &gitDir, StringAttr &revision, StringAttr &relPath)
 {
     assertex(fullName);
-    const char *git = strstr(fullName, PATHSEPSTR ".git" PATHSEPSTR);
+    const char *git = strstr(fullName, ".git" PATHSEPSTR "{" );
     assertex(git);
-    const char *tail = git+6;
+    const char *tail = git+5;
     gitDir.set(fullName, tail-fullName);
-    if (*tail=='{')
-    {
+    assertex (*tail=='{');
+    tail++;
+    const char *end = strchr(tail, '}');
+    if (!end)
+        throw MakeStringException(0, "Invalid git repository filename - no matching } found");
+    revision.set(tail, end - tail);
+    tail = end+1;
+    if (*tail==PATHSEPCHAR)
         tail++;
-        const char *end = strchr(tail, '}');
-        if (!end)
-            throw MakeStringException(0, "Invalid git repository filename - no matching } found");
-        revision.set(tail, end - tail);
-        tail = end+1;
-        if (*tail==PATHSEPCHAR)
-            tail++;
-        else if (*tail != 0)
-            throw MakeStringException(0, "Invalid git repository filename - " PATHSEPSTR " expected after }");
-    }
-    else
-        revision.clear();
+    else if (*tail != 0)
+        throw MakeStringException(0, "Invalid git repository filename - " PATHSEPSTR " expected after }");
     if (tail && *tail)
     {
         StringBuffer s(tail);
@@ -62,13 +59,17 @@ static void splitGitFileName(const char *fullName, StringAttr &gitDir, StringAtt
     }
     else
         relPath.clear();
+    // Check it's a valid git repository
+    StringBuffer configName(gitDir);
+    configName.append("config");
+    if (!checkFileExists(configName.str()))
+        throw MakeStringException(0, "Invalid git repository - config file %s not found", configName.str());
 }
 
 static StringBuffer & buildGitFileName(StringBuffer &fullname, const char *gitDir, const char *revision, const char *relPath)
 {
     fullname.append(gitDir);
-    if (revision && *revision)
-        fullname.append('{').append(revision).append('}').append(PATHSEPCHAR);
+    fullname.append('{').append(revision).append('}').append(PATHSEPCHAR);
     if (relPath && *relPath)
         fullname.append(relPath);
     return fullname;
@@ -80,7 +81,7 @@ public:
     IMPLEMENT_IINTERFACE;
     GitRepositoryFileIO(const char * gitDirectory, const char * revision, const char * relFileName)
     {
-        VStringBuffer gitcmd("git --git-dir=%s show %s:%s", gitDirectory, revision ? revision : "HEAD", relFileName);
+        VStringBuffer gitcmd("git --git-dir=%s show %s:%s", gitDirectory, (revision && *revision) ? revision : "HEAD", relFileName);
         Owned<IPipeProcess> pipe = createPipeProcess();
         if (pipe->run("git", gitcmd, ".", false, true, false, 0))
         {
@@ -267,6 +268,10 @@ static IFile *createGitFile(const char *gitFileName)
     StringBuffer fname(gitFileName);
     assertex(fname.length());
     removeTrailingPathSepChar(fname);
+    StringAttr gitDirectory, revision, relDir;
+    splitGitFileName(fname, gitDirectory, revision, relDir);
+    if (relDir.isEmpty())
+        return new GitRepositoryFile(fname, 0, true, true);  // Special case the root - ugly but apparently necessary
     Owned<IDirectoryIterator> dir = createGitRepositoryDirectoryIterator(fname, NULL, false, true);
     if (dir->first())
     {
@@ -427,7 +432,7 @@ public:
 protected:
     static bool isGitFileName(const char *fileName)
     {
-        if (fileName && strstr(fileName, PATHSEPSTR ".git" PATHSEPSTR))
+        if (fileName && strstr(fileName, ".git" PATHSEPSTR "{"))
             return true;
         return false;
     }

ecl/eclcc/eclcc.cpp

@@ -283,6 +283,7 @@ public:
 
 protected:
     void addFilenameDependency(StringBuffer & target, EclCompileInstance & instance, const char * filename);
+    void applyApplicationOptions(IWorkUnit * wu);
     void applyDebugOptions(IWorkUnit * wu);
     bool checkWithinRepository(StringBuffer & attributePath, const char * sourcePathname);
     IFileIO * createArchiveOutputFile(EclCompileInstance & instance);
@@ -337,6 +338,7 @@ protected:
 
     IFileArray inputFiles;
     StringArray inputFileNames;
+    StringArray applicationOptions;
     StringArray debugOptions;
     StringArray compileOptions;
     StringArray linkOptions;
@@ -612,6 +614,25 @@ void EclCC::applyDebugOptions(IWorkUnit * wu)
     }
 }
 
+void EclCC::applyApplicationOptions(IWorkUnit * wu)
+{
+    ForEachItemIn(i, applicationOptions)
+    {
+        const char * option = applicationOptions.item(i);
+        const char * eq = strchr(option, '=');
+        if (eq)
+        {
+            StringAttr name;
+            name.set(option, eq-option);
+            wu->setApplicationValue("eclcc", name, eq+1, true);
+        }
+        else
+        {
+            wu->setApplicationValueInt("eclcc", option, 1, true);
+        }
+    }
+}
+
 //=========================================================================================
 
 ICppCompiler * EclCC::createCompiler(const char * coreName, const char * sourceDir, const char * targetDir)
@@ -975,6 +996,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     instance.wu->setCloneable(true);
 
     applyDebugOptions(instance.wu);
+    applyApplicationOptions(instance.wu);
 
     if (optTargetCompiler != DEFAULT_COMPILER)
         instance.wu->setDebugValue("targetCompiler", compilerTypeText[optTargetCompiler], true);
@@ -1670,7 +1692,11 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     for (; !iter.done(); iter.next())
     {
         const char * arg = iter.query();
-        if (iter.matchOption(tempArg, "--allow"))
+        if (iter.matchFlag(tempArg, "-a"))
+        {
+            applicationOptions.append(tempArg);
+        }
+        else if (iter.matchOption(tempArg, "--allow"))
         {
             allowedPermissions.append(tempArg);
         }
@@ -1703,10 +1729,9 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchFlag(optArchive, "-E"))
         {
         }
-        else if (memcmp(arg, "-f", 2)==0)
+        else if (iter.matchFlag(tempArg, "-f"))
         {
-            if (arg[2])
-                debugOptions.append(arg+2);
+            debugOptions.append(tempArg);
         }
         else if (iter.matchFlag(tempBool, "-g"))
         {
@@ -1949,6 +1974,7 @@ const char * const helpText[] = {
     "    -shared       Generate workunit shared object instead of a stand-alone exe",
     "",
     "Other options:",
+    "!   -aoption[=value] Set an application option",
     "!   --allow=str   Allow use of named feature",
     "!   -b            Batch mode.  Each source file is processed in turn.  Output",
     "!                 name depends on the input filename",

ecl/eclccserver/CMakeLists.txt

@@ -52,3 +52,5 @@ target_link_libraries ( eclccserver
          workunit
     )
 
+HPCC_ADD_SUBDIRECTORY (vchooks "PLATFORM")
+

ecl/eclccserver/eclccserver.cpp

@@ -130,6 +130,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
 {
     StringAttr wuid;
     Owned<IWorkUnit> workunit;
+    StringBuffer idxStr;
 
     virtual void reportError(IException *e)
     {
@@ -196,6 +197,54 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         }
     }
 
+    void processOption(const char *option, const char *value, StringBuffer &eclccCmd, StringBuffer &eclccProgName, IPipeProcess &pipe, bool isLocal)
+    {
+        if (memicmp(option, "eclcc-", 6) == 0 || *option=='-')
+        {
+            //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
+            const char * start = option + (*option=='-' ? 1 : 6);
+            const char * dash = strchr(start, '-');     // position of second dash, if present
+            StringAttr optName;
+            if (dash)
+                optName.set(start, dash-start);
+            else
+                optName.set(start);
+
+            if (stricmp(optName, "hook") == 0)
+            {
+                if (isLocal)
+                    throw MakeStringException(0, "eclcc-hook option can not be set per-workunit");  // for security reasons
+                eclccProgName.set(value);
+            }
+            else if (stricmp(optName, "compileOption") == 0)
+                eclccCmd.appendf(" -Wc,%s", value);
+            else if (stricmp(optName, "includeLibraryPath") == 0)
+                eclccCmd.appendf(" -I%s", value);
+            else if (stricmp(optName, "libraryPath") == 0)
+                eclccCmd.appendf(" -L%s", value);
+            else if (stricmp(start, "-allow")==0)
+            {
+                if (isLocal)
+                    throw MakeStringException(0, "eclcc-allow option can not be set per-workunit");  // for security reasons
+                eclccCmd.appendf(" -%s=%s", start, value);
+            }
+            else
+                eclccCmd.appendf(" -%s=%s", start, value);
+        }
+        else if (strchr(option, '-'))
+        {
+            StringBuffer envVar;
+            if (isLocal)
+                envVar.append("WU_");
+            envVar.append(option);
+            envVar.toUpperCase();
+            envVar.replace('-','_');
+            pipe.setenv(envVar, value);
+        }
+        else
+            eclccCmd.appendf(" -f%s=%s", option, value);
+    }
+
     bool compile(const char *wuid, const char *target, const char *targetCluster)
     {
         Owned<IConstWUQuery> query = workunit->getQuery();
@@ -210,7 +259,8 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         query->getQueryText(eclQuery);
         query->getQueryMainDefinition(mainDefinition);
 
-        StringBuffer eclccCmd("eclcc -shared");
+        StringBuffer eclccProgName("eclcc");
+        StringBuffer eclccCmd(" -shared");
         if (eclQuery.length())
             eclccCmd.append(" -");
         if (mainDefinition.length())
@@ -218,25 +268,17 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         if (workunit->getDebugValueBool("addTimingToWorkunit", true))
             eclccCmd.append(" --timings");
 
+        Owned<IPipeProcess> pipe = createPipeProcess();
+        pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
         Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
         ForEach(*options)
         {
             IPropertyTree &option = options->query();
             const char *name = option.queryProp("@name");
             const char *value = option.queryProp("@value");
-            const char *cluster = option.queryProp("@cluster");
+            const char *cluster = option.queryProp("@cluster");                // if cluster is set it's specific to a particular target
             if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
-            {
-                // options starting '-' are simply passed through to eclcc as name=value
-                // others are passed as -foption=value
-                // if cluster is set it's specific to a particular target
-                eclccCmd.append(" ");
-                if (name[0]!='-')
-                    eclccCmd.append("-f");
-                eclccCmd.append(name);
-                if (value)
-                    eclccCmd.append('=').append(value);
-            }
+                processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
         }
         eclccCmd.appendf(" -o%s", wuid);
         eclccCmd.appendf(" -platform=%s", target);
@@ -247,30 +289,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
             SCMStringBuffer debugStr, valueStr;
             debugValues->str(debugStr);
             workunit->getDebugValue(debugStr.str(), valueStr);
-            if (memicmp(debugStr.str(), "eclcc-", 6) == 0)
-            {
-                //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
-                const char * start = debugStr.str() + 6;
-                const char * dash = strchr(start, '-');
-                StringAttr optName;
-                if (dash)
-                    optName.set(start, dash-start);
-                else
-                    optName.set(start);
-
-                if (stricmp(optName, "compileOption") == 0)
-                    eclccCmd.appendf(" -Wc,%s", valueStr.str());
-                else if (stricmp(optName, "includeLibraryPath") == 0)
-                    eclccCmd.appendf(" -I%s", valueStr.str());
-                else if (stricmp(optName, "libraryPath") == 0)
-                    eclccCmd.appendf(" -L%s", valueStr.str());
-                else if (stricmp(start, "-allow")==0)
-                    ; // Don't allow people to grant themselves permissions
-                else
-                    eclccCmd.appendf(" -%s=%s", start, valueStr.str());
-            }
-            else
-                eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str());
+            processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
         }
         if (workunit->getResultLimit())
         {
@@ -279,9 +298,9 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
         try
         {
             unsigned time = msTick();
-            Owned<IPipeProcess> pipe = createPipeProcess();
             Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
-            pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
+            eclccCmd.insert(0, eclccProgName);
+            pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0);
             errorReader->start();
             try
             {
@@ -341,6 +360,11 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
 
 public:
     IMPLEMENT_IINTERFACE;
+    EclccCompileThread(unsigned _idx)
+    {
+        idxStr.append(_idx);
+    }
+
     virtual void init(void *param)
     {
         wuid.set((const char *) param);
@@ -478,7 +502,7 @@ class EclccServer : public CInterface, implements IThreadFactory, implements IAb
     Owned<IThreadPool> pool;
 
     unsigned threadsActive;
-    unsigned maxThreadsActive;
+    CriticalSection threadActiveCrit;
     bool running;
     CSDSServerStatus serverstatus;
     Owned<IJobQueue> queue;
@@ -489,7 +513,6 @@ public:
         : queueName(_queueName), poolSize(_poolSize), serverstatus("ECLCCserver")
     {
         threadsActive = 0;
-        maxThreadsActive = 0;
         running = false;
         pool.setown(createThreadPool("eclccServerPool", this, NULL, poolSize, INFINITE));
         serverstatus.queryProperties()->setProp("@queue",queueName.get());
@@ -551,7 +574,8 @@ public:
 
     virtual IPooledThread *createNew()
     {
-        return new EclccCompileThread();
+        CriticalBlock b(threadActiveCrit);
+        return new EclccCompileThread(threadsActive++);
     }
 
     virtual bool onAbort() 

ecl/eclccserver/vchooks/CMakeLists.txt

@@ -0,0 +1,21 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+FOREACH( iFILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/git.sh
+)
+    install ( PROGRAMS ${iFILES} DESTINATION ${EXEC_DIR} COMPONENT Runtime )
+ENDFOREACH ( iFILES )

ecl/eclccserver/vchooks/git.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+################################################################################
+# Hook to use a git repository for eclcc compilation
+# Files will be fetched from the remote repo before each compile
+#
+# This hook expecte environment variables to be set as follows:
+# GIT_URL       - The remote repository location
+# GIT_BRANCH    - The branch/commit/tag to use  (if ommitted, master is assumed)
+# GIT_DIRECTORY - can be set to allow the dir to be maintained separately - it is
+#                 assumed (at present) that it will refer to a 'bare' repository
+################################################################################
+
+originalDir=$PWD
+
+## Some options can be overridden per workunit, and should accept 0 to mean false
+
+if [ -n "$WU_GIT_VERBOSE" ]; then GIT_VERBOSE=$WU_GIT_VERBOSE ; fi
+if [ "$GIT_VERBOSE" == 0 2> /dev/null ] ; then unset GIT_VERBOSE ; fi
+if [ -n "$WU_GIT_FETCH_EXPIRES" ] ; then GIT_FETCH_EXPIRES=$WU_GIT_FETCH_EXPIRES ; fi
+if [ -n "$WU_GIT_IGNORE_FETCH_ERRORS" ] ; then GIT_IGNORE_FETCH_ERRORS=$WU_GIT_IGNORE_FETCH_ERRORS ; fi
+if [ "$GIT_IGNORE_FETCH_ERRORS" == 0 2> /dev/null ] ; then unset GIT_IGNORE_FETCH_ERRORS ;  fi
+
+function fetch_repo {
+    repo=$1
+    prefix="$(echo $repo | tr '[a-z]' '[A-Z]')"
+    eval git_url=\${${prefix}_URL}
+    eval wu_git_branch=\${WU_${prefix}_BRANCH}
+    eval git_branch=\${${prefix}_BRANCH}
+    eval git_branch_locked=\${${prefix}_BRANCH_LOCKED}
+    eval git_directory=\${${prefix}_DIRECTORY}
+
+    if [ -z "$git_url" ]; then
+        echo "Need to set ${prefix}_URL" 1>&2
+        exit 2
+    fi
+
+    if [ -n "$wu_git_branch" ]; then
+        if [ -z "$git_branch_locked" ]; then
+            echo "GIT: Overriding branch is not allowed" 1>&2
+            exit 2
+        else
+            git_branch=$wu_git_branch
+        fi
+    fi
+
+    if [ -z "$git_branch" ]; then
+        git_branch=master
+        if [ -n "$GIT_VERBOSE" ]; then
+            echo "GIT: No branch specified for $git_url - assuming master" 1>&2
+        fi
+    else
+        if [ -n "$GIT_VERBOSE" ]; then
+            echo "GIT: using branch $git_branch for $git_url" 1>&2
+        fi
+    fi
+
+    if [ -z "$git_directory" ]; then
+        # We are executed in the eclccserver's home dir - typically /var/lib/HPCCSystems/myeclccserver
+        mkdir -p $PWD/repos/
+        if [ $? -ne 0 ]; then
+            echo "Unable to create directory $PWD/repos/" 1>&2
+            exit 2
+        fi
+        cd $PWD/repos/
+
+        # URL is likely to be of the form git@github.com:path/to/dir.git
+        # MORE - we could check it is of the expected form...
+        splitURL=(${git_url//[:\/]/ })
+        splitLen=${#splitURL[@]}
+        tail=${splitURL[$splitLen-1]}
+
+        # tail is now the directory that git clone would create for this url
+        git_directory=$PWD/$tail
+
+        if [ ! -d $git_directory ] ; then
+            if [ -n "$GIT_VERBOSE" ]; then
+                echo "GIT: performing initial clone"
+                git clone --bare $git_url 1>&2 || exit $?
+            else
+                git clone --bare $git_url 2>&1 >/dev/null
+                if [ $? -ne 0 ]; then
+                    echo "Failed to run git clone $git_url" 1>&2
+                    exit 2
+                fi
+            fi
+        fi
+    fi
+    cd $git_directory
+    fetch_needed=1
+    if [ -n "$GIT_FETCH_EXPIRES" -a -f .last_fetched ]; then
+        lastfetch=$(cat .last_fetched)
+        now=$(date +%s)
+        let expires=$lastfetch+$GIT_FETCH_EXPIRES
+        if [ $now -le $expires ]; then
+            unset fetch_needed
+        fi
+    fi
+    if [ -n "$fetch_needed" ]; then
+        if [ -n "$GIT_VERBOSE" ]; then
+            echo "GIT: using directory $git_directory" 1>&2
+            echo "GIT: Running git fetch $git_url" 1>&2
+            git fetch $git_url 2>&1 >/dev/null
+        else
+            git fetch $git_url >/dev/null
+        fi
+        if [ $? -ne 0 ]; then
+            echo "Failed to run git fetch $git_url" 1>&2
+            if [ -n "$GIT_IGNORE_FETCH_ERRORS" ]; then
+                exit 2
+            fi
+        fi
+        date +%s >.last_fetched
+    fi
+    # Map the branch to a SHA, to avoid issues with the branch being updated by another eclcc process
+    # while this one is compiling (not 100% failsafe, but good enough)
+    last_commit=$(git rev-parse --short $git_branch)
+    if [ $? -ne 0 ]; then
+        echo "Failed to run git rev-parse $git_branch" 1>&2
+        exit 2
+    fi
+    export GIT_INCLUDE_PATH="${GIT_INCLUDE_PATH} -I${git_directory}/{$last_commit}/ -a${repo}_commit=${last_commit}"
+}
+
+repositories=(${GIT_REPOSITORIES//:/ })
+for repo in ${repositories[@]} ; do
+  fetch_repo $repo
+done
+
+cd $originalDir
+if [ -n "$GIT_VERBOSE" ]; then
+    echo GIT: calling eclcc $GIT_INCLUDE_PATH "$@"  1>&2
+fi
+eclcc $GIT_INCLUDE_PATH "$@"

system/jlib/jthread.cpp

@@ -1258,6 +1258,9 @@ class CWindowsPipeProcess: public CInterface, implements IPipeProcess
     CriticalSection sect;
     bool aborted;
     StringAttr allowedprogs;
+    StringArray envVars;
+    StringArray envValues;
+
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -1334,6 +1337,9 @@ public:
         StartupInfo.hStdInput  = hasinput?hProgInput:GetStdHandle(STD_INPUT_HANDLE);
         
         PROCESS_INFORMATION ProcessInformation;
+
+        // MORE - should create a new environment block that is copy of parent's, then set all the values in envVars/envValues, and pass it
+
         if (!CreateProcess(NULL, (char *)prog, NULL,NULL,TRUE,0,NULL, dir&&*dir?dir:NULL, &StartupInfo,&ProcessInformation)) {
             if (_title) {
                 StringBuffer errstr;
@@ -1352,6 +1358,16 @@ public:
             CloseHandle(hProgError);
         return true;
     }
+
+    virtual void setenv(const char *var, const char *value)
+    {
+        assertex(var);
+        if (!value)
+            value = "";
+        envVars.append(var);
+        envValues.append(value);
+    }
+
     size32_t read(size32_t sz, void *buf)
     {
         DWORD sizeRead;
@@ -1752,6 +1768,8 @@ protected: friend class PipeWriterThread;
     MemoryBuffer stderrbuf;
     size32_t stderrbufsize;
     StringAttr allowedprogs;
+    StringArray envVars;
+    StringArray envValues;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -1876,6 +1894,10 @@ public:
                 if (chdir(dir) == -1)
                     throw MakeStringException(-1, "CLinuxPipeProcess::run: could not change dir to %s", dir.get());
             }
+            ForEachItemIn(idx, envVars)
+            {
+                ::setenv(envVars.item(idx), envValues.item(idx), 1);
+            }
             execvp(argv[0],argv);
             _exit(START_FAILURE);    // must be _exit!!     
         }
@@ -1941,6 +1963,15 @@ public:
         }
         return true;
     }
+
+    virtual void setenv(const char *var, const char *value)
+    {
+        assertex(var);
+        if (!value)
+            value = "";
+        envVars.append(var);
+        envValues.append(value);
+    }
     
     size32_t read(size32_t sz, void *buf)
     {

system/jlib/jthread.hpp

@@ -266,6 +266,7 @@ interface IPipeProcess: extends IInterface
     virtual void abort() = 0;
     virtual void notifyTerminated(HANDLE pid,unsigned retcode) = 0; // internal
     virtual HANDLE getProcessHandle() = 0;                          // used to auto kill
+    virtual void setenv(const char *var, const char *value) = 0;  // Set a value to be passed in the called process environment
 };
 
 extern jlib_decl IPipeProcess *createPipeProcess(const char *allowedprograms=NULL);