HPCC-9984 Compile direct from a version control system

Add ability to support multiple repositories.

Translate branches to SHA's for atomicity.

Save sha in workunit application values.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -283,6 +283,7 @@ public:
 
 protected:
     void addFilenameDependency(StringBuffer & target, EclCompileInstance & instance, const char * filename);
+    void applyApplicationOptions(IWorkUnit * wu);
     void applyDebugOptions(IWorkUnit * wu);
     bool checkWithinRepository(StringBuffer & attributePath, const char * sourcePathname);
     IFileIO * createArchiveOutputFile(EclCompileInstance & instance);
@@ -337,6 +338,7 @@ protected:
 
     IFileArray inputFiles;
     StringArray inputFileNames;
+    StringArray applicationOptions;
     StringArray debugOptions;
     StringArray compileOptions;
     StringArray linkOptions;
@@ -612,6 +614,25 @@ void EclCC::applyDebugOptions(IWorkUnit * wu)
     }
 }
 
+void EclCC::applyApplicationOptions(IWorkUnit * wu)
+{
+    ForEachItemIn(i, applicationOptions)
+    {
+        const char * option = applicationOptions.item(i);
+        const char * eq = strchr(option, '=');
+        if (eq)
+        {
+            StringAttr name;
+            name.set(option, eq-option);
+            wu->setApplicationValue("eclcc", name, eq+1, true);
+        }
+        else
+        {
+            wu->setApplicationValueInt("eclcc", option, 1, true);
+        }
+    }
+}
+
 //=========================================================================================
 
 ICppCompiler * EclCC::createCompiler(const char * coreName, const char * sourceDir, const char * targetDir)
@@ -975,6 +996,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     instance.wu->setCloneable(true);
 
     applyDebugOptions(instance.wu);
+    applyApplicationOptions(instance.wu);
 
     if (optTargetCompiler != DEFAULT_COMPILER)
         instance.wu->setDebugValue("targetCompiler", compilerTypeText[optTargetCompiler], true);
@@ -1664,7 +1686,12 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     for (; !iter.done(); iter.next())
     {
         const char * arg = iter.query();
-        if (iter.matchOption(tempArg, "--allow"))
+        if (memcmp(arg, "-a", 2)==0)
+        {
+            if (arg[2])
+                applicationOptions.append(arg+2);
+        }
+        else if (iter.matchOption(tempArg, "--allow"))
         {
             allowedPermissions.append(tempArg);
         }
@@ -1943,6 +1970,7 @@ const char * const helpText[] = {
     "    -shared       Generate workunit shared object instead of a stand-alone exe",
     "",
     "Other options:",
+    "!   -aoption[=value] Set an application option",
     "!   --allow=str   Allow use of named feature",
     "!   -b            Batch mode.  Each source file is processed in turn.  Output",
     "!                 name depends on the input filename",

ecl/eclccserver/vchooks/git.sh

@@ -27,82 +27,105 @@
 ################################################################################
 
 originalDir=$PWD
-if [ -z "$GIT_URL" ]; then
-    echo "Need to set GIT_URL" 1>&2
-    exit 2
-fi
-
 if [ -n "$WU_GIT_VERBOSE" ]; then
     GIT_VERBOSE=1
 fi
 
-if [ -n "$WU_GIT_BRANCH" ]; then
-    if [ -z GIT_BRANCH_LOCKED ]; then
-        echo "GIT: Overrideing branch is not allowed" 1>&2
+function fetch_repo {
+    repo=$1
+    prefix="$(echo $repo | tr '[a-z]' '[A-Z]')"
+    eval git_url=\${${prefix}_URL}
+    eval wu_git_branch=\${WU_${prefix}_BRANCH}
+    eval git_branch=\${${prefix}_BRANCH}
+    eval git_branch_locked=\${${prefix}_BRANCH_LOCKED}
+    eval git_directory=\${${prefix}_DIRECTORY}
+
+    if [ -z "$git_url" ]; then
+        echo "Need to set ${prefix}_URL" 1>&2
         exit 2
-    else
-        GIT_BRANCH=$WU_GIT_BRANCH
     fi
-fi
 
-if [ -z "$GIT_BRANCH" ]; then
-    GIT_BRANCH=master
-    if [ -n "$GIT_VERBOSE" ]; then
-        echo "GIT: No branch specified - assuming master" 1>&2
-    fi
-else
-    if [ -n "$GIT_VERBOSE" ]; then
-        echo "GIT: using branch $GIT_BRANCH" 1>&2
+    if [ -n "$wu_git_branch" ]; then
+        if [ -z git_branch_locked ]; then
+            echo "GIT: Overriding branch is not allowed" 1>&2
+            exit 2
+        else
+            git_branch=$wu_git_branch
+        fi
     fi
-fi
 
-if [ -z "$GIT_DIRECTORY" ]; then
-    # We are executed in the eclccserver's home dir - typically /var/lib/HPCCSystems/myeclccserver
-    mkdir -p $PWD/repos/
-    if [ $? -ne 0 ]; then
-        echo "Unable to create directory $PWD/repos/" 1>&2
-        exit 2
+    if [ -z "$git_branch" ]; then
+        git_branch=master
+        if [ -n "$GIT_VERBOSE" ]; then
+            echo "GIT: No branch specified for $git_url - assuming master" 1>&2
+        fi
+    else
+        if [ -n "$GIT_VERBOSE" ]; then
+            echo "GIT: using branch $git_branch for $git_url" 1>&2
+        fi
     fi
-    cd $PWD/repos/
 
-    # URL is likely to be of the form git@github.com:path/to/dir.git
-    # MORE - we could check it is of the expected form...
-    splitURL=(${GIT_URL//[:\/]/ })
-    splitLen=${#splitURL[@]}
-    tail=${splitURL[$splitLen-1]}
+    if [ -z "$git_directory" ]; then
+        # We are executed in the eclccserver's home dir - typically /var/lib/HPCCSystems/myeclccserver
+        mkdir -p $PWD/repos/
+        if [ $? -ne 0 ]; then
+            echo "Unable to create directory $PWD/repos/" 1>&2
+            exit 2
+        fi
+        cd $PWD/repos/
 
-    # tail is now the directory that git clone would create for this url
-    GIT_DIRECTORY=$PWD/$tail
+        # URL is likely to be of the form git@github.com:path/to/dir.git
+        # MORE - we could check it is of the expected form...
+        splitURL=(${git_url//[:\/]/ })
+        splitLen=${#splitURL[@]}
+        tail=${splitURL[$splitLen-1]}
 
-    if [ ! -d $GIT_DIRECTORY ] ; then
-        if [ -n "$GIT_VERBOSE" ]; then
-            echo "GIT: performing initial clone"
-            git clone --bare $GIT_URL 1>&2 || exit $?
-        else
-            git clone --bare $GIT_URL 2>&1 >/dev/null
-            if [ $? -ne 0 ]; then
-                echo "Failed to run git clone $GIT_URL" 1>&2
-                exit 2
+        # tail is now the directory that git clone would create for this url
+        git_directory=$PWD/$tail
+
+        if [ ! -d $git_directory ] ; then
+            if [ -n "$GIT_VERBOSE" ]; then
+                echo "GIT: performing initial clone"
+                git clone --bare $git_url 1>&2 || exit $?
+            else
+                git clone --bare $git_url 2>&1 >/dev/null
+                if [ $? -ne 0 ]; then
+                    echo "Failed to run git clone $git_url" 1>&2
+                    exit 2
+                fi
             fi
         fi
     fi
-fi
-
-cd $GIT_DIRECTORY
-# MORE - may want to not do every time? Add option to check time since last fetched?
-if [ -n "$GIT_VERBOSE" ]; then
-    echo "GIT: using directory $GIT_DIRECTORY" 1>&2
-    git fetch $GIT_URL ${GIT_BRANCH}  1>&2 || exit $?
-else
-    git fetch $GIT_URL ${GIT_BRANCH} 2>&1 >/dev/null || exit $?
+    cd $git_directory
+    # MORE - may want to not do every time? Add option to check time since last fetched?
+    if [ -n "$GIT_VERBOSE" ]; then
+        echo "GIT: using directory $git_directory" 1>&2
+        echo "GIT: Running git fetch $git_url" 1>&2
+        git fetch $git_url 2>&1 >/dev/null
+    else
+        git fetch $git_url >/dev/null
+    fi
     if [ $? -ne 0 ]; then
-        echo "Failed to run git fetch $GIT_URL ${GIT_BRANCH}" 1>&2
+        echo "Failed to run git fetch $git_url" 1>&2
         exit 2
     fi
-fi
+    # Map the branch to a SHA, to avoid issues with the branch being updated by another eclcc process
+    # while this one is compiling (not 100% failsafe, but good enough)
+    last_commit=$(git rev-parse --short $git_branch)
+    if [ $? -ne 0 ]; then
+        echo "Failed to run git rev-parse $git_branch" 1>&2
+        exit 2
+    fi
+    export GIT_INCLUDE_PATH="${GIT_INCLUDE_PATH} -I${git_directory}/{$last_commit}/ -a${repo}_commit=${last_commit}"
+}
+
+repositories=(${GIT_REPOSITORIES//:/ })
+for repo in ${repositories[@]} ; do
+  fetch_repo $repo
+done
 
 cd $originalDir
 if [ -n "$GIT_VERBOSE" ]; then
-    echo GIT: calling eclcc -I$GIT_DIRECTORY/{$GIT_BRANCH}/ "$@"  1>&2
+    echo GIT: calling eclcc $GIT_INCLUDE_PATH "$@"  1>&2
 fi
-eclcc -I$GIT_DIRECTORY/{$GIT_BRANCH}/ "$@"
+eclcc $GIT_INCLUDE_PATH "$@"