Merge pull request #13490 from ghalliday/issue23683

HPCC-23683 Remove dependence of eclcc on environment

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/hpcc/templates/eclccserver.yaml

@@ -45,6 +45,7 @@ spec:
       - name: {{ .name }}
         type: thor
         prefix: {{ .prefix | default "null" }}
+        width: {{ mul (.numSlaves | default 1) ( .channelsPerSlave | default 1) }}
 {{- end }}
   eclccserver-jobspec.yaml: |
     apiVersion: batch/v1

ecl/eclcc/CMakeLists.txt

@@ -66,7 +66,6 @@ target_link_libraries ( eclcc
          deftype 
          dafsclient 
          dalibase 
-         environment
          workunit 
          thorhelper 
          hql
@@ -74,6 +73,10 @@ target_link_libraries ( eclcc
          dllserver
     )
 
+if (CONTAINERIZED)
+  target_link_libraries ( eclcc environment )
+endif()
+
 IF (USE_ZLIB)
     target_link_libraries ( eclcc
         ${ZLIB_LIBRARIES}

ecl/eclcc/eclcc.cpp

@@ -59,7 +59,9 @@
 #include "reservedwords.hpp"
 #include "eclcc.hpp"
 
+#ifndef CONTAINERIZED
 #include "environment.hpp"
+#endif
 
 #ifdef _USE_CPPUNIT
 #include <cppunit/extensions/TestFactoryRegistry.h>
@@ -82,6 +84,7 @@
 
 //The following flag is used to speed up closedown by not freeing items
 static bool optReleaseAllMemory = false;
+static Owned<IPropertyTree> configuration;
 
 #if defined(_WIN32) && defined(_DEBUG)
 static HANDLE leakHandle;
@@ -337,6 +340,7 @@ protected:
     StringAttr optComponentName;
     StringAttr optDFS;
     StringAttr optCluster;
+    StringAttr optConfig;
     StringAttr optScope;
     StringAttr optUser;
     StringAttr optPassword;
@@ -494,6 +498,14 @@ static int doMain(int argc, const char *argv[])
     return 0;
 }
 
+
+static constexpr const char * defaultYaml = R"!!(
+version: "1.0"
+eclccserver:
+    name: eclccserver
+)!!";
+
+
 int main(int argc, const char *argv[])
 {
     EnableSEHtoExceptionMapping();
@@ -501,6 +513,8 @@ int main(int argc, const char *argv[])
     InitModuleObjects();
     queryStderrLogMsgHandler()->setMessageFields(0);
 
+    configuration.setown(loadConfiguration(defaultYaml, argv, "eclccserver", "ECLCCSERVER", nullptr, nullptr));
+
     // Turn logging down (we turn it back up if -v option seen)
     Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(MSGAUD_user| MSGAUD_operator, MSGCLS_error);
     queryLogMsgManager()->changeMonitorFilter(queryStderrLogMsgHandler(), filter);
@@ -514,6 +528,7 @@ int main(int argc, const char *argv[])
         _exit(exitCode);
     }
 
+    configuration.clear();
     releaseAtoms();
     ClearTypeCache();   // Clear this cache before the file hooks are unloaded
     removeFileHooks();
@@ -2275,8 +2290,10 @@ bool EclCC::checkDaliConnected() const
 unsigned EclCC::lookupClusterSize() const
 {
     CriticalBlock b(dfsCrit);  // Overkill at present but maybe one day codegen will start threading? If it does the stack is also iffy!
+#ifndef CONTAINERIZED
     if (!optDFS || disconnectReported || !checkDaliConnected())
         return 0;
+#endif
     if (prevClusterSize != -1)
         return (unsigned) prevClusterSize;
     const char *cluster = clusters ? clusters.tos() : optCluster.str();
@@ -2284,8 +2301,17 @@ unsigned EclCC::lookupClusterSize() const
         prevClusterSize = 0;
     else
     {
+#ifdef CONTAINERIZED
+        VStringBuffer xpath("queues[@name=\"%s\"]", cluster);
+        IPropertyTree * queue = configuration->queryPropTree(xpath);
+        if (queue)
+            prevClusterSize = queue->getPropInt("@width", 1);
+        else
+            prevClusterSize = 0;
+#else
         Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster);
         prevClusterSize = clusterInfo ? clusterInfo->getSize() : 0;
+#endif
     }
     DBGLOG("Cluster %s has size %d", cluster, prevClusterSize);
     return prevClusterSize;
@@ -2490,6 +2516,9 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(optCluster, "-cluster"))
         {
         }
+        else if (iter.matchOption(optConfig, "--config"))
+        {
+        }
         else if (iter.matchOption(optDFS, "-dfs") || /*deprecated*/ iter.matchOption(optDFS, "-dali"))
         {
             // Note - we wait until first use before actually connecting to dali
@@ -2805,6 +2834,9 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         }
         else if (arg[0] == '-')
         {
+            //If --config has been specified, then ignore any unknown options beginning with -- since they will be added to the globals.
+            if ((arg[1] == '-') && optConfig)
+                continue;
             UERRLOG("Error: unrecognised option %s",arg);
             usage();
             return 1;

ecl/eclccserver/eclccserver.cpp

@@ -33,7 +33,8 @@
 #include "dalienv.hpp"
 #endif
 
-Owned<IPropertyTree> globals;
+static Owned<IPropertyTree> globals;
+static const char * * globalArgv = nullptr;
 
 //------------------------------------------------------------------------------------------------------------------
 // We use a separate thread for reading eclcc's stderr output. This prevents the thread that is
@@ -307,6 +308,10 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         splitDirTail(queryCurrentProcessPath(), eclccProgName);
         eclccProgName.append("eclcc");
         StringBuffer eclccCmd(" -shared");
+        //Clone all the options that were passed to eclccserver (but not the filename) and also pass them to eclcc
+        for (const char * * pArg = globalArgv+1; *pArg; pArg++)
+            eclccCmd.append(' ').append(*pArg);
+
         if (eclQuery.length())
             eclccCmd.append(" -");
         if (mainDefinition.length())
@@ -797,6 +802,7 @@ int main(int argc, const char *argv[])
 
     try
     {
+        globalArgv = argv;
         globals.setown(loadConfiguration(defaultYaml, argv, "eclccserver", "ECLCCSERVER", "eclccserver.xml", nullptr));
     }
     catch (IException * e)