Merge pull request #13465 from ghalliday/issue23670

HPCC-23670 Ensure containerized dllserver is independent of environment

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/CMakeLists.txt

@@ -39,12 +39,15 @@ include_directories (
          ./../../system/include 
          ./../../dali/base 
          ./../../system/jlib 
-         ./../../common/environment
          ./../../system/security/shared
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )
 
+if (NOT CONTAINERIZED)
+  include_directories( ./../../common/environment )
+endif()
+
 IF (NOT WIN32)
   include_directories(${BINUTILS_INCLUDE_DIR})
 ENDIF()
@@ -61,8 +64,12 @@ target_link_libraries ( dllserver
          eclrtl 
          dafsclient
          dalibase 
-         environment 
     )
+
+if (NOT CONTAINERIZED)
+  target_link_libraries ( dllserver environment )
+endif()
+
 if (NOT WIN32)
   target_link_libraries ( dllserver ${BINUTILS_LIBRARIES})
 endif()

common/dllserver/dllserver.cpp

@@ -24,9 +24,12 @@
 #include "daclient.hpp"
 #include "dasds.hpp"
 #include "rmtclient.hpp"
-#include "dalienv.hpp"
 #include "thorplugin.hpp"
 
+#ifndef _CONTAINERIZED
+#include "dalienv.hpp"
+#endif
+
 #ifdef _WIN32
 #define DEFAULT_SERVER_ROOTDIR          "c:\\HPCCSystems\\hpcc-data\\temp\\dllserver"
 #else
@@ -744,25 +747,22 @@ IDllServer & queryDllServer()
     CriticalBlock b(dllServerCrit);
     if (!dllServer)
     {
-        if (isCloud())
-        {
-            const char* dllserver_root = getenv("HPCC_DLLSERVER_PATH");
-            assertex(dllserver_root != nullptr);
-            dllServer = new SharedVolumeDllServer(dllserver_root);
-        }
-        else
+#ifdef _CONTAINERIZED
+        const char* dllserver_root = getenv("HPCC_DLLSERVER_PATH");
+        assertex(dllserver_root != nullptr);
+        dllServer = new SharedVolumeDllServer(dllserver_root);
+#else
+        const char* dllserver_root = getenv("DLLSERVER_ROOT");
+        StringBuffer dir;
+        if(dllserver_root == NULL)
         {
-            const char* dllserver_root = getenv("DLLSERVER_ROOT");
-            StringBuffer dir;
-            if(dllserver_root == NULL)
-            {
-                if (envGetConfigurationDirectory("temp","dllserver","dllserver",dir)) // not sure if different instance might be better but never separated in past
-                    dllserver_root = dir.str();
-                else
-                    dllserver_root = DEFAULT_SERVER_ROOTDIR;
-            }
-            dllServer = new DllServer(dllserver_root);
+            if (envGetConfigurationDirectory("temp","dllserver","dllserver",dir)) // not sure if different instance might be better but never separated in past
+                dllserver_root = dir.str();
+            else
+                dllserver_root = DEFAULT_SERVER_ROOTDIR;
         }
+        dllServer = new DllServer(dllserver_root);
+#endif
     }
 
     return *dllServer;

common/workunit/workunit.cpp

@@ -12863,7 +12863,7 @@ extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, c
 {
     StringBuffer fullPathName;
     makeAbsolutePath(name, fullPathName);
-    if (isCloud())
+    if (isContainerized())
     {
         const char *dllserver_root = getenv("HPCC_DLLSERVER_PATH");
         assertex(dllserver_root != nullptr);

dali/base/dasds.cpp

@@ -8763,7 +8763,7 @@ public:
             throw;
         }
         // In nas/non-local storage mode, create a published named group for 1-way files to use
-        if (isCloud())
+        if (isContainerized())
             queryNamedGroupStore().ensureNasGroup(1);
         storeLoaded = true;
         manager->start();

dali/server/daserver.cpp

@@ -144,7 +144,7 @@ void usage(void)
  */
 static bool populateWhiteListFromEnvironment(IWhiteListWriter &writer)
 {
-    if (isCloud())
+    if (isContainerized())
         return false;
     Owned<IRemoteConnection> conn = querySDS().connect("/Environment", 0, 0, INFINITE);
     assertex(conn);

ecl/hthor/hthor.cpp

@@ -352,7 +352,7 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex
         OwnedRoxieString cluster(iwHelper ? iwHelper->getCluster(clusterIdx++) : dwHelper->getCluster(clusterIdx++));
         if(!cluster)
             break;
-        if (isCloud())
+        if (isContainerized())
             throw makeStringException(0, "Output clusters not supported in cloud environment");
         if(!clusterHandler)
         {
@@ -654,7 +654,7 @@ void CHThorDiskWriteActivity::publish()
         // add cluster
         StringBuffer mygroupname;
         Owned<IGroup> mygrp;
-        if (isCloud())
+        if (isContainerized())
         {
             queryNamedGroupStore().getNasGroupName(mygroupname, 1);
             mygrp.setown(queryNamedGroupStore().lookup(mygroupname));
@@ -1222,7 +1222,7 @@ void CHThorIndexWriteActivity::execute()
         // add cluster
         StringBuffer mygroupname;
         Owned<IGroup> mygrp = NULL;
-        if (isCloud())
+        if (isContainerized())
         {
             queryNamedGroupStore().getNasGroupName(mygroupname, 1);
             mygrp.setown(queryNamedGroupStore().lookup(mygroupname));

roxie/ccd/ccdserver.cpp

@@ -11720,7 +11720,7 @@ protected:
             OwnedRoxieString cluster(helper.getCluster(clusterIdx));
             if(!cluster)
                 break;
-            if (isCloud())
+            if (isContainerized())
                 throw makeStringException(0, "Output clusters not supported in cloud environment");
             clusters.append(cluster);
             clusterIdx++;
@@ -11732,7 +11732,7 @@ protected:
         }
         else
         {
-            if (isCloud())
+            if (isContainerized())
             {
                 StringBuffer nasGroupName;
                 queryNamedGroupStore().getNasGroupName(nasGroupName, 1);

system/jlib/jutil.cpp

@@ -1692,14 +1692,13 @@ void doStackProbe()
 #pragma GCC diagnostic pop
 #endif
 
-extern jlib_decl bool isCloud()
+extern jlib_decl bool isContainerized()
 {
-    static bool cloudy = []()
-        {
-            const char *env = getenv("HPCC_containerized");
-            return (env && atoi(env)==1);
-        }();
-    return cloudy;
+#ifdef _CONTAINERIZED
+    return true;
+#else
+    return false;
+#endif
 }
 
 #ifdef _WIN32

system/jlib/jutil.hpp

@@ -284,7 +284,7 @@ extern jlib_decl unsigned msTick();
 extern jlib_decl unsigned usTick();
 extern jlib_decl int write_pidfile(const char * instance);
 extern jlib_decl void doStackProbe();
-extern jlib_decl bool isCloud();
+extern jlib_decl bool isContainerized();
 
 #ifndef arraysize
 #define arraysize(T) (sizeof(T)/sizeof(*T))

thorlcr/master/thmastermain.cpp

@@ -857,7 +857,7 @@ int main( int argc, const char *argv[]  )
         kjServiceMpTag = allocateClusterMPTag();
 
         unsigned numSlaves = 0;
-        if (isCloud())
+        if (isContainerized())
         {
             if (!globals->hasProp("@numSlaves"))
                 throw makeStringException(0, "Number of slaves not defined (numSlaves)");

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -231,7 +231,7 @@ public:
          * that matches the width of the cluster.
          * Also create a 1-way named group, that is used in special cases, e.g. BUILDINDEX,FEW
          */
-        if (isCloud())
+        if (isContainerized())
             queryNamedGroupStore().ensureNasGroup(queryClusterWidth());
     }
     StringBuffer &mangleLFN(CJobBase &job, const char *lfn, StringBuffer &out)
@@ -381,7 +381,7 @@ public:
 
     IFileDescriptor *create(CJobBase &job, const char *logicalName, StringArray &groupNames, IArrayOf<IGroup> &groups, bool overwriteok, unsigned helperFlags=0, bool nonLocalIndex=false, unsigned restrictedWidth=0)
     {
-        if (isCloud())
+        if (isContainerized())
         {
             StringBuffer nasGroupName;
             // NB: normally size = queryClusterWidth(), but can be 1 (e.g. if BUILDINDEX,FEW)
@@ -742,7 +742,7 @@ void fillClusterArray(CJobBase &job, const char *filename, StringArray &clusters
     }
     else
     {
-        if (isCloud())
+        if (isContainerized())
             throw makeStringException(0, "Output clusters not supported in cloud environment");
         const char *cluster = clusters.item(0);
         Owned<IGroup> group = queryNamedGroupStore().lookup(cluster);