Merge pull request #14133 from jakesmith/hpcc-24689-dafsstorage

HPCC-24689 Add an option to use dafilersv storage in cloud

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

thorlcr/master/thmastermain.cpp

@@ -967,6 +967,34 @@ int main( int argc, const char *argv[]  )
                 throwStringExceptionV(0, "Failed to connect to all nodes");
             PROGLOG("verified mp connection to rest of cluster");
 
+#ifdef _CONTAINERIZED
+            if (globals->getPropBool("@_dafsStorage"))
+            {
+/* NB: This option is a developer option only.
+
+ * It is intended to be used to bring up a temporary Thor instance that uses local node storage,
+ * as the data plane.
+ * 
+ * It is likely to be deprecated or need reworking, when DFS is refactored to use SP's properly.
+ * 
+ * The mechanism works by:
+ * a) Creating a pseudo StoragePlane (publishes group to Dali).
+ * b) Spins up a dafilesrv thread in each slave container.
+ * c) Changes the default StoragePlane used to publish files, to point to the SP/group created in step (a).
+ * 
+ * In this way, a Thor instance, whilst up, will act similarly to a bare-metal system, using local disks as storage.
+ * This allows quick cloud based allocation/simulation of bare-metal type clusters for testing purposes.
+ * 
+ * NB: This isn't a real StoragePlane, and it will not be accessible by any other component.
+ *
+ */
+                StringBuffer uniqueGrpName;
+                queryNamedGroupStore().addUnique(&queryProcessGroup(), uniqueGrpName);
+                // change default plane
+                queryComponentConfig().setProp("storagePlane", uniqueGrpName);
+                PROGLOG("Persistent Thor group created with group name: %s", uniqueGrpName.str());
+            }
+#endif
             LOG(MCauditInfo, ",Progress,Thor,Startup,%s,%s,%s,%s",nodeGroup.str(),thorname,queueName.str(),logUrl.str());
             auditStartLogged = true;
 

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -518,9 +518,12 @@ public:
                 ForEachItemIn(gn, groupNames)
                 {
 #ifdef _CONTAINERIZED
-                    Owned<IStoragePlane> plane = getStoragePlane(groupNames.item(gn));
-                    assertex(plane);
-                    curDir.append(plane->queryPrefix());
+                    if (!globals->getPropBool("@_dafsStorage"))
+                    {
+                        Owned<IStoragePlane> plane = getStoragePlane(groupNames.item(gn));
+                        assertex(plane);
+                        curDir.append(plane->queryPrefix());
+                    }
 #else
                     if (!getConfigurationDirectory(globals->queryPropTree("Directories"), "data", "thor", groupNames.item(gn), curDir))
                         makePhysicalPartName(logicalName, 0, 0, curDir, 0, os); // legacy

thorlcr/slave/CMakeLists.txt

@@ -31,29 +31,30 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../thorutil 
-         ./../../fs/dafsclient 
-         ./../../system/jhtree 
-         ./../../system/mp 
-         ./../../common/workunit 
-         ./../shared 
-         .
-         ./../../common/environment 
-         ./../../common/deftype 
-         ./../../system/include 
-         ./../../dali/base 
-         ./../../rtl/include 
-         ./../../common/dllserver 
-         ./../msort 
-         ./../../system/jlib 
-         ./../../rtl/eclrtl 
-         ./../master 
-         ./../graph 
-         ./../../common/thorhelper 
-         ./../../roxie/roxiemem
+         ${HPCC_SOURCE_DIR}/common/deftype 
+         ${HPCC_SOURCE_DIR}/common/dllserver 
+         ${HPCC_SOURCE_DIR}/common/environment 
+         ${HPCC_SOURCE_DIR}/common/thorhelper 
+         ${HPCC_SOURCE_DIR}/common/workunit 
+         ${HPCC_SOURCE_DIR}/dali/base 
+         ${HPCC_SOURCE_DIR}/fs/dafsclient 
+         ${HPCC_SOURCE_DIR}/fs/dafsserver
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl 
+         ${HPCC_SOURCE_DIR}/rtl/include 
+         ${HPCC_SOURCE_DIR}/roxie/roxiemem
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/jhtree 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/system/mp 
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/thorlcr/graph 
+         ${HPCC_SOURCE_DIR}/thorlcr/master 
+         ${HPCC_SOURCE_DIR}/thorlcr/msort 
+         ${HPCC_SOURCE_DIR}/thorlcr/shared 
+         ${HPCC_SOURCE_DIR}/thorlcr/slave 
+         ${HPCC_SOURCE_DIR}/thorlcr/thorutil 
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
-         ./../../system/security/shared
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -84,4 +85,8 @@ target_link_libraries (  thorslave_lcr
          activityslaves_lcr 
     )
 
-
+if (CONTAINERIZED) # only used when dev. option _dafsStorage is enabled
+target_link_libraries (  thorslave_lcr 
+         dafsserver
+    )
+endif()

thorlcr/slave/thslavemain.cpp

@@ -52,6 +52,10 @@
 
 #include "slavmain.hpp"
 
+#ifdef _CONTAINERIZED
+#include "dafsserver.hpp"
+#endif
+
 // #define USE_MP_LOG
 
 static INode *masterNode = NULL;
@@ -528,6 +532,43 @@ int main( int argc, const char *argv[]  )
             if (pinterval)
                 startPerformanceMonitor(pinterval, PerfMonStandard, nullptr);
 
+#ifdef _CONTAINERIZED
+            class CServerThread : public CSimpleInterfaceOf<IThreaded>
+            {
+                CThreaded threaded;
+                Owned<IRemoteFileServer> dafsInstance;
+            public:
+                CServerThread() : threaded("CServerThread")
+                {
+                    dafsInstance.setown(createRemoteFileServer());
+                    threaded.init(this);
+                }
+                ~CServerThread()
+                {
+                    PROGLOG("Stopping dafilesrv");
+                    dafsInstance->stop();
+                    threaded.join();
+                }
+            // IThreaded
+                virtual void threadmain() override
+                {
+                    SocketEndpoint listenEp(DAFILESRV_PORT);
+                    try
+                    {
+                        PROGLOG("Starting dafilesrv");
+                        dafsInstance->run(SSLNone, listenEp);
+                    }
+                    catch (IException *e)
+                    {
+                        EXCLOG(e, "dafilesrv error");
+                        throw;
+                    }
+                }
+            };
+            OwnedPtr<CServerThread> dafsThread;
+            if (globals->getPropBool("@_dafsStorage"))
+                dafsThread.setown(new CServerThread);
+#endif
             installDefaultFileHooks(globals);
             slaveMain(jobListenerStopped, slaveLogHandler);
         }