HPCC-19233 Add env. option to force local file to be remote read

Option 'forceRemotePattern' added to environment.conf, if set
components calling enableForceRemoteReads() will treat local files
as if they were remote files (i.e. read them from dafilesrv).

enableForceRemoteReads() calls are added to hthor, roxie, thor
and dfuserver. So setting e.g.:
 forceRemotePattern=/var/lib/HPCCSystems/hpcc-data/*

will cause those components to read any local file under that
path via dafilesrv.

Also:
1) remove automatically installed dafilersv hook from
dafilesrv itself at startup.
2) Fix a couple of Roxie issues spotted when force remote files
are on. There were a couple of places where the filename was
assume to be the same as IFile path name.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/remote/rmtfile.cpp

@@ -21,6 +21,7 @@
 #include "jlib.hpp"
 #include "jio.hpp"
 #include "jlog.hpp"
+#include "jregexp.hpp"
 
 #include "jmutex.hpp"
 #include "jfile.hpp"
@@ -205,6 +206,7 @@ CDaliServixFilter *createDaliServixFilter(IPropertyTree &filterProps)
 class CDaliServixIntercept: public CInterface, implements IDaFileSrvHook
 {
     CIArrayOf<CDaliServixFilter> filters;
+    StringAttr forceRemotePattern;
 
     void addFilter(CDaliServixFilter *filter)
     {
@@ -216,6 +218,10 @@ class CDaliServixIntercept: public CInterface, implements IDaFileSrvHook
     }
 public:
     IMPLEMENT_IINTERFACE;
+    virtual void forceRemote(const char *pattern)
+    {
+        forceRemotePattern.set(pattern);
+    }
     virtual IFile * createIFile(const RemoteFilename & filename)
     {
         SocketEndpoint ep = filename.queryEndpoint();
@@ -259,6 +265,14 @@ public:
                 )
                 return createDaliServixFile(filename);  
         }
+        else if (forceRemotePattern)
+        {
+            StringBuffer localPath;
+            filename.getLocalPath(localPath);
+            // must be local to be here, check if matches forceRemotePattern
+            if (WildMatch(localPath, forceRemotePattern, false))
+                return createDaliServixFile(filename);
+        }
         return NULL;
     }
     virtual void addSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace)
@@ -883,3 +897,16 @@ IDaFileSrvHook *queryDaFileSrvHook()
 {
     return DaliServixIntercept;
 }
+
+void enableForceRemoteReads()
+{
+    const char *forceRemotePattern = queryEnvironmentConf().queryProp("forceRemotePattern");
+    if (!isEmptyString(forceRemotePattern))
+        queryDaFileSrvHook()->forceRemote(forceRemotePattern);
+}
+
+bool testForceRemote(const char *path)
+{
+    const char *forceRemotePattern = queryEnvironmentConf().queryProp("forceRemotePattern");
+    return !isEmptyString(forceRemotePattern) && WildMatch(path, forceRemotePattern, false);
+}

common/remote/rmtfile.hpp

@@ -41,6 +41,7 @@ extern REMOTE_API void filenameToUrl(StringBuffer & out, const char * filename);
 
 interface IDaFileSrvHook : extends IRemoteFileCreateHook
 {
+    virtual void forceRemote(const char *pattern) = 0; // NB: forces all local files matching pattern to be remote reads
     virtual void addSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace) = 0;
     virtual void addRangeFilter(const char *range, const char *dir, const char *sourceRange, bool trace) = 0;
     virtual IPropertyTree *addFilters(IPropertyTree *filters, const SocketEndpoint *ipAddress) = 0;
@@ -58,6 +59,9 @@ extern REMOTE_API unsigned short getActiveDaliServixPort(const IpAddress &ip);
 extern REMOTE_API unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver);
 extern REMOTE_API unsigned getDaliServixVersion(const SocketEndpoint &ep,StringBuffer &ver);
 extern REMOTE_API DAFS_OS getDaliServixOs(const SocketEndpoint &ep);
+extern REMOTE_API void enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+extern REMOTE_API bool testForceRemote(const char *path); // return true if forceRemote setup/pattern will make this path a remote read.
+
 
 extern REMOTE_API void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
 // redirects a daliservix file to a local mount. To remove redirect use NULL for mount dir or NULL for dir

dali/dafilesrv/dafilesrv.cpp

@@ -25,6 +25,7 @@
 #include "jfile.hpp"
 #include "jlog.hpp"
 #include "jmisc.hpp"
+#include "rmtfile.hpp"
 #include "dalienv.hpp"
 
 #ifdef _MSC_VER
@@ -332,6 +333,14 @@ int initDaemon()
 int main(int argc,char **argv) 
 {
     InitModuleObjects();
+
+    /* The dafilesrv hook is installed via the MODULE_INIT process
+     * but it is not wanted in dafilesrv itself, so remove it now.
+    */
+    IDaFileSrvHook *remoteHook = queryDaFileSrvHook();
+    if (remoteHook)
+        removeIFileCreateHook(remoteHook);
+
     EnableSEHtoExceptionMapping();
 #ifndef __64BIT__
     // Restrict stack sizes on 32-bit systems

dali/dfu/dfuserver.cpp

@@ -237,6 +237,8 @@ int main(int argc, const char *argv[])
 
             writeSentinelFile(sentinelFile);
 
+            enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+
             engine->joinListeners();
             if (replserver.get())
                 replserver->stopServer();

ecl/eclagent/eclagent.cpp

@@ -3432,6 +3432,8 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     if (globals->getPropInt("DAFILESRVCACHE", 1))
         setDaliServixSocketCaching(true);
 
+    enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+
     try
     {
 #ifdef MONITOR_ECLAGENT_STATUS  

initfiles/etc/DIR_NAME/environment.conf.in

@@ -58,3 +58,5 @@ additionalPlugins=python2
 # To en-/disable Drop Zone restriction.
 # Default is enabled (true).
 useDropZoneRestriction=true
+# If set, will force matching local file paths to become remote reads, e.g:
+#forceRemotePattern=/var/lib/HPCCSystems/hpcc-data/eclagent/*

roxie/ccd/ccdfile.cpp

@@ -743,7 +743,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress
             ret->setRemote(true);
         }
         ret->setCache(this);
-        files.setValue(localLocation, (ILazyFileIO *)ret);
+        files.setValue(local->queryFilename(), (ILazyFileIO *)ret);
         return ret.getClear();
     }
 

roxie/ccd/ccdmain.cpp

@@ -1022,6 +1022,8 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             openMulticastSocket();
 
         setDaliServixSocketCaching(true);  // enable daliservix caching
+        enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+
         loadPlugins();
         createDelayedReleaser();
         globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());

roxie/ccd/ccdserver.cpp

@@ -12247,7 +12247,14 @@ public:
         if(clusterHandler)
             clusterHandler->splitPhysicalFilename(dir, base);
         else
-            splitFilename(filename.str(), &dir, &dir, &base, &base);
+        {
+            // Check filename is URL and get localpath, only actually necessary if force remote reads are are on
+            RemoteFilename rfn;
+            rfn.setRemotePath(filename);
+            StringBuffer localPath;
+            rfn.getLocalPath(localPath);
+            splitFilename(localPath, &dir, &dir, &base, &base);
+        }
 
         desc->setDefaultDir(dir.str());
 

system/jlib/jfile.cpp

@@ -4102,24 +4102,40 @@ IFile * createIFile(const char * filename)
     IFile *ret = createContainedIFileByHook(filename);
     if (ret)
         return ret;
-    bool linremote=(memcmp(filename,"//",2)==0);
-    if (!linremote&&(memcmp(filename,"\\\\",2)!=0)) // see if remote looking
-        return new CFile(filename);
+
     RemoteFilename rfn;
     rfn.setRemotePath(filename);
+
     if (rfn.isNull())
         throw MakeStringException(-1, "CreateIFile cannot resolve %s", filename);
-    if (rfn.isLocal()) { // ignore dafilesrv request if local and standard port
+
+    ret = createIFileByHook(rfn);           // use daliservix in preference
+    if (ret)
+        return ret;
+
+    // NB: This is forcing OS path access if not a url beginning '//' or '\\'
+    bool linremote=(memcmp(filename,"//",2)==0);
+    if (!linremote&&(memcmp(filename,"\\\\",2)!=0)) // see if remote looking
+        return new CFile(filename);
+
+    if (rfn.isLocal())
+    {
         StringBuffer tmplocal;
-        return new CFile(rfn.getLocalPath(tmplocal).str());
+        rfn.getLocalPath(tmplocal);
+        return new CFile(tmplocal);
     }
+
+    /* NB: to get here, no hook has returned a result and the file is non-local and prefixed with // or \\ */
 #ifdef _WIN32
+    /* NB: this windows specific code below should really be refactored into the standard
+     * hook mechanism. And any path translation should be left/done on the remote side
+     * once it gets to dafilersv.
+     */
     StringBuffer tmplocal;
-    if (linremote||(rfn.queryEndpoint().port!=0)) {
-        ret = createIFileByHook(rfn);           // use daliservix in preference
-        if (ret) 
-            return ret;             
-        while (*filename) {                             // no daliservix so swap '/' for '\' and hope for best
+    if (linremote||(rfn.queryEndpoint().port!=0))
+    {
+        while (*filename)                             // no daliservix so swap '/' for '\' and hope for best
+        {
             if (*filename=='/')
                 tmplocal.append('\\');
             else
@@ -4127,21 +4143,22 @@ IFile * createIFile(const char * filename)
             filename++;
         }
         filename =tmplocal.str();
-    }   
+    }
     return new CWindowsRemoteFile(filename);
 #else
 #ifdef USE_SAMBA
-    if(strncmp(filename, "smb://", 6) == 0)
-        return new CSambaRemoteFile(filename);
-    if(memcmp(filename, "//", 2) == 0) {
+    if (memcmp(filename, "//", 2) == 0)
+    {
         StringBuffer smbfile("smb:");
         smbfile.append(filename);
         return new CSambaRemoteFile(smbfile.str());
     }
-    if(memcmp(filename, "\\\\", 2) == 0) {
+    if (memcmp(filename, "\\\\", 2) == 0)
+    {
         StringBuffer smbfile("smb:");
         int i = 0;
-        while(filename[i]) {
+        while(filename[i])
+        {
             if(filename[i] == '\\')
                 smbfile.append('/');
             else
@@ -4153,18 +4170,12 @@ IFile * createIFile(const char * filename)
 #else
     if (memcmp(filename,"smb://",6)==0)  // don't support samba - try remote
         return createIFile(filename+4);
-    ret = createIFileByHook(rfn);
-    if (!ret) 
-        throw MakeStringException(-1, "CreateIFile::cannot attach to %s. (remote.so not linked?)", filename);
-    return ret;
 #endif
-    return new CFile(filename);
-
+    throw MakeStringException(-1, "createIFile: cannot attach to %s", filename);
 #endif
 }
 
 
-
 IFileIOStream * createIOStream(IFileIO * file)
 {
     return new CFileIOStream(file);

thorlcr/master/thgraphmanager.cpp

@@ -21,6 +21,7 @@
 #include "jfile.hpp"
 #include "jmutex.hpp"
 #include "jlog.hpp"
+#include "rmtfile.hpp"
 
 #include "portlist.h"
 #include "wujobq.hpp"
@@ -1053,6 +1054,8 @@ void thorMain(ILogMsgHandler *logHandler)
         CThorResourceMaster masterResource;
         setIThorResource(masterResource);
 
+        enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+
         Owned<CJobManager> jobManager = new CJobManager(logHandler);
         try {
             LOG(MCdebugProgress, thorJob, "Listening for graph");

thorlcr/slave/thslavemain.cpp

@@ -381,6 +381,8 @@ int main( int argc, char *argv[]  )
             if (daFileSrvHook) // probably always installed
                 daFileSrvHook->addFilters(globals->queryPropTree("NAS"), &slfEp);
 
+            enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+
             StringBuffer thorPath;
             globals->getProp("@thorPath", thorPath);
             recursiveCreateDirectory(thorPath.str());