HPCC-21719 Reorganize dafilesrv and client code

Prompted by circular dependency issues (effecting window compiles)
but have wanted to do anyway.
Separates client code from dafilesrv server code.
Streaming code in own module/library.
Remote code unrelated to dafilesrv separated (left in libremote)

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

CMakeLists.txt

@@ -172,6 +172,7 @@ elseif ( NOT MAKE_DOCS_ONLY )
     HPCC_ADD_SUBDIRECTORY (ecl)
     HPCC_ADD_SUBDIRECTORY (ecllibrary)
     HPCC_ADD_SUBDIRECTORY (esp)
+    HPCC_ADD_SUBDIRECTORY (fs)
     HPCC_ADD_SUBDIRECTORY (plugins)
     HPCC_ADD_SUBDIRECTORY (roxie)
     HPCC_ADD_SUBDIRECTORY (rtl)

common/dllserver/CMakeLists.txt

@@ -33,7 +33,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient
          ./../../system/mp 
          ./../../rtl/eclrtl 
          ./../../system/include 
@@ -59,7 +59,7 @@ target_link_libraries ( dllserver
          mp 
          hrpc 
          eclrtl 
-         remote 
+         dafsclient
          dalibase 
          environment 
     )

common/dllserver/dllserver.cpp

@@ -23,7 +23,7 @@
 #include "dllservererr.hpp"
 #include "daclient.hpp"
 #include "dasds.hpp"
-#include "rmtfile.hpp"
+#include "rmtclient.hpp"
 #include "dalienv.hpp"
 #include "thorplugin.hpp"
 

common/environment/CMakeLists.txt

@@ -38,6 +38,7 @@ include_directories (
          ./../../common/remote 
          ./../../system/mp 
          ./../../system/include 
+         ./../../fs/dafsclient
          ./../../dali/base 
          ./../../system/jlib
          ./../../system/security/shared

common/remote/CMakeLists.txt

@@ -27,21 +27,10 @@ project( remote )
 HPCC_ADD_SUBDIRECTORY(hooks)
 
 set (    SRCS 
-         rmtfile.cpp 
          rmtpass.cpp 
          rmtspawn.cpp 
          rmtssh.cpp 
          rmtsmtp.cpp 
-         dafsstream.cpp
-         sockfile.cpp
-         
-         remoteerr.hpp
-         rmtfile.hpp
-         rmtpass.hpp
-         rmtsmtp.hpp
-         rmtspawn.hpp
-         rmtssh.hpp
-         sockfile.hpp
     )
 if (USE_URIPARSER)
   set ( SRCS
@@ -61,12 +50,12 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/security/shared
          ${HPCC_SOURCE_DIR}/system/security/securesocket
          ${HPCC_SOURCE_DIR}/system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
          ${HPCC_SOURCE_DIR}/testing/unittests
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
          ${HPCC_SOURCE_DIR}/ecl/hql
          ${HPCC_SOURCE_DIR}/common/deftype
-         ${HPCC_SOURCE_DIR}/dali/base
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )

common/remote/rmtfile.cpp

@@ -1,915 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#include "platform.h"
-#include "portlist.h"
-
-#include "jlib.hpp"
-#include "jio.hpp"
-#include "jlog.hpp"
-#include "jregexp.hpp"
-
-#include "jmutex.hpp"
-#include "jfile.hpp"
-
-#include "sockfile.hpp"
-#include "rmtfile.hpp"
-#include "remoteerr.hpp"
-
-//----------------------------------------------------------------------------
-
-//#define TEST_DAFILESRV_FOR_UNIX_PATHS     // probably not needed
-
-static class CSecuritySettings
-{
-    unsigned short daliServixPort;
-public:
-    CSecuritySettings()
-    {
-        querySecuritySettings(nullptr, &daliServixPort, nullptr, nullptr, nullptr);
-    }
-
-    unsigned short queryDaliServixPort() { return daliServixPort; }
-} securitySettings;
-
-
-unsigned short getDaliServixPort()
-{
-    return securitySettings.queryDaliServixPort();
-}
-
-
-void setCanAccessDirectly(RemoteFilename & file,bool set)
-{
-    if (set)
-        file.setPort(0);
-    else if (file.getPort()==0)                 // foreign daliservix may be passed in
-        file.setPort(getDaliServixPort());
-
-}
-
-bool canAccessDirectly(const RemoteFilename & file) // not that well named but historical
-{
-    return (file.getPort()==0);
-}
-
-void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir)
-{
-    setDafsLocalMountRedirect(ip,dir,mountdir);
-}
-
-
-
-class CDaliServixFilter : public CInterface
-{
-protected:
-    StringAttr dir, sourceRangeText;
-    SocketEndpointArray sourceRangeIps;
-    bool sourceRangeHasPorts, trace;
-
-    bool checkForPorts(SocketEndpointArray &ips)
-    {
-        ForEachItemIn(i, ips)
-        {
-           if (ips.item(i).port)
-               return true;
-        }
-        return false;
-    }
-public:
-    CDaliServixFilter(const char *_dir, const char *sourceRange, bool _trace) : dir(_dir), trace(_trace)
-    {
-        if (sourceRange)
-        {
-            sourceRangeText.set(sourceRange);
-            sourceRangeIps.fromText(sourceRange, 0);
-            sourceRangeHasPorts = checkForPorts(sourceRangeIps);
-        }
-        else
-            sourceRangeHasPorts = false;
-    }
-    bool queryTrace() const { return trace; }
-    const char *queryDirectory() const { return dir; }
-    bool testPath(const char *path) const
-    {
-        if (!dir) // if no dir in filter, match any
-            return true;
-        else
-            return startsWith(path, dir.get());
-    }
-    bool applyFilter(const SocketEndpoint &ep) const
-    {
-        if (sourceRangeText.length())
-        {
-            SocketEndpoint _ep = ep;
-            if (!sourceRangeHasPorts) // if source range doesn't have ports, only check ip
-                _ep.port = 0;
-            return NotFound != sourceRangeIps.find(_ep);
-        }
-        // NB: If no source range, use target range to decide if filter should apply
-        return testEp(ep);
-    }
-    virtual bool testEp(const SocketEndpoint &ep) const = 0;
-    virtual StringBuffer &getInfo(StringBuffer &info)
-    {
-        if (dir.length())
-            info.append(", dir=").append(dir.get());
-        if (sourceRangeText.get())
-            info.append(", sourcerange=").append(sourceRangeText.get());
-        info.append(", trace=(").append(trace ? "true" : "false").append(")");
-        return info;
-    }
-};
-
-class CDaliServixSubnetFilter : public CDaliServixFilter
-{
-    IpSubNet ipSubNet;
-public:
-    CDaliServixSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace) :
-        CDaliServixFilter(dir, sourceRange, trace)
-    {
-        if (!ipSubNet.set(subnet, mask))
-            throw MakeStringException(0, "Invalid sub net definition: %s, %s", subnet, mask);
-    }
-    virtual bool testEp(const SocketEndpoint &ep) const
-    {
-        return ipSubNet.test(ep);
-    }
-    virtual StringBuffer &getInfo(StringBuffer &info)
-    {
-        info.append("subnet=");
-        ipSubNet.getNetText(info);
-        info.append(", mask=");
-        ipSubNet.getMaskText(info);
-        CDaliServixFilter::getInfo(info);
-        return info;
-    }
-};
-
-class CDaliServixRangeFilter : public CDaliServixFilter
-{
-    StringAttr rangeText;
-    SocketEndpointArray rangeIps;
-    bool rangeIpsHavePorts;
-public:
-    CDaliServixRangeFilter(const char *_range, const char *dir, const char *sourceRange, bool trace)
-        : CDaliServixFilter(dir, sourceRange, trace)
-    {
-        rangeText.set(_range);
-        rangeIps.fromText(_range, 0);
-        rangeIpsHavePorts = checkForPorts(rangeIps);
-    }
-    virtual bool testEp(const SocketEndpoint &ep) const
-    {
-        SocketEndpoint _ep = ep;
-        if (!rangeIpsHavePorts) // if range doesn't have ports, only check ip
-            _ep.port = 0;
-        return NotFound != rangeIps.find(_ep);
-    }
-    virtual StringBuffer &getInfo(StringBuffer &info)
-    {
-        info.append("range=").append(rangeText.get());
-        CDaliServixFilter::getInfo(info);
-        return info;
-    }
-};
-
-CDaliServixFilter *createDaliServixFilter(IPropertyTree &filterProps)
-{
-    CDaliServixFilter *filter = NULL;
-    const char *dir = filterProps.queryProp("@directory");
-    const char *sourceRange = filterProps.queryProp("@sourcerange");
-    bool trace = filterProps.getPropBool("@trace");
-    if (filterProps.hasProp("@subnet"))
-        filter = new CDaliServixSubnetFilter(filterProps.queryProp("@subnet"), filterProps.queryProp("@mask"), dir, sourceRange, trace);
-    else if (filterProps.hasProp("@range"))
-        filter = new CDaliServixRangeFilter(filterProps.queryProp("@range"), dir, sourceRange, trace);
-    else
-        throw MakeStringException(0, "Unknown DaliServix filter definition");
-    return filter;
-}
-
-class CDaliServixIntercept: public CInterface, implements IDaFileSrvHook
-{
-    CIArrayOf<CDaliServixFilter> filters;
-    StringAttr forceRemotePattern;
-
-    void addFilter(CDaliServixFilter *filter)
-    {
-        filters.append(*filter);
-        StringBuffer msg("DaFileSrvHook: adding translateToLocal [");
-        filter->getInfo(msg);
-        msg.append("]");
-        PROGLOG("%s", msg.str());
-    }
-public:
-    IMPLEMENT_IINTERFACE;
-    virtual void forceRemote(const char *pattern)
-    {
-        forceRemotePattern.set(pattern);
-    }
-    virtual IFile * createIFile(const RemoteFilename & filename)
-    {
-        SocketEndpoint ep = filename.queryEndpoint();
-        bool noport = (ep.port==0);
-        setDafsEndpointPort(ep);
-        if (!filename.isLocal()||(ep.port!=DAFILESRV_PORT && ep.port!=SECURE_DAFILESRV_PORT)) // assume standard port is running on local machine
-        {
-#ifdef __linux__
-#ifndef USE_SAMBA
-            if (noport && filters.ordinality())
-            {
-                ForEachItemIn(sn, filters)
-                {
-                    CDaliServixFilter &filter = filters.item(sn);
-                    if (filter.testEp(ep))
-                    {
-                        StringBuffer lPath;
-                        filename.getLocalPath(lPath);
-                        if (filter.testPath(lPath.str()))
-                        {
-                            if (filter.queryTrace())
-                            {
-                                StringBuffer fromPath;
-                                filename.getRemotePath(fromPath);
-                                PROGLOG("Redirecting path: '%s' to '%s", fromPath.str(), lPath.str());
-                            }
-                            return ::createIFile(lPath.str());
-                        }
-                    }
-                }
-            }
-            return createDaliServixFile(filename);
-#endif
-#endif
-            if (!noport)            // expect all filenames that specify port to be dafilesrc or daliservix
-                return createDaliServixFile(filename);  
-            if (filename.isUnixPath()
-#ifdef TEST_DAFILESRV_FOR_UNIX_PATHS        
-                &&testDaliServixPresent(ep)
-#endif
-                )
-                return createDaliServixFile(filename);  
-        }
-        else if (forceRemotePattern)
-        {
-            StringBuffer localPath;
-            filename.getLocalPath(localPath);
-            // must be local to be here, check if matches forceRemotePattern
-            if (WildMatch(localPath, forceRemotePattern, false))
-                return createDaliServixFile(filename);
-        }
-        return NULL;
-    }
-    virtual void addSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace)
-    {
-        Owned<CDaliServixFilter> filter = new CDaliServixSubnetFilter(subnet, mask, dir, sourceRange, trace);
-        addFilter(filter.getClear());
-    }
-    virtual void addRangeFilter(const char *range, const char *dir, const char *sourceRange, bool trace)
-    {
-        Owned<CDaliServixFilter> filter = new CDaliServixRangeFilter(range, dir, sourceRange, trace);
-        addFilter(filter.getClear());
-    }
-    virtual IPropertyTree *addFilters(IPropertyTree *config, const SocketEndpoint *myEp)
-    {
-        if (!config)
-            return NULL;
-        Owned<IPropertyTree> result;
-        Owned<IPropertyTreeIterator> iter = config->getElements("Filter");
-        ForEach(*iter)
-        {
-            Owned<CDaliServixFilter> filter = createDaliServixFilter(iter->query());
-            // Only add filters where myIP matches filter criteria
-            if (!myEp || filter->applyFilter(*myEp))
-            {
-                addFilter(filter.getClear());
-                if (!result)
-                    result.setown(createPTree());
-                result->addPropTree("Filter", LINK(&iter->query()));
-            }
-        }
-        return result.getClear();
-    }
-    virtual IPropertyTree *addMyFilters(IPropertyTree *config, SocketEndpoint *myEp)
-    {
-        if (myEp)
-            return addFilters(config, myEp);
-        else
-        {
-            SocketEndpoint ep;
-            GetHostIp(ep);
-            return addFilters(config, &ep);
-        }
-    }
-    virtual void clearFilters()
-    {
-        filters.kill();
-    }
-} *DaliServixIntercept = NULL;
-
-unsigned short getActiveDaliServixPort(const IpAddress &ip)
-{
-    if (ip.isNull())
-        return 0;
-    SocketEndpoint ep(0, ip);
-    setDafsEndpointPort(ep);
-    try {
-        Owned<ISocket> socket = connectDafs(ep, 10000);
-        return ep.port;
-    }
-    catch (IException *e)
-    {
-        e->Release();
-    }
-    return 0;
-}
-
-bool testDaliServixPresent(const IpAddress &ip)
-{
-    return getActiveDaliServixPort(ip) != 0;
-}
-
-unsigned getDaliServixVersion(const SocketEndpoint &_ep,StringBuffer &ver)
-{
-    SocketEndpoint ep(_ep);
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return 0;
-    try
-    {
-        Owned<ISocket> socket = connectDafs(ep, 10000);
-        return getRemoteVersion(socket,ver);
-    }
-    catch (IException *e)
-    {
-        EXCLOG(e,"getDaliServixVersion");
-        e->Release();
-    }
-    return 0;
-}
-
-struct CDafsOsCacheEntry
-{
-    SocketEndpoint ep;
-    DAFS_OS os;
-    time_t at;
-};
-
-class CDafsOsCache: public SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>
-{
-    void onAdd(void *) {}
-
-    void onRemove(void *et)
-    {
-        CDafsOsCacheEntry *e = (CDafsOsCacheEntry *)et;
-        delete e;
-    }
-    unsigned getHashFromElement(const void *e) const
-    {
-        const CDafsOsCacheEntry &elem=*(const CDafsOsCacheEntry *)e;        
-        return elem.ep.hash(0);
-    }
-
-    unsigned getHashFromFindParam(const void *fp) const
-    {
-        return ((const SocketEndpoint *)fp)->hash(0);
-    }
-
-    const void * getFindParam(const void *p) const
-    {
-        const CDafsOsCacheEntry &elem=*(const CDafsOsCacheEntry *)p;        
-        return (void *)&elem.ep;
-    }
-
-    bool matchesFindParam(const void * et, const void *fp, unsigned) const
-    {
-        return ((CDafsOsCacheEntry *)et)->ep.equals(*(SocketEndpoint *)fp);
-    }
-
-    IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(CDafsOsCacheEntry,SocketEndpoint);
-
-public:
-    static CriticalSection crit;
-
-    CDafsOsCache() 
-    {
-    }
-    ~CDafsOsCache()
-    {
-        SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::_releaseAll();
-    }
-    DAFS_OS lookup(const SocketEndpoint &ep,ISocket *sock)
-    {
-        CriticalBlock block(crit);
-        CDafsOsCacheEntry *r = SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::find(&ep);
-        bool needupdate=false;
-        unsigned t = (unsigned)time(NULL);
-        if (!r) {
-            r = new CDafsOsCacheEntry;
-            r->ep = ep;
-            needupdate = true;
-            SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::add(*r);
-        }
-        else
-            needupdate = (t-r->at>60*5);        // update every 5 mins
-        if (needupdate) {
-            r->os = DAFS_OSunknown;
-            StringBuffer ver;
-            unsigned ret;
-            if (sock)
-                ret = getRemoteVersion(sock,ver);
-            else
-                ret = getDaliServixVersion(ep,ver);
-            if (ret!=0) { // if cross-os needs dafilesrv
-                if (strstr(ver.str(),"Linux")!=NULL)
-                    r->os = DAFS_OSlinux;
-                else if (strstr(ver.str(),"Windows")!=NULL)
-                    r->os = DAFS_OSwindows;
-                else if (strstr(ver.str(),"Solaris")!=NULL)
-                    r->os = DAFS_OSsolaris;
-            }
-            r->at = t;
-        }
-        return r->os;
-    }
-};
-
-
-CriticalSection CDafsOsCache::crit;
-
-
-DAFS_OS getDaliServixOs(const SocketEndpoint &ep,ISocket *socket)
-{
-#ifdef _DEBUG
-    if (ep.isLocal())
-#ifdef _WIN32
-        return DAFS_OSwindows;
-#else
-        return DAFS_OSlinux;
-#endif
-#endif
-    static CDafsOsCache cache;
-    return cache.lookup(ep,socket);
-}
-
-
-
-DAFS_OS getDaliServixOs(const SocketEndpoint &ep)
-{
-    return getDaliServixOs(ep,NULL);
-}
-
-
-unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver)
-{
-    SocketEndpoint ep(0,ip);
-    return getDaliServixVersion(ep,ver);
-}
-
-extern REMOTE_API int setDafileSvrTraceFlags(const SocketEndpoint &_ep,byte flags)
-{
-    SocketEndpoint ep(_ep);
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return -3;
-    try {
-        Owned<ISocket> socket = connectDafs(ep, 5000);
-        return setDafsTrace(socket, flags);
-    }
-    catch (IException *e)
-    {
-        EXCLOG(e,"setDafileSvrTraceFlags");
-        e->Release();
-    }
-    return -2;
-}
-
-extern REMOTE_API int setDafileSvrThrottleLimit(const SocketEndpoint &_ep, ThrottleClass throttleClass, unsigned throttleLimit, unsigned throttleDelayMs, unsigned throttleCPULimit, unsigned queueLimit, StringBuffer *errMsg)
-{
-    SocketEndpoint ep(_ep);
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return -3;
-    try {
-        Owned<ISocket> socket = connectDafs(ep, 5000);
-        return setDafsThrottleLimit(socket, throttleClass, throttleLimit, throttleDelayMs, throttleCPULimit, queueLimit, errMsg);
-    }
-    catch (IException *e)
-    {
-        EXCLOG(e,"setDafileSvrThrottleLimit");
-        e->Release();
-    }
-    return -2;
-}
-
-extern REMOTE_API int getDafileSvrInfo(const SocketEndpoint &_ep, unsigned level, StringBuffer &retstr)
-{
-    SocketEndpoint ep(_ep);
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return false;
-    try {
-        Owned<ISocket> socket = connectDafs(ep, 5000);
-        return getDafsInfo(socket, level, retstr);
-    }
-    catch (IException *e)
-    {
-        EXCLOG(e,"getDafileSvrInfo");
-        e->Release();
-    }
-    return -2;
-}
-
-
-void remoteExtractBlobElements(const char * prefix, const RemoteFilename &file, ExtractedBlobArray & extracted)
-{
-    SocketEndpoint ep(file.queryEndpoint());
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return;
-    StringBuffer filename;
-    remoteExtractBlobElements(ep, prefix, file.getLocalPath(filename).str(), extracted);
-}
-
-
-IFile *createDaliServixFile(const RemoteFilename & file)
-{
-    SocketEndpoint ep(file.queryEndpoint());
-    setDafsEndpointPort(ep);
-    if (ep.isNull())
-        return NULL;
-    StringBuffer path;
-    file.getLocalPath(path);
-    return createRemoteFile(ep, path.str());
-}
-
-void setDaliServixSocketCaching(bool set)
-{
-    clientSetDaliServixSocketCaching(set);
-}
-
-void disconnectRemoteFile(IFile *file)
-{
-    clientDisconnectRemoteFile(file);
-}
-
-void disconnectRemoteIoOnExit(IFileIO *fileio,bool set)
-{
-    clientDisconnectRemoteIoOnExit(fileio,set);
-}
-
-
-bool resetRemoteFilename(IFile *file, const char *newname)
-{
-    return clientResetFilename(file,newname); 
-}
-
-
-void enableAuthentication(bool set)
-{
-    enableDafsAuthentication(set);
-}
-
-bool asyncCopyFileSection(const char *uuid,                 // from genUUID - must be same for subsequent calls
-                            IFile *from,                        // expected to be remote
-                            RemoteFilename &to,
-                            offset_t toofs,                     // (offset_t)-1 created file and copies to start
-                            offset_t fromofs,
-                            offset_t size,                      // (offset_t)-1 for all file
-                            ICopyFileProgress *progress,
-                            unsigned timeout                    // 0 to start, non-zero to wait
-                        )
-{
-    return  clientAsyncCopyFileSection(uuid,from,to,toofs,fromofs,size,progress,timeout);
-}
-
-
-void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime)
-{
-    clientSetRemoteFileTimeouts(maxconnecttime,maxreadtime);
-}
-
-unsigned validateNodes(const SocketEndpointArray &epso,const char *dataDir, const char *mirrorDir, bool chkver, SocketEndpointArray &failures, UnsignedArray &failedcodes, StringArray &failedmessages, const char *filename)
-{
-    // used for detecting duff nodes
-    IPointerArrayOf<ISocket> sockets;
-    // dedup nodes
-    SocketEndpointArray eps;
-    ForEachItemIn(i1,epso)
-        eps.appendUniq(epso.element(i1));
-    unsigned to=30*1000;
-    unsigned n=eps.ordinality();    // use approx log scale (timeout is long but only for failure situation)
-    while (n>1) {
-        n/=2;
-        to+=30*1000;
-    }
-    multiConnect(eps,sockets,to);
-    ForEachItemIn(i,eps) {
-        if (sockets.item(i)==NULL) {
-            failures.append(eps.item(i));
-            failedcodes.append(DAFS_VALIDATE_CONNECT_FAIL);
-            failedmessages.append("Connect failure");
-        }
-    }
-
-    CriticalSection sect;
-    class casyncfor: public CAsyncFor
-    {
-        const SocketEndpointArray &eps;
-        const IPointerArrayOf<ISocket> &sockets;
-        SocketEndpointArray &failures;
-        StringArray &failedmessages;
-        UnsignedArray &failedcodes;
-        CriticalSection &sect;
-        StringAttr dataDir, mirrorDir;
-        bool chkv;
-        const char *filename;
-public:
-        casyncfor(const SocketEndpointArray &_eps,const IPointerArrayOf<ISocket> &_sockets,const char *_dataDir,const char *_mirrorDir,bool _chkv, const char *_filename,SocketEndpointArray &_failures, StringArray &_failedmessages,UnsignedArray &_failedcodes,CriticalSection &_sect)
-            : eps(_eps), sockets(_sockets), failures(_failures),
-              failedmessages(_failedmessages), failedcodes(_failedcodes), sect(_sect),
-              dataDir(_dataDir), mirrorDir(_mirrorDir)
-        { 
-            chkv = _chkv;
-            filename = _filename;
-        }
-        void Do(unsigned i)
-        {
-            ISocket *sock = sockets.item(i);
-            if (!sock)
-                return;
-            SocketEndpoint ep = eps.item(i);
-            bool iswin;
-            unsigned code = 0;
-            StringBuffer errstr;
-            StringBuffer ver;
-            try {
-                getRemoteVersion(sock,ver);
-                iswin = (strstr(ver.str(),"Windows")!=NULL);
-            }
-            catch (IException *e)
-            {
-                code = DAFS_VALIDATE_CONNECT_FAIL;
-                e->errorMessage(errstr);
-                e->Release();
-            }
-            if (!code&&chkv) {
-                const char *rv = ver.str();
-                const char *v = remoteServerVersionString();
-                while (*v&&(*v!='-')&&(*v==*rv)) {
-                    v++;
-                    rv++;
-                }
-                if (*rv!=*v) {
-                    if (*rv) {
-                        while (*rv&&(*rv!='-'))
-                            rv++;
-                        while (*v&&(*v!='-'))
-                            v++;
-                        StringBuffer wanted(v-remoteServerVersionString(),remoteServerVersionString());
-                        ver.setLength(rv-ver.str());
-                        if (strcmp(ver.str(),wanted.str())<0) { // allow >
-                            code = DAFS_VALIDATE_BAD_VERSION;
-                            errstr.appendf("Mismatch dafilesrv version ");
-                            errstr.append(rv-ver.str(),ver.str());
-                            errstr.append(", wanted ");
-                            errstr.append(v-remoteServerVersionString(),remoteServerVersionString());
-                        }
-                    }
-                    else {
-                        code = DAFS_VALIDATE_CONNECT_FAIL;
-                        errstr.appendf("could not contact dafilesrv");
-                    }
-                }
-            }
-            if (!code&&(dataDir.get()||mirrorDir.get())) {
-                clientAddSocketToCache(ep,sock);
-                const char *drivePath = NULL;
-                const char *drivePaths[2];
-                unsigned drives=0;
-                if (mirrorDir.get()) drivePaths[drives++] = mirrorDir.get();
-                if (dataDir.get()) drivePaths[drives++] = dataDir.get();
-                do
-                {
-                    StringBuffer path(drivePaths[--drives]);
-                    addPathSepChar(path);
-                    if (filename)
-                        path.append(filename);
-                    else {
-                        path.append("dafs_");
-                        genUUID(path);
-                        path.append(".tmp");
-                    }
-                    RemoteFilename rfn;
-                    rfn.setPath(ep,path);
-                    Owned<IFile> file = createIFile(rfn);
-                    size32_t sz;
-                    StringBuffer ds;
-                    try {
-                        StringBuffer fullPath;
-                        rfn.getRemotePath(fullPath);
-                        recursiveCreateDirectoryForFile(fullPath);
-                        Owned<IFileIO> fileio = file->open(IFOcreate);
-                        CDateTime dt;
-                        dt.setNow();
-                        dt.getString(ds);
-                        sz = ds.length()+1;
-                        assertex(sz<64);
-                        fileio->write(0,sz,ds.str());
-                    }
-                    catch (IException *e) {
-                        if (e->errorCode()==DISK_FULL_EXCEPTION_CODE)
-                            code |=  (drivePath==dataDir.get()?DAFS_VALIDATE_DISK_FULL_DATA:DAFS_VALIDATE_DISK_FULL_MIRROR);
-                        else
-                            code |=  (drivePath==dataDir.get()?DAFS_VALIDATE_WRITE_FAIL_DATA:DAFS_VALIDATE_WRITE_FAIL_MIRROR);
-                        if (errstr.length())
-                            errstr.append(',');
-                        e->errorMessage(errstr);
-                        e->Release();
-                        continue; // no use trying read
-                    }
-                    try {
-                        Owned<IFileIO> fileio = file->open(IFOread);
-                        char buf[64];
-                        size32_t rd = fileio->read(0,sizeof(buf)-1,buf);
-                        if ((rd!=sz)||(memcmp(buf,ds.str(),sz)!=0)) {
-                            StringBuffer s;
-                            ep.getIpText(s);
-                            throw MakeStringException(-1,"Data discrepancy on disk read of %s of %s",path.str(),s.str());
-                        }
-                    }
-                    catch (IException *e) {
-                        code |=  (drivePath==dataDir.get()?DAFS_VALIDATE_READ_FAIL_DATA:DAFS_VALIDATE_READ_FAIL_MIRROR);
-                        if (errstr.length())
-                            errstr.append(',');
-                        e->errorMessage(errstr);
-                        e->Release();
-                    }
-                    if (!filename||!*filename) {
-                        // delete file created
-                        try {
-                            file->remove();
-                        }
-                        catch (IException *e) {
-                            e->Release();           // supress error
-                        }
-                    }
-                }
-                while (0 != drives);
-            }
-            if (code) {
-                CriticalBlock block(sect);
-                failures.append(ep);
-                failedcodes.append(code);
-                failedmessages.append(errstr.str());
-            }
-        }
-    } afor(eps,sockets,dataDir,mirrorDir,chkver,filename,failures,failedmessages,failedcodes,sect);
-    afor.For(eps.ordinality(), 10, false, true);
-    return failures.ordinality();
-}
-
-static PointerArrayOf<SharedObject> *hookDlls;
-
-static void installFileHook(const char *hookFileSpec);
-
-extern REMOTE_API void installFileHooks(const char *hookFileSpec)
-{
-    if (!hookDlls)
-        hookDlls = new PointerArrayOf<SharedObject>;
-    const char * cursor = hookFileSpec;
-    for (;*cursor;)
-    {
-        StringBuffer file;
-        while (*cursor && *cursor != ENVSEPCHAR)
-            file.append(*cursor++);
-        if(*cursor)
-            cursor++;
-        if(!file.length())
-            continue;
-        installFileHook(file);
-    }
-}
-
-typedef void *(HookInstallFunction)();
-
-static void installFileHook(const char *hookFile)
-{
-    StringBuffer dirPath, dirTail, absolutePath;
-    splitFilename(hookFile, &dirPath, &dirPath, &dirTail, &dirTail);
-    makeAbsolutePath(dirPath.str(), absolutePath);
-    if (!containsFileWildcard(dirTail))
-    {
-        addPathSepChar(absolutePath).append(dirTail);
-        Owned<IFile> file = createIFile(absolutePath);
-        if (file->isDirectory() == foundYes)
-        {
-            installFileHooks(addPathSepChar(absolutePath).append('*'));
-        }
-        else if (file->isFile() == foundYes)
-        {
-            HookInstallFunction *hookInstall;
-            SharedObject *so = new SharedObject(); // MORE - this leaks! Kind-of deliberate right now...
-            if (so->load(file->queryFilename(), false) &&
-                (hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "installFileHook")) != NULL)
-            {
-                hookInstall();
-                hookDlls->append(so);
-            }
-            else
-            {
-                so->unload();
-                delete so;
-                DBGLOG("File hook library %s could not be loaded", hookFile);
-            }
-        }
-        else
-        {
-            DBGLOG("File hook library %s not found", hookFile);
-        }
-    }
-    else
-    {
-        Owned<IDirectoryIterator> dir = createDirectoryIterator(absolutePath, dirTail);
-        ForEach(*dir)
-        {
-            const char *name = dir->query().queryFilename();
-            if (name && *name && *name != '.')
-                installFileHook(name);
-        }
-    }
-}
-
-// Should be called before closedown, ideally. MODEXIT tries to mop up but may be too late to do so cleanly
-
-extern REMOTE_API void removeFileHooks()
-{
-    if (hookDlls)
-    {
-        ForEachItemIn(idx, *hookDlls)
-        {
-            SharedObject *so = hookDlls->item(idx);
-            HookInstallFunction *hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "removeFileHook");
-            if (hookInstall)
-                hookInstall();
-            delete so;
-        }
-        delete hookDlls;
-        hookDlls = NULL;
-    }
-}
-
-MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
-{
-    if(!DaliServixIntercept)
-    {
-        DaliServixIntercept = new CDaliServixIntercept;
-        addIFileCreateHook(DaliServixIntercept);
-    }
-    return true;
-}
-
-MODULE_EXIT()
-{
-    if(DaliServixIntercept)
-    {
-        // delete ConnectionTable;              // too late to delete (jsocket closed down)
-        removeIFileCreateHook(DaliServixIntercept);
-        ::Release(DaliServixIntercept);
-        DaliServixIntercept = NULL;
-    }
-    removeFileHooks();
-}
-
-IDaFileSrvHook *queryDaFileSrvHook()
-{
-    return DaliServixIntercept;
-}
-
-void enableForceRemoteReads()
-{
-    const char *forceRemotePattern = queryEnvironmentConf().queryProp("forceRemotePattern");
-    if (!isEmptyString(forceRemotePattern))
-        queryDaFileSrvHook()->forceRemote(forceRemotePattern);
-}
-
-bool testForceRemote(const char *path)
-{
-    const char *forceRemotePattern = queryEnvironmentConf().queryProp("forceRemotePattern");
-    return !isEmptyString(forceRemotePattern) && WildMatch(path, forceRemotePattern, false);
-}

common/remote/rmtfile.hpp

@@ -1,111 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef RMTFILE_HPP
-#define RMTFILE_HPP
-
-#include "jsocket.hpp"
-#include "jfile.hpp"
-
-#include "sockfile.hpp"
-
-#ifdef REMOTE_EXPORTS
-#define REMOTE_API DECL_EXPORT
-#else
-#define REMOTE_API DECL_IMPORT
-#endif
-
-enum DAFS_OS
-{
-    DAFS_OSunknown,
-    DAFS_OSwindows,
-    DAFS_OSlinux,
-    DAFS_OSsolaris
-};
-
-extern REMOTE_API void filenameToUrl(StringBuffer & out, const char * filename);
-
-interface IDaFileSrvHook : extends IRemoteFileCreateHook
-{
-    virtual void forceRemote(const char *pattern) = 0; // NB: forces all local files matching pattern to be remote reads
-    virtual void addSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace) = 0;
-    virtual void addRangeFilter(const char *range, const char *dir, const char *sourceRange, bool trace) = 0;
-    virtual IPropertyTree *addFilters(IPropertyTree *filters, const SocketEndpoint *ipAddress) = 0;
-    virtual IPropertyTree *addMyFilters(IPropertyTree *filters, SocketEndpoint *myEp=NULL) = 0;
-    virtual void clearFilters() = 0;
-};
-extern REMOTE_API IDaFileSrvHook *queryDaFileSrvHook();
-extern REMOTE_API unsigned short getDaliServixPort();  // assumed just the one for now
-extern REMOTE_API void setCanAccessDirectly(RemoteFilename & file,bool set);
-extern REMOTE_API void setDaliServixSocketCaching(bool set);
-extern REMOTE_API bool canAccessDirectly(const RemoteFilename & file);
-extern REMOTE_API IFile *createDaliServixFile(const RemoteFilename & file);
-extern REMOTE_API bool testDaliServixPresent(const IpAddress &ip);
-extern REMOTE_API unsigned short getActiveDaliServixPort(const IpAddress &ip);
-extern REMOTE_API unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver);
-extern REMOTE_API unsigned getDaliServixVersion(const SocketEndpoint &ep,StringBuffer &ver);
-extern REMOTE_API DAFS_OS getDaliServixOs(const SocketEndpoint &ep);
-extern REMOTE_API void enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
-extern REMOTE_API bool testForceRemote(const char *path); // return true if forceRemote setup/pattern will make this path a remote read.
-
-
-extern REMOTE_API void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
-// redirects a daliservix file to a local mount. To remove redirect use NULL for mount dir or NULL for dir
-
-
-extern REMOTE_API void remoteExtractBlobElements(const char * prefix, const RemoteFilename &file, ExtractedBlobArray & extracted);
-
-extern REMOTE_API int setDafileSvrTraceFlags(const SocketEndpoint &ep,byte flags);
-extern REMOTE_API int setDafileSvrThrottleLimit(const SocketEndpoint &_ep, ThrottleClass throttleClass, unsigned throttleLimit, unsigned throttleDelayMs, unsigned throttleCPULimit, unsigned queueLimit, StringBuffer *errMsg=NULL);
-extern REMOTE_API int getDafileSvrInfo(const SocketEndpoint &ep, unsigned level, StringBuffer &retstr);
-
-extern REMOTE_API void disconnectRemoteFile(IFile *file);
-extern REMOTE_API void disconnectRemoteIoOnExit(IFileIO *fileio,bool set=true);
-
-extern REMOTE_API bool resetRemoteFilename(IFile *file, const char *newname); // returns false if not remote
-
-
-extern REMOTE_API void enableAuthentication(bool set=true); // default enabled for clients, disabled for server
-
-extern REMOTE_API bool asyncCopyFileSection(const char *uuid,   // from genUUID - must be same for subsequent calls
-                            IFile *from,                        // expected to be remote
-                            RemoteFilename &to,
-                            offset_t toofs,                     // (offset_t)-1 created file and copies to start
-                            offset_t fromofs,
-                            offset_t size,                      // (offset_t)-1 for all file
-                            ICopyFileProgress *progress,
-                            unsigned timeout                    // 0 to start, non-zero to wait
-                        ); // returns true when done
-
-extern REMOTE_API void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime);
-
-#define DAFS_VALIDATE_CONNECT_FAIL  (0x01)
-#define DAFS_VALIDATE_BAD_VERSION   (0x02)
-#define DAFS_VALIDATE_WRITE_FAIL_DATA  (0x12)
-#define DAFS_VALIDATE_READ_FAIL_DATA   (0x14)
-#define DAFS_VALIDATE_DISK_FULL_DATA   (0x18)
-#define DAFS_VALIDATE_WRITE_FAIL_MIRROR  (0x22)
-#define DAFS_VALIDATE_READ_FAIL_MIRROR   (0x24)
-#define DAFS_VALIDATE_DISK_FULL_MIRROR   (0x28)
-#define DAFS_SCRIPT_FAIL            (0x40)
-                                
-extern REMOTE_API unsigned validateNodes(const SocketEndpointArray &eps,const char *dataDir, const char *mirrorDir, bool chkver, SocketEndpointArray &failures, UnsignedArray &failedcodes, StringArray &failedmessages, const char *filename=NULL);
-
-extern REMOTE_API void installFileHooks(const char *filespec);
-extern REMOTE_API void removeFileHooks(); // Should be called before closedown
-
-#endif

common/remote/rmtpass.cpp

@@ -26,7 +26,6 @@
 #include "portlist.h"
 #include "jsocket.hpp"
 
-#include "rmtfile.hpp"
 #include "rmtpass.hpp"
 
 void CachedPasswordProvider::addPasswordForFilename(const char * filename)

common/remote/rmtspawn.cpp

@@ -21,6 +21,11 @@
 #include "portlist.h"
 
 
+/* NB: error numbers/text are currently in remoteerr.hpp 
+ * Which mainly contains error codes used by dafilesrv and clients
+ * This is not really related, but for legacy reasons, the error code
+ * numbers remain the same, and remain within the same header file.
+ */
 #include "remoteerr.hpp"
 
 #include "rmtspawn.hpp"

common/remote/sockfile.hpp

@@ -1,164 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef SOCKFILE_HPP
-#define SOCKFILE_HPP
-
-#include "jsocket.hpp"
-#include "jfile.hpp"
-
-#ifdef REMOTE_EXPORTS
-#define REMOTE_API DECL_EXPORT
-#else
-#define REMOTE_API DECL_IMPORT
-#endif
-
-#define DAFILESRV_METAINFOVERSION 2
-
-#define DAFILESRV_STREAMREAD_MINVERSION 22
-#define DAFILESRV_STREAMGENERAL_MINVERSION 25
-
-enum ThrottleClass
-{
-    ThrottleStd,
-    ThrottleSlow,
-    ThrottleClassMax
-};
-
-// RemoteFileServer throttling defaults
-#define DEFAULT_THREADLIMIT 100
-#define DEFAULT_THREADLIMITDELAYMS (60*1000)
-#define DEFAULT_ASYNCCOPYMAX 10
-
-#define DEFAULT_STDCMD_PARALLELREQUESTLIMIT 80
-#define DEFAULT_STDCMD_THROTTLEDELAYMS 1000
-#define DEFAULT_STDCMD_THROTTLECPULIMIT 85
-#define DEFAULT_STDCMD_THROTTLEQUEUELIMIT 1000
-
-#define DEFAULT_SLOWCMD_PARALLELREQUESTLIMIT 20
-#define DEFAULT_SLOWCMD_THROTTLEDELAYMS 5000
-#define DEFAULT_SLOWCMD_THROTTLECPULIMIT 75
-#define DEFAULT_SLOWCMD_THROTTLEQUEUELIMIT 1000
-
-
-enum RowServiceCfg
-{
-    rs_off,     // No dedicated row service, allows row service commands on std. dafilesrv port.
-    rs_on,      // Dedicated row service on own port accepting authorized signed connections only. Row service commands on std. dafilersv port will be refused.
-    rs_both,    // Dedicated row service on own port accepting authorized signed connections only. Still accepts unsigned connection on std. dafilesrv port
-    rs_onssl,   // Same as rs_on, but SSL
-    rs_bothssl  // Same as rs_only, but SSL
-};
-
-interface IRemoteFileServer : extends IInterface
-{
-    virtual void run(DAFSConnectCfg connectMethod, const SocketEndpoint &listenep, unsigned sslPort=0, const SocketEndpoint *rowServiceEp=nullptr, bool rowServiceSSL=false, bool rowServiceOnStdPort=true) = 0;
-    virtual void run(DAFSConnectCfg _connectMethod, ISocket *listenSocket, ISocket *secureSocket, ISocket *rowServiceSocket) = 0;
-    virtual void stop() = 0;
-    virtual unsigned idleTime() = 0; // in ms
-    virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;
-    virtual StringBuffer &getStats(StringBuffer &stats, bool reset) = 0;
-};
-
-interface IRemoteRowServer : extends IInterface
-{
-    virtual void run(unsigned port=0) = 0;
-    virtual void stop() = 0;
-    virtual unsigned idleTime() = 0; // in ms
-    virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;
-    virtual StringBuffer &getStats(StringBuffer &stats, bool reset) = 0;
-};
-
-#define FILESRV_VERSION 25 // don't forget VERSTRING in sockfile.cpp
-
-interface IKeyManager;
-interface IDelayedFile;
-interface IDaFsConnection;
-
-extern REMOTE_API IFile * createRemoteFile(SocketEndpoint &ep,const char * _filename);
-extern REMOTE_API unsigned getRemoteVersion(ISocket * _socket, StringBuffer &ver);
-extern REMOTE_API unsigned getCachedRemoteVersion(IDaFsConnection &daFsConnection);
-extern REMOTE_API unsigned getCachedRemoteVersion(const SocketEndpoint &ep, bool secure);
-extern REMOTE_API unsigned stopRemoteServer(ISocket * _socket);
-extern REMOTE_API const char *remoteServerVersionString();
-extern REMOTE_API IRemoteFileServer * createRemoteFileServer(unsigned maxThreads=DEFAULT_THREADLIMIT, unsigned maxThreadsDelayMs=DEFAULT_THREADLIMITDELAYMS, unsigned maxAsyncCopy=DEFAULT_ASYNCCOPYMAX, IPropertyTree *keyPairInfo=nullptr);
-extern REMOTE_API int setDafsTrace(ISocket * socket,byte flags);
-extern REMOTE_API int setDafsThrottleLimit(ISocket * socket, ThrottleClass throttleClass, unsigned throttleLimit, unsigned throttleDelayMs, unsigned throttleCPULimit, unsigned queueLimit, StringBuffer *errMsg=NULL);
-extern REMOTE_API bool enableDafsAuthentication(bool on);
-extern REMOTE_API void remoteExtractBlobElements(const SocketEndpoint &ep, const char * prefix, const char * filename, ExtractedBlobArray & extracted);
-extern REMOTE_API int getDafsInfo(ISocket * socket, unsigned level, StringBuffer &retstr);
-extern REMOTE_API void setDafsEndpointPort(SocketEndpoint &ep);
-extern REMOTE_API void setDafsLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
-extern REMOTE_API ISocket *connectDafs(SocketEndpoint &ep, unsigned timeoutms); // NOTE: might alter ep.port if configured for multiple ports ...
-extern REMOTE_API ISocket *checkSocketSecure(ISocket *socket);
-
-
-extern REMOTE_API void setRemoteOutputCompressionDefault(const char *type);
-extern REMOTE_API const char *queryOutputCompressionDefault();
-
-interface IOutputMetaData;
-class RowFilter;
-interface IRemoteFileIO : extends IFileIO
-{
-    virtual void addVirtualFieldMapping(const char *fieldName, const char *fieldValue) = 0;
-    virtual void ensureAvailable() = 0;
-};
-extern REMOTE_API IRemoteFileIO *createRemoteFilteredFile(SocketEndpoint &ep, const char * filename, IOutputMetaData *actual, IOutputMetaData *projected, const RowFilter &fieldFilters, bool compressed, bool grouped, unsigned __int64 chooseNLimit);
-
-interface IIndexLookup;
-extern REMOTE_API IIndexLookup *createRemoteFilteredKey(SocketEndpoint &ep, const char * filename, unsigned crc, IOutputMetaData *actual, IOutputMetaData *projected, const RowFilter &fieldFilters, unsigned __int64 chooseNLimit);
-
-typedef unsigned char RemoteFileCommandType;
-extern REMOTE_API RemoteFileCommandType queryRemoteStreamCmd(); // used by testsocket only
-
-interface IFileDescriptor;
-typedef IFileDescriptor *(*FileDescriptorFactoryType)(IPropertyTree *);
-extern REMOTE_API void configureRemoteCreateFileDescriptorCB(FileDescriptorFactoryType cb);
-
-
-// client only
-extern FileDescriptorFactoryType queryRemoteCreateFileDescriptorCB();
-interface IDaFsConnection : extends IInterface
-{
-    virtual void close(int handle) = 0;
-    virtual void send(MemoryBuffer &sendMb, MemoryBuffer &reply) = 0;
-    virtual unsigned getVersion(StringBuffer &ver) = 0;
-    virtual const SocketEndpoint &queryEp() const = 0;
-};
-
-IDaFsConnection *createDaFsConnection(const SocketEndpoint &ep, DAFSConnectCfg connectMethod, const char *tracing);
-
-extern void clientSetDaliServixSocketCaching(bool set);
-extern void clientDisconnectRemoteFile(IFile *file);
-extern void clientDisconnectRemoteIoOnExit(IFileIO *fileio,bool set);
-
-extern bool clientResetFilename(IFile *file, const char *newname); // returns false if not remote
-
-extern bool clientAsyncCopyFileSection(const char *uuid,    // from genUUID - must be same for subsequent calls
-                        IFile *from,                        // expected to be remote
-                        RemoteFilename &to,
-                        offset_t toofs,                     // (offset_t)-1 created file and copies to start
-                        offset_t fromofs,
-                        offset_t size,                      // (offset_t)-1 for all file
-                        ICopyFileProgress *progress,
-                        unsigned timeout                    // 0 to start, non-zero to wait
-                        ); // returns true when done
-
-extern void clientSetRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime);
-extern void clientAddSocketToCache(SocketEndpoint &ep,ISocket *socket);
-
-#endif

dali/CMakeLists.txt

@@ -14,7 +14,6 @@
 #    limitations under the License.
 ################################################################################
 HPCC_ADD_SUBDIRECTORY (base)
-HPCC_ADD_SUBDIRECTORY (dafilesrv  "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (dalidiag "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (dalistop "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (datest "PLATFORM")

dali/base/CMakeLists.txt

@@ -56,15 +56,15 @@ set (    INCLUDES
     )   
 
 include_directories ( 
-         ./../../common/deftype 
-         ./../../common/remote 
-         ./../server 
-         ./../../system/mp 
-         ./../../system/include 
-         ./../../system/jlib 
-         ./../../rtl/include 
-         ./../../system/security/shared
-         ./../../system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/common/deftype 
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
+         ${HPCC_SOURCE_DIR}/dali/server 
+         ${HPCC_SOURCE_DIR}/system/mp 
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/rtl/include 
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/cryptohelper
     )
 
 ADD_DEFINITIONS( -DLOGMSGCOMPONENT=3 -D_USRDLL -DDALI_EXPORTS )
@@ -75,11 +75,13 @@ if(NOT PLUGIN)
     install(
         TARGETS dalibase
         RUNTIME DESTINATION ${EXEC_DIR}
-        LIBRARY DESTINATION ${LIB_DIR})
+        LIBRARY DESTINATION ${LIB_DIR}
+    )
     target_link_libraries(
         dalibase 
         jlib
         mp 
         hrpc 
-        remote)
+        dafsclient
+    )
 endif()

dali/base/dadfs.cpp

@@ -36,7 +36,6 @@
 #include "dautils.hpp"
 #include "danqs.hpp"
 #include "mputil.hpp"
-#include "rmtfile.hpp"
 #include "dadfs.hpp"
 #include "eclhelper.hpp"
 #include "seclib.hpp"

dali/base/dafdesc.cpp

@@ -26,7 +26,7 @@
 #include "jptree.hpp"
 #include "jlzw.hpp"
 #include "dafdesc.hpp"
-#include "rmtfile.hpp"
+#include "rmtclient.hpp"
 #include "dautils.hpp"
 #include "dasds.hpp"
 
@@ -3109,16 +3109,3 @@ void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file)
     }
 }
 
-static IFileDescriptor *factoryForRemoteFileDescriptor(IPropertyTree *fileInfo)
-{
-    if (fileInfo)
-        return deserializeFileDescriptorTree(fileInfo);
-    else
-        return createFileDescriptor();
-}
-
-extern da_decl FileDescriptorFactoryType queryFileDescriptorFactory()
-{
-    return factoryForRemoteFileDescriptor;
-}
-

dali/base/dafdesc.hpp

@@ -371,8 +371,5 @@ inline DFD_OS SepCharBaseOs(char c)
 
 extern da_decl void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file);
 
-typedef IFileDescriptor *(*FileDescriptorFactoryType)(IPropertyTree *);
-extern da_decl FileDescriptorFactoryType queryFileDescriptorFactory();
-
 
 #endif

dali/base/dautils.cpp

@@ -26,7 +26,6 @@
 #include "jprop.hpp"
 #include "jregexp.hpp"
 #include "jset.hpp"
-#include "rmtfile.hpp"
 
 #include "mpbase.hpp"
 #include "dautils.hpp"

dali/daliadmin/CMakeLists.txt

@@ -29,7 +29,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../server 
          ./../base 
          ./../../system/mp 
@@ -49,7 +49,7 @@ install ( TARGETS daliadmin RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries ( daliadmin  
          jlib
          mp 
-         remote 
+         dafsclient 
          dalibase 
          workunit
          dllserver

dali/datest/datest.cmake

@@ -34,6 +34,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/jlib
          ${HPCC_SOURCE_DIR}/system/security/shared
          ${HPCC_SOURCE_DIR}/esp/clients/wsdfuaccess
+         ${HPCC_SOURCE_DIR}/fs/dafsstream
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
     )
@@ -44,7 +45,8 @@ target_link_libraries ( datest
          jlib
          mp 
          hrpc 
-         remote
+         dafsclient
+         dafsstream
          eclrtl
          wsdfuaccess 
          dalibase 

dali/datest/datest.cpp

@@ -24,7 +24,6 @@
 #include "jmisc.hpp"
 #include "mpbase.hpp"
 #include "mpcomm.hpp"
-#include "sockfile.hpp"
 
 #include "daclient.hpp"
 #include "dadfs.hpp"
@@ -34,6 +33,7 @@
 #include "dautils.hpp"
 #include "dasess.hpp"
 #include "mplog.hpp"
+#include "rmtclient.hpp"
 
 #include "rtlformat.hpp"
 
@@ -2240,8 +2240,6 @@ void TestSDS1()
 
 void testDfuStreamRead(const char *fname)
 {
-    configureRemoteCreateFileDescriptorCB(queryFileDescriptorFactory());
-
     // reads a DFS file
     try
     {
@@ -2291,8 +2289,6 @@ void testDfuStreamRead(const char *fname)
 
 void testDfuStreamWrite(const char *fname)
 {
-    configureRemoteCreateFileDescriptorCB(queryFileDescriptorFactory());
-
     // reads a DFS file and writes it to <filename>_copy
     try
     {
@@ -2397,8 +2393,6 @@ void testDfuStreamWrite(const char *fname)
 
 void testDfuStreamCopy(const char *srcFileName)
 {
-    configureRemoteCreateFileDescriptorCB(queryFileDescriptorFactory());
-
     // reads a DFS file and writes it to <filename>_copy
     try
     {

dali/datest/dfuwutest.cmake

@@ -31,7 +31,7 @@ set (    SRCS
 include_directories ( 
          ./../dfu 
          ./../base 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/mp 
          . 
          ./../../system/include 
@@ -47,7 +47,7 @@ target_link_libraries ( dfuwutest
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient 
          dalibase 
          dfuwu 
          environment 

dali/daunittest/daunittest.cpp

@@ -21,7 +21,6 @@
 
 
 #include "mpcomm.hpp"
-#include "sockfile.hpp"
 
 #include "daclient.hpp"
 #include "dadfs.hpp"

dali/dfu/dfuserver.cmake

@@ -33,7 +33,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/mp 
          ./../../system/jhtree 
          ./../../rtl/eclrtl 
@@ -57,7 +57,7 @@ target_link_libraries ( dfuserver
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient 
          dalibase 
          environment 
          dllserver 

dali/dfuXRefLib/CMakeLists.txt

@@ -35,7 +35,7 @@ set (    SRCS
 
 include_directories ( 
          ./../../esp/bindings/SOAP/xpp 
-         ./../../common/remote 
+         ./../../fs/dafsclient
          ./../server 
          ./../../system/mp 
          . 
@@ -56,7 +56,7 @@ target_link_libraries ( dfuXRefLib
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient 
          dalibase 
          environment 
     )

dali/dfuplus/CMakeLists.txt

@@ -47,7 +47,8 @@ include_directories (
          ./../../system/jlib
          ./../../esp/bindings
          ./../../common/workunit
-         ./../../common/remote
+         ./../../fs/dafsclient
+         ./../../fs/dafsserver
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )
@@ -61,7 +62,8 @@ add_dependencies ( dfuplus espscm )
 install ( TARGETS dfuplus RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries ( dfuplus
          jlib
-         remote
+         dafsclient
+         dafsserver
          xmllib
          esphttp
     )

dali/dfuplus/dfuplus.cpp

@@ -30,7 +30,8 @@
 
 #ifdef DAFILESRV_LOCAL
 #include "rmtfile.hpp"
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
+#include "dafsserver.hpp"
 
 #include "jutil.hpp"
 

dali/dfuxref/CMakeLists.txt

@@ -29,7 +29,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient
          ./../server 
          ./../../system/mp 
          . 
@@ -50,7 +50,7 @@ target_link_libraries ( dfuxref
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient
          dalibase 
          environment 
          dfuXRefLib 

dali/ft/dalift.cmake

@@ -36,7 +36,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient
          ./../../system/mp 
          ./../base 
          ./../../system/include 
@@ -55,7 +55,7 @@ target_link_libraries ( dalift
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient
          dalibase 
          environment 
     )

dali/sasha/CMakeLists.txt

@@ -51,7 +51,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/mp 
          ${HPCC_SOURCE_DIR}/system/include 
          ${HPCC_SOURCE_DIR}/system/jlib 
-         ${HPCC_SOURCE_DIR}/common/remote 
+         ${HPCC_SOURCE_DIR}/fs/dafsclient 
          ${HPCC_SOURCE_DIR}/plugins/workunitservices 
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
@@ -78,7 +78,7 @@ endif ( ${HAVE_LIBCRYPT} )
 
 target_link_libraries ( saserver 
          jlib 
-         remote 
+         dafsclient 
          dalibase 
          environment 
          dllserver 
@@ -98,7 +98,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/mp 
          ${HPCC_SOURCE_DIR}/system/include 
          ${HPCC_SOURCE_DIR}/system/jlib 
-         ${HPCC_SOURCE_DIR}/common/remote 
+         ${HPCC_SOURCE_DIR}/fs/dafsclient 
     )
 
 ADD_DEFINITIONS ( -D_CONSOLE )

dali/sasha/saverify.cpp

@@ -16,7 +16,7 @@
 #include "sautil.hpp"
 #include "dautils.hpp"
 #include "rmtfile.hpp"
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
 
 //#define TESTING
 

dali/server/CMakeLists.txt

@@ -32,7 +32,7 @@ set (    SRCS
     )
 
 include_directories ( 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/mp 
          ./../base 
          ./../../system/include 
@@ -60,7 +60,7 @@ target_link_libraries ( daserver
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient 
          dalibase 
     )
 IF (USE_OPENLDAP)

ecl/ecl-bundle/CMakeLists.txt

@@ -42,7 +42,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/include
          ${HPCC_SOURCE_DIR}/system/jlib
          ${HPCC_SOURCE_DIR}/common/workunit
-         ${HPCC_SOURCE_DIR}/common/remote
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
          ${HPCC_SOURCE_DIR}/esp/clients
          ${HPCC_SOURCE_DIR}/esp/bindings
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp

ecl/eclcc/CMakeLists.txt

@@ -44,7 +44,7 @@ include_directories (
          ./../../rtl/include 
          ./../../rtl/eclrtl 
          ./../../system/jlib 
-         ./../../common/remote
+         ./../../fs/dafsclient
          ./../../system/security/shared
          ./../../system/security/zcrypt
     )
@@ -63,7 +63,7 @@ target_link_libraries ( eclcc
          nbcd 
          eclrtl 
          deftype 
-         remote 
+         dafsclient 
          dalibase 
          workunit 
          thorhelper 

ecl/hthor/CMakeLists.txt

@@ -65,6 +65,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/common/environment
          ${HPCC_SOURCE_DIR}/ecl/schedulectrl
          ${HPCC_SOURCE_DIR}/ecl/hql
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
          ${HPCC_SOURCE_DIR}/dali/ft
@@ -79,7 +80,7 @@ target_link_libraries ( hthor
          jlib
          mp
          hrpc
-         remote
+         dafsclient
          dalibase
          environment
          dllserver

ecl/hthor/hthor.cpp

@@ -32,6 +32,8 @@
 #include "jfile.hpp"
 #include "keybuild.hpp"
 
+#include "rmtclient.hpp"
+
 #include "hrpc.hpp"
 #include "hrpcsock.hpp"
 

esp/clients/wsdfuaccess/CMakeLists.txt

@@ -39,7 +39,9 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/security/shared
          ${HPCC_SOURCE_DIR}/system/security/securesocket
          ${HPCC_SOURCE_DIR}/system/security/cryptohelper
-         ${HPCC_SOURCE_DIR}/common/remote
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
+         ${HPCC_SOURCE_DIR}/fs/dafsstream
+         ${HPCC_SOURCE_DIR}/fs/dafsserver
          ${HPCC_SOURCE_DIR}/common/environment
          ${HPCC_SOURCE_DIR}/common/thorhelper
          ${HPCC_SOURCE_DIR}/rtl/include
@@ -65,9 +67,12 @@ target_link_libraries ( wsdfuaccess
          jlib
          xmllib 
          esphttp
-         remote
+         dafsclient
+         dafsserver
+         dafsstream
          thorhelper
          dalibase
+         environment
          ${CPPUNIT_LIBRARIES}
     )
 

esp/clients/wsdfuaccess/wsdfuaccess.cpp

@@ -29,8 +29,8 @@
 #include "dadfs.hpp"
 #include "dasess.hpp"
 #include "thorcommon.hpp"
-#include "sockfile.hpp"
 #include "digisign.hpp"
+#include "rmtclient.hpp"
 
 #include "eclwatch_errorlist.hpp" // only for ECLWATCH_FILE_NOT_EXIST
 #include "soapmessage.hpp"
@@ -549,7 +549,7 @@ StringBuffer &encodeDFUFileMeta(StringBuffer &metaInfoBlob, IPropertyTree *metaI
 
 #ifdef _USE_CPPUNIT
 #include "unittests.hpp"
-#include "sockfile.hpp"
+#include "dafsserver.hpp"
 #include "rmtfile.hpp"
 #include "dafscommon.hpp"
 #include "portlist.h"
@@ -639,8 +639,6 @@ protected:
     }
     void testDaFsStreaming(bool compressed, bool grouped)
     {
-        configureRemoteCreateFileDescriptorCB(queryFileDescriptorFactory());
-
         const char *thorInstance = "mythor";
         const char *groupName = thorInstance;
         const char *fname = ".::dfuaccess::testfname1";

esp/services/ws_dfu/CMakeLists.txt

@@ -63,7 +63,8 @@ include_directories (
          ./../../../system/security/cryptohelper 
          ./../../../system/include 
          ./../../../common/workunit 
-         ./../../../common/remote 
+         ./../../../fs/dafsclient 
+         ./../../../fs/dafsstream
          ./../../../common/roxiecommlib
          ./../../clients 
          ./../../../dali/base 
@@ -84,7 +85,8 @@ target_link_libraries ( ws_dfu
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient
+         dafsclient 
          nbcd 
          eclrtl 
          jhtree 

esp/services/ws_fs/CMakeLists.txt

@@ -57,7 +57,7 @@ include_directories (
          ./../../bindings 
          ./../../smc/SMCLib 
          ./../../bindings/SOAP/xpp 
-         ./../../../common/remote 
+         ./../../../fs/dafsclient 
          ./../../../common/workunit
     )
 

esp/services/ws_packageprocess/CMakeLists.txt

@@ -42,7 +42,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/common/thorhelper
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
          ${HPCC_SOURCE_DIR}/dali/dfu
-         ${HPCC_SOURCE_DIR}/common/remote
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
          ${HPCC_SOURCE_DIR}/common/workunit
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/esp/smc/SMCLib

esp/services/ws_workunits/CMakeLists.txt

@@ -53,6 +53,7 @@ include_directories (
          ./../../../dali/dfu
          ./../../../dali/sasha
          ./../../../common/remote
+         ./../../../fs/dafsclient
          ./../../../system/jlib
          ./../../../common/environment
          ./../../../rtl/eclrtl
@@ -94,6 +95,7 @@ target_link_libraries ( ws_workunits
          mp
          hrpc
          remote
+         dafsclient
          xmllib
          esphttp
          dalibase

dali/dafilesrv/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -14,9 +14,8 @@
 #    limitations under the License.
 ################################################################################
 
-
-
-include (dafilesrv.cmake)
-if ( PLATFORM )
-    include (dafscontrol.cmake)
-endif()
+HPCC_ADD_SUBDIRECTORY (dafsserver)
+HPCC_ADD_SUBDIRECTORY (dafilesrv)
+HPCC_ADD_SUBDIRECTORY (dafscontrol)
+HPCC_ADD_SUBDIRECTORY (dafsclient)
+HPCC_ADD_SUBDIRECTORY (dafsstream)

dali/dafilesrv/dafilesrv.cmake

@@ -29,12 +29,16 @@ set (    SRCS
 
 include_directories ( 
          ${HPCC_SOURCE_DIR}/system/hrpc 
-         ${HPCC_SOURCE_DIR}/common/remote 
          ${HPCC_SOURCE_DIR}/system/include 
          ${HPCC_SOURCE_DIR}/system/jhtree
          ${HPCC_SOURCE_DIR}/system/jlib 
-         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/system/mp
          ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
+         ${HPCC_SOURCE_DIR}/fs/dafsserver
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${HPCC_SOURCE_DIR}/common/environment 
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )
@@ -50,5 +54,6 @@ target_link_libraries ( dafilesrv
          jlib
          remote
          dalibase
+         dafsserver
     )
 

dali/dafilesrv/dafilesrv.cpp

@@ -25,7 +25,6 @@
 #include "jfile.hpp"
 #include "jlog.hpp"
 #include "jmisc.hpp"
-#include "rmtfile.hpp"
 #include "dalienv.hpp"
 #include "dafdesc.hpp"
 
@@ -40,7 +39,8 @@ static const char* defaultRowSericeConfiguration = "RowSvc";
 
 #include "remoteerr.hpp"
 #include "dafscommon.hpp"
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
+#include "dafsserver.hpp"
 
 void usage()
 {
@@ -341,13 +341,6 @@ int main(int argc,char **argv)
 {
     InitModuleObjects();
 
-    /* The dafilesrv hook is installed via the MODULE_INIT process
-     * but it is not wanted in dafilesrv itself, so remove it now.
-    */
-    IDaFileSrvHook *remoteHook = queryDaFileSrvHook();
-    if (remoteHook)
-        removeIFileCreateHook(remoteHook);
-
     EnableSEHtoExceptionMapping();
 #ifndef __64BIT__
     // Restrict stack sizes on 32-bit systems
@@ -516,7 +509,7 @@ int main(int argc,char **argv)
         }
         else if ((argv[i][0]=='-')&&(toupper(argv[i][1])=='T')&&(!argv[i][2]||isdigit(argv[i][2]))) {
             if (argv[i][2])
-                setDafsTrace(NULL,(byte)atoi(argv[i]+2));
+                setDaliServerTrace((byte)atoi(argv[i]+2));
             i++;
             isdaemon = false;
         }
@@ -861,7 +854,6 @@ int main(int argc,char **argv)
     server.setown(createRemoteFileServer(maxThreads, maxThreadsDelayMs, maxAsyncCopy, keyPairInfo));
     server->setThrottle(ThrottleStd, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
     server->setThrottle(ThrottleSlow, parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
-    configureRemoteCreateFileDescriptorCB(queryFileDescriptorFactory());
 
     class CPerfHook : public CSimpleInterfaceOf<IPerfMonHook>
     {

fs/dafsclient/CMakeLists.txt

@@ -0,0 +1,65 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: dafsclient 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for dafsclient
+#####################################################
+
+project( dafsclient ) 
+
+set (    SRCS 
+         rmtfile.cpp 
+         rmtclient.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/hrpc 
+         ${HPCC_SOURCE_DIR}/system/mp 
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/system/jhtree
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/testing/unittests
+         ${HPCC_SOURCE_DIR}/rtl/include
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DDAFSCLIENT_EXPORTS )
+
+HPCC_ADD_LIBRARY( dafsclient SHARED ${SRCS}  )
+install ( TARGETS dafsclient RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+target_link_libraries ( dafsclient 
+    jlib
+    jhtree 
+    hql   
+    mp
+    remote
+    )
+
+IF (USE_OPENSSL)
+    target_link_libraries ( dafsclient 
+    	securesocket
+    )
+ENDIF()

common/remote/dafscommon.hpp

@@ -18,6 +18,19 @@
 #ifndef DAFSCOMMON_HPP
 #define DAFSCOMMON_HPP
 
+#define DAFILESRV_VERSION_MAJOR 2
+#define DAFILESRV_VERSION_MINOR 5
+#define MAJORMINOR(MAJOR, MINOR) MAJOR ## MINOR
+#define DAFILESRV_VERSION_JOIN(X, Y) MAJORMINOR(X, Y)
+#define DAFILESRV_VERSION DAFILESRV_VERSION_JOIN(DAFILESRV_VERSION_MAJOR, DAFILESRV_VERSION_MINOR)
+#define TOVERSIONSTRING_JOIN(MAJOR, MINOR, OS) "DS V" #MAJOR "." #MINOR OS " "
+#define TOVERSIONSTRING(MAJOR, MINOR, OS) TOVERSIONSTRING_JOIN(MAJOR, MINOR , OS)
+#ifdef _WIN32
+#define DAFILESRV_VERSIONSTRING TOVERSIONSTRING(DAFILESRV_VERSION_MAJOR, DAFILESRV_VERSION_MINOR, "Windows")
+#else
+#define DAFILESRV_VERSIONSTRING TOVERSIONSTRING(DAFILESRV_VERSION_MAJOR, DAFILESRV_VERSION_MINOR, "Linux")
+#endif
+
 const unsigned RFEnoerror = 0;
 
 enum
@@ -99,6 +112,9 @@ enum DAFS_ERROR_CODES {
     DAFSERR_cmdstream_generalwritefailure   = -13
 };
 
+
+typedef unsigned char RemoteFileCommandType;
+
 inline MemoryBuffer & initSendBuffer(MemoryBuffer & buff)
 {
     buff.setEndian(__BIG_ENDIAN);       // transfer as big endian...
@@ -106,4 +122,27 @@ inline MemoryBuffer & initSendBuffer(MemoryBuffer & buff)
     return buff;
 }
 
+enum ThrottleClass
+{
+    ThrottleStd,
+    ThrottleSlow,
+    ThrottleClassMax
+};
+
+// RemoteFileServer throttling defaults
+#define DEFAULT_THREADLIMIT 100
+#define DEFAULT_THREADLIMITDELAYMS (60*1000)
+#define DEFAULT_ASYNCCOPYMAX 10
+
+#define DEFAULT_STDCMD_PARALLELREQUESTLIMIT 80
+#define DEFAULT_STDCMD_THROTTLEDELAYMS 1000
+#define DEFAULT_STDCMD_THROTTLECPULIMIT 85
+#define DEFAULT_STDCMD_THROTTLEQUEUELIMIT 1000
+
+#define DEFAULT_SLOWCMD_PARALLELREQUESTLIMIT 20
+#define DEFAULT_SLOWCMD_THROTTLEDELAYMS 5000
+#define DEFAULT_SLOWCMD_THROTTLECPULIMIT 75
+#define DEFAULT_SLOWCMD_THROTTLEQUEUELIMIT 1000
+
+
 #endif // DAFSCOMMON_HPP

fs/dafsclient/rmtclient.hpp

@@ -0,0 +1,100 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef RMTCLIENT_HPP
+#define RMTCLIENT_HPP
+
+#include "jsocket.hpp"
+#include "jfile.hpp"
+
+#include "remoteerr.hpp"
+#include "dafscommon.hpp"
+
+#ifdef DAFSCLIENT_EXPORTS
+#define DAFSCLIENT_API DECL_EXPORT
+#else
+#define DAFSCLIENT_API DECL_IMPORT
+#endif
+
+#define DAFILESRV_METAINFOVERSION 2
+
+#define DAFILESRV_STREAMREAD_MINVERSION 22
+#define DAFILESRV_STREAMGENERAL_MINVERSION 25
+
+typedef int RemoteFileIOHandle;
+// backward compatible modes
+typedef enum { compatIFSHnone, compatIFSHread, compatIFSHwrite, compatIFSHexec, compatIFSHall} compatIFSHmode;
+
+enum DAFS_OS
+{
+    DAFS_OSunknown,
+    DAFS_OSwindows,
+    DAFS_OSlinux,
+    DAFS_OSsolaris
+};
+
+extern DAFSCLIENT_API IFile * createRemoteFile(SocketEndpoint &ep,const char * _filename);
+
+
+interface IDaFsConnection;
+extern DAFSCLIENT_API unsigned short getDaliServixPort();  // assumed just the one for now
+extern DAFSCLIENT_API void setCanAccessDirectly(RemoteFilename & file,bool set);extern DAFSCLIENT_API unsigned getRemoteVersion(ISocket * _socket, StringBuffer &ver);
+extern DAFSCLIENT_API unsigned getCachedRemoteVersion(IDaFsConnection &daFsConnection);
+extern DAFSCLIENT_API unsigned getCachedRemoteVersion(const SocketEndpoint &ep, bool secure);
+extern DAFSCLIENT_API int setDafsTrace(ISocket * socket,byte flags);
+extern DAFSCLIENT_API int setDafsThrottleLimit(ISocket * socket, ThrottleClass throttleClass, unsigned throttleLimit, unsigned throttleDelayMs, unsigned throttleCPULimit, unsigned queueLimit, StringBuffer *errMsg=NULL);
+extern DAFSCLIENT_API bool enableDafsAuthentication(bool on);
+extern DAFSCLIENT_API int getDafsInfo(ISocket * socket, unsigned level, StringBuffer &retstr);
+extern DAFSCLIENT_API void setDafsEndpointPort(SocketEndpoint &ep);
+extern DAFSCLIENT_API void setDafsLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
+extern DAFSCLIENT_API ISocket *connectDafs(SocketEndpoint &ep, unsigned timeoutms); // NOTE: might alter ep.port if configured for multiple ports ...
+extern DAFSCLIENT_API ISocket *checkSocketSecure(ISocket *socket);
+extern DAFSCLIENT_API unsigned short getActiveDaliServixPort(const IpAddress &ip);
+extern DAFSCLIENT_API unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver);
+extern DAFSCLIENT_API unsigned getDaliServixVersion(const SocketEndpoint &ep,StringBuffer &ver);
+extern DAFSCLIENT_API DAFS_OS getDaliServixOs(const SocketEndpoint &ep);
+extern DAFSCLIENT_API bool testDaliServixPresent(const IpAddress &ip);
+
+
+
+extern DAFSCLIENT_API unsigned stopRemoteServer(ISocket *socket);
+
+
+extern DAFSCLIENT_API int setDafileSvrTraceFlags(const SocketEndpoint &ep,byte flags);
+extern DAFSCLIENT_API int setDafileSvrThrottleLimit(const SocketEndpoint &_ep, ThrottleClass throttleClass, unsigned throttleLimit, unsigned throttleDelayMs, unsigned throttleCPULimit, unsigned queueLimit, StringBuffer *errMsg=NULL);
+extern DAFSCLIENT_API int getDafileSvrInfo(const SocketEndpoint &ep, unsigned level, StringBuffer &retstr);
+
+
+extern DAFSCLIENT_API RemoteFileCommandType queryRemoteStreamCmd(); // used by testsocket only
+
+interface IDaFsConnection : extends IInterface
+{
+    virtual void close(int handle) = 0;
+    virtual void send(MemoryBuffer &sendMb, MemoryBuffer &reply) = 0;
+    virtual unsigned getVersion(StringBuffer &ver) = 0;
+    virtual const SocketEndpoint &queryEp() const = 0;
+};
+
+extern DAFSCLIENT_API IDAFS_Exception *createDafsException(int code, const char *msg);
+extern DAFSCLIENT_API IDAFS_Exception *createDafsExceptionVA(int code, const char *format, va_list args) __attribute__((format(printf,2,0)));
+extern DAFSCLIENT_API IDAFS_Exception *createDafsExceptionV(int code, const char *format, ...) __attribute__((format(printf,2,3)));
+
+extern DAFSCLIENT_API IDaFsConnection *createDaFsConnection(const SocketEndpoint &ep, DAFSConnectCfg connectMethod, const char *tracing);
+
+extern DAFSCLIENT_API unsigned stopRemoteServer(ISocket * _socket);
+
+#endif

fs/dafsclient/rmtclient_impl.hpp

@@ -0,0 +1,210 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef RMTCLIENT_IMPL_HPP
+#define RMTCLIENT_IMPL_HPP
+
+#include "rmtclient.hpp"
+
+#ifdef _DEBUG
+//#define SIMULATE_PACKETLOSS 1
+#endif
+
+#if SIMULATE_PACKETLOSS
+
+#define TESTING_FAILURE_RATE_LOST_SEND  10 // per 1000
+#define TESTING_FAILURE_RATE_LOST_RECV  10 // per 1000
+#define DUMMY_TIMEOUT_MAX (1000*10)
+
+
+struct DAFSCLIENT_API dummyReadWrite
+{
+    static ISocket *timeoutreadsock = NULL; // used to trigger
+    class X
+    {
+        dummyReadWrite *parent;
+    public:
+        X(dummyReadWrite *_parent)
+        {
+            parent = _parent;
+        }
+        ~X()
+        {
+            delete parent;
+        }
+    };
+
+    class TimeoutSocketException: public CInterface, public IJSOCK_Exception
+    {
+    public:
+        IMPLEMENT_IINTERFACE;
+
+        TimeoutSocketException()
+        {
+        }
+
+        virtual ~TimeoutSocketException()
+        {
+        }
+
+        int errorCode() const { return JSOCKERR_timeout_expired; }
+        StringBuffer &  errorMessage(StringBuffer &str) const
+        {
+            return str.append("timeout expired");
+        }
+        MessageAudience errorAudience() const
+        {
+            return MSGAUD_user;
+        }
+    };
+
+
+    ISocket *sock;
+
+    dummyReadWrite(ISocket *_sock)
+    {
+        sock = _sock;
+    }
+
+    void readtms(void* buf, size32_t min_size, size32_t max_size, size32_t &size_read, time_t timeout)
+    {
+        X x(this);
+        unsigned t = msTick();
+        unsigned r = getRandom();
+        bool timeoutread = (timeoutreadsock==sock);
+        timeoutreadsock=NULL;
+        if (!timeoutread)
+            sock->readtms(buf, min_size, max_size, size_read, timeout);
+        if (timeoutread||((TESTING_FAILURE_RATE_LOST_RECV>0)&&(r%1000<TESTING_FAILURE_RATE_LOST_RECV))) {
+            PrintStackReport();
+            if (timeoutread)
+                PROGLOG("** Simulate timeout");
+            else
+                PROGLOG("** Simulate Packet loss (size %d,%d)",min_size,max_size);
+            if (timeout>DUMMY_TIMEOUT_MAX)
+                timeout = DUMMY_TIMEOUT_MAX;
+            t = msTick()-t;
+            if (t<timeout)
+                Sleep(timeout-t);
+            IJSOCK_Exception *e = new TimeoutSocketException;
+            throw e;
+        }
+    }
+    size32_t write(void const* buf, size32_t size)
+    {
+        X x(this);
+        timeoutreadsock=NULL;
+        unsigned r = getRandom();
+        if ((TESTING_FAILURE_RATE_LOST_SEND>0)&&(r%1000<TESTING_FAILURE_RATE_LOST_SEND)) {
+            PrintStackReport();
+            PROGLOG("** Simulate Packet loss (size %d)",size);
+            timeoutreadsock=sock;
+            return size;
+        }
+        return sock->write(buf,size);
+    }
+};
+
+#define SOCKWRITE(sock) (new dummyReadWrite(sock))->write
+#define SOCKREADTMS(sock) (new dummyReadWrite(sock))->readtms
+#else
+#define SOCKWRITE(sock) sock->write
+#define SOCKREADTMS(sock) sock->readtms
+#endif
+
+#ifdef SIMULATE_PACKETLOSS
+#define NORMAL_RETRIES      (1)
+#define LENGTHY_RETRIES     (1)
+#else
+#define NORMAL_RETRIES      (3)
+#define LENGTHY_RETRIES     (12)
+#endif
+extern DAFSCLIENT_API void sendDaFsBuffer(ISocket * socket, MemoryBuffer & src, bool testSocketFlag=false);
+extern DAFSCLIENT_API size32_t receiveDaFsBufferSize(ISocket * socket, unsigned numtries=NORMAL_RETRIES,CTimeMon *timemon=NULL);
+extern DAFSCLIENT_API void receiveDaFsBuffer(ISocket * socket, MemoryBuffer & tgt, unsigned numtries=1, size32_t maxsz=0x7fffffff);
+extern DAFSCLIENT_API void cleanupDaFsSocket(ISocket *sock);
+extern DAFSCLIENT_API byte traceFlags;
+extern DAFSCLIENT_API bool AuthenticationEnabled;
+#define TF_TRACE (traceFlags&1)
+#define TF_TRACE_PRE_IO (traceFlags&2)
+#define TF_TRACE_FULL (traceFlags&4)
+#define TF_TRACE_CLIENT_CONN (traceFlags&8)
+#define TF_TRACE_TREE_COPY (traceFlags&0x10)
+#define TF_TRACE_CLIENT_STATS (traceFlags&0x20)
+
+
+class CRemoteBase : public CSimpleInterfaceOf<IDaFsConnection>
+{
+    Owned<ISocket>          socket;
+    static  SocketEndpoint  lastfailep;
+    static unsigned         lastfailtime;
+    DAFSConnectCfg          connectMethod;
+
+    void connectSocket(SocketEndpoint &ep, unsigned localConnectTime=0, unsigned localRetries=0);
+    void killSocket(SocketEndpoint &tep);
+
+protected: friend class CRemoteFileIO;
+
+    StringAttr          filename;
+    CriticalSection     crit;
+    SocketEndpoint      ep;
+
+    void sendRemoteCommand(MemoryBuffer & src, MemoryBuffer & reply, bool retry=true, bool lengthy=false, bool handleErrCode=true);
+    void sendRemoteCommand(MemoryBuffer & src, bool retry);
+    void throwUnauthenticated(const IpAddress &ip,const char *user,unsigned err=0);
+    void sendAuthentication(const IpAddress &serverip);
+public:
+    CRemoteBase(const SocketEndpoint &_ep, const char * _filename);
+    CRemoteBase(const SocketEndpoint &_ep, DAFSConnectCfg _connectMethod, const char * _filename);
+    void disconnect();
+    const char *queryLocalName()
+    {
+        return filename;
+    }
+// IDaFsConnection impl.
+    virtual void close(int handle) override;
+    virtual void send(MemoryBuffer &sendMb, MemoryBuffer &reply) override;
+    virtual unsigned getVersion(StringBuffer &ver) override;
+    virtual const SocketEndpoint &queryEp() const override;
+};
+
+typedef enum { ACScontinue, ACSdone, ACSerror} AsyncCommandStatus;
+
+
+extern void clientSetDaliServixSocketCaching(bool set);
+extern void clientDisconnectRemoteFile(IFile *file);
+extern void clientDisconnectRemoteIoOnExit(IFileIO *fileio,bool set);
+
+extern bool clientResetFilename(IFile *file, const char *newname); // returns false if not remote
+
+extern bool clientAsyncCopyFileSection(const char *uuid,    // from genUUID - must be same for subsequent calls
+                        IFile *from,                        // expected to be remote
+                        RemoteFilename &to,
+                        offset_t toofs,                     // (offset_t)-1 created file and copies to start
+                        offset_t fromofs,
+                        offset_t size,                      // (offset_t)-1 for all file
+                        ICopyFileProgress *progress,
+                        unsigned timeout                    // 0 to start, non-zero to wait
+                        ); // returns true when done
+
+extern void clientSetRemoteFileTimeouts(unsigned maxconnecttime, unsigned maxreadtime);
+extern void clientAddSocketToCache(SocketEndpoint &ep, ISocket *socket);
+
+
+
+#endif //

fs/dafsclient/rmtfile.hpp

@@ -0,0 +1,121 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef RMTFILE_HPP
+#define RMTFILE_HPP
+
+#include "jsocket.hpp"
+#include "jfile.hpp"
+
+#include "dafscommon.hpp"
+#include "rmtclient.hpp"
+
+#ifdef DAFSCLIENT_EXPORTS
+#define DAFSCLIENT_API DECL_EXPORT
+#else
+#define DAFSCLIENT_API DECL_IMPORT
+#endif
+
+extern DAFSCLIENT_API void filenameToUrl(StringBuffer & out, const char * filename);
+
+interface IDaFileSrvHook : extends IRemoteFileCreateHook
+{
+    virtual void forceRemote(const char *pattern) = 0; // NB: forces all local files matching pattern to be remote reads
+    virtual void addSubnetFilter(const char *subnet, const char *mask, const char *dir, const char *sourceRange, bool trace) = 0;
+    virtual void addRangeFilter(const char *range, const char *dir, const char *sourceRange, bool trace) = 0;
+    virtual IPropertyTree *addFilters(IPropertyTree *filters, const SocketEndpoint *ipAddress) = 0;
+    virtual IPropertyTree *addMyFilters(IPropertyTree *filters, SocketEndpoint *myEp=NULL) = 0;
+    virtual void clearFilters() = 0;
+};
+extern DAFSCLIENT_API IDaFileSrvHook *queryDaFileSrvHook();
+
+extern DAFSCLIENT_API void setDaliServixSocketCaching(bool set);
+extern DAFSCLIENT_API bool canAccessDirectly(const RemoteFilename & file);
+extern DAFSCLIENT_API IFile *createDaliServixFile(const RemoteFilename & file);
+extern DAFSCLIENT_API void enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
+extern DAFSCLIENT_API bool testForceRemote(const char *path); // return true if forceRemote setup/pattern will make this path a remote read.
+
+extern DAFSCLIENT_API IDirectoryIterator *createRemoteDirectorIterator(const SocketEndpoint &ep, const char *name, MemoryBuffer &state);
+extern DAFSCLIENT_API bool serializeRemoteDirectoryIterator(MemoryBuffer &tgt, IDirectoryIterator *iter, size32_t bufsize, bool first);
+extern DAFSCLIENT_API void serializeRemoteDirectoryDiff(MemoryBuffer &tgt, IDirectoryDifferenceIterator *iter);
+
+extern DAFSCLIENT_API void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
+// redirects a daliservix file to a local mount. To remove redirect use NULL for mount dir or NULL for dir
+
+
+extern DAFSCLIENT_API void remoteExtractBlobElements(const char * prefix, const RemoteFilename &file, ExtractedBlobArray & extracted);
+
+
+extern DAFSCLIENT_API void disconnectRemoteFile(IFile *file);
+extern DAFSCLIENT_API void disconnectRemoteIoOnExit(IFileIO *fileio,bool set=true);
+
+extern DAFSCLIENT_API bool resetRemoteFilename(IFile *file, const char *newname); // returns false if not remote
+
+
+extern DAFSCLIENT_API void enableAuthentication(bool set=true); // default enabled for clients, disabled for server
+
+extern DAFSCLIENT_API bool asyncCopyFileSection(const char *uuid,   // from genUUID - must be same for subsequent calls
+                            IFile *from,                        // expected to be remote
+                            RemoteFilename &to,
+                            offset_t toofs,                     // (offset_t)-1 created file and copies to start
+                            offset_t fromofs,
+                            offset_t size,                      // (offset_t)-1 for all file
+                            ICopyFileProgress *progress,
+                            unsigned timeout                    // 0 to start, non-zero to wait
+                        ); // returns true when done
+
+extern DAFSCLIENT_API void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime);
+
+#define DAFS_VALIDATE_CONNECT_FAIL  (0x01)
+#define DAFS_VALIDATE_BAD_VERSION   (0x02)
+#define DAFS_VALIDATE_WRITE_FAIL_DATA  (0x12)
+#define DAFS_VALIDATE_READ_FAIL_DATA   (0x14)
+#define DAFS_VALIDATE_DISK_FULL_DATA   (0x18)
+#define DAFS_VALIDATE_WRITE_FAIL_MIRROR  (0x22)
+#define DAFS_VALIDATE_READ_FAIL_MIRROR   (0x24)
+#define DAFS_VALIDATE_DISK_FULL_MIRROR   (0x28)
+#define DAFS_SCRIPT_FAIL            (0x40)
+                                
+extern DAFSCLIENT_API unsigned validateNodes(const SocketEndpointArray &eps,const char *dataDir, const char *mirrorDir, bool chkver, SocketEndpointArray &failures, UnsignedArray &failedcodes, StringArray &failedmessages, const char *filename=NULL);
+
+extern DAFSCLIENT_API void installFileHooks(const char *filespec);
+extern DAFSCLIENT_API void removeFileHooks(); // Should be called before closedown
+
+extern DAFSCLIENT_API void setRemoteOutputCompressionDefault(const char *type);
+extern DAFSCLIENT_API const char *queryOutputCompressionDefault();
+
+extern DAFSCLIENT_API void remoteExtractBlobElements(const SocketEndpoint &ep, const char * prefix, const char * filename, ExtractedBlobArray & extracted);
+
+//// legacy implementations of the streaming support (to be replaced by dafsstream.*)
+
+interface IOutputMetaData;
+class RowFilter;
+interface IRemoteFileIO : extends IFileIO
+{
+    virtual void addVirtualFieldMapping(const char *fieldName, const char *fieldValue) = 0;
+    virtual void ensureAvailable() = 0;
+};
+extern DAFSCLIENT_API IRemoteFileIO *createRemoteFilteredFile(SocketEndpoint &ep, const char * filename, IOutputMetaData *actual, IOutputMetaData *projected, const RowFilter &fieldFilters, bool compressed, bool grouped, unsigned __int64 chooseNLimit);
+
+interface IIndexLookup;
+extern DAFSCLIENT_API IIndexLookup *createRemoteFilteredKey(SocketEndpoint &ep, const char * filename, unsigned crc, IOutputMetaData *actual, IOutputMetaData *projected, const RowFilter &fieldFilters, unsigned __int64 chooseNLimit);
+
+////
+
+
+
+#endif

fs/dafsclient/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>fs/dafsclient</title>
+
+    <para>
+        The fs/dafsclient directory contains the sources for the fs/dafsclient library.
+    </para>
+</section>

dali/dafilesrv/dafscontrol.cmake

@@ -34,8 +34,10 @@ include_directories (
          ./../../common/remote 
          ./../../system/mp 
          ./../../system/include 
+         ./../../system/security/shared
          ./../../dali/base 
          ./../../system/jlib 
+         ./../../fs/dafsclient 
          ./../../common/environment 
     )
 
@@ -46,7 +48,7 @@ target_link_libraries ( dafscontrol
          jlib
          mp 
          hrpc 
-         remote 
+         dafsclient 
          dalibase 
          environment 
     )

dali/dafilesrv/dafscontrol.cpp

@@ -28,7 +28,7 @@
 #include "dafdesc.hpp"
 #include "dasds.hpp"
 #include "rmtfile.hpp"
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
 
 #include "dalienv.hpp"
 
@@ -172,7 +172,7 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali
                 case AMcheckvermajor: {
                     // compares versions up to the '-'
                         const char *rv = verstr.str();
-                        const char *v = remoteServerVersionString();
+                        const char *v = DAFILESRV_VERSIONSTRING;
                         if (mode!=AMcheckvermajor) {
                             while (*v&&(*v!='-')&&(*v==*rv)) {
                                 v++;
@@ -232,10 +232,10 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali
             }
             unsigned numok = eps.ordinality()-result.ordinality();
             if (mode==AMcheckvermajor)
-                PROGLOG("%s: %d node%s running version %.1f of DAFILESRV",grpip,numok,(numok!=1)?"s":"",((double)FILESRV_VERSION)/10.0);
+                PROGLOG("%s: %d node%s running version %.1f of DAFILESRV",grpip,numok,(numok!=1)?"s":"",((double)DAFILESRV_VERSION)/10.0);
             else {
                 StringBuffer vs;
-                const char *v = remoteServerVersionString();
+                const char *v = DAFILESRV_VERSIONSTRING;
                 while (*v&&(*v!='-'))
                     vs.append(*(v++));
                 PROGLOG("%s: %d node%s running version %s of DAFILESRV",grpip,numok,(numok!=1)?"s":"",vs.str());
@@ -307,7 +307,7 @@ int main(int argc, char* argv[])
                 break;
             }
             if (stricmp(argv[ai],"myver")==0) {
-                const char *v = remoteServerVersionString();
+                const char *v = DAFILESRV_VERSIONSTRING;
                 StringBuffer vs;
                 if (memicmp(v,"DS V",4)==0)
                     v += 4;
@@ -323,21 +323,21 @@ int main(int argc, char* argv[])
                 if (ai+1>=ac) 
                     usage(); 
                 else
-                    applyNodes(argv[ai+1], AMstop, FILESRV_VERSION,isdali,quiet);
+                    applyNodes(argv[ai+1], AMstop, DAFILESRV_VERSION,isdali,quiet);
                 break;
             }
             if (stricmp(argv[ai],"stopver")==0) {
                 if (ai+1>=ac) 
                     usage(); 
                 else
-                    applyNodes(argv[ai+1], AMstopver, FILESRV_VERSION,isdali,quiet);
+                    applyNodes(argv[ai+1], AMstopver, DAFILESRV_VERSION,isdali,quiet);
                 break;
             }
             if (stricmp(argv[ai],"check")==0) {
                 if (ai+1>=ac) 
                     usage(); 
                 else
-                    if (applyNodes(argv[ai+1], AMcheck, FILESRV_VERSION,isdali,quiet)>0)
+                    if (applyNodes(argv[ai+1], AMcheck, DAFILESRV_VERSION,isdali,quiet)>0)
                         ret = 1;
                 break;
             }
@@ -345,7 +345,7 @@ int main(int argc, char* argv[])
                 if (ai+1>=ac) 
                     usage();
                 else
-                    if (applyNodes(argv[ai+1], AMcheckver, FILESRV_VERSION,isdali,quiet)>0)
+                    if (applyNodes(argv[ai+1], AMcheckver, DAFILESRV_VERSION,isdali,quiet)>0)
                         ret = 1;
                 break;
             }
@@ -353,7 +353,7 @@ int main(int argc, char* argv[])
                 if (ai+1>=ac) 
                     usage();
                 else
-                    if (applyNodes(argv[ai+1], AMcheckvermajor, FILESRV_VERSION,isdali,quiet)>0)
+                    if (applyNodes(argv[ai+1], AMcheckvermajor, DAFILESRV_VERSION,isdali,quiet)>0)
                         ret = 1;
                 break;
             }
@@ -361,7 +361,7 @@ int main(int argc, char* argv[])
                 if (ai+1>=ac) 
                     usage(); 
                 else
-                    applyNodes(argv[ai+1], AMver, FILESRV_VERSION,isdali,quiet);
+                    applyNodes(argv[ai+1], AMver, DAFILESRV_VERSION,isdali,quiet);
                 break;
             }
             if (stricmp(argv[ai],"trace")==0) {

fs/dafscontrol/sourcedoc.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>fs/dafscontrol</title>
+
+    <para>
+        The fs/dafscontrol directory contains the sources for the dafscontrol utility
+        that is used to interfact and control dafilesrv
+    </para>
+</section>

fs/dafsserver/CMakeLists.txt

@@ -0,0 +1,71 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: dafsserver 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for dafs
+#####################################################
+
+project( dafsserver ) 
+
+set (    SRCS 
+         dafsserver.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/system/jhtree
+         ${HPCC_SOURCE_DIR}/system/mp
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/rtl/include
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/ecl/hql
+         ${HPCC_SOURCE_DIR}/common/deftype
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${HPCC_SOURCE_DIR}/testing/unittests
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DDAFSSERVER_EXPORTS )
+
+HPCC_ADD_LIBRARY( dafsserver SHARED ${SRCS}  )
+install ( TARGETS dafsserver RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+target_link_libraries ( dafsserver 
+    mp
+    eclrtl
+    jlib
+    jhtree 
+    hql   
+    dafsclient
+    dalibase
+    ${CPPUNIT_LIBRARIES}
+    )
+
+IF (USE_OPENSSL)
+    target_link_libraries ( dafsserver 
+    	securesocket
+    )
+ENDIF()

fs/dafsserver/dafsserver.hpp

@@ -0,0 +1,88 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef SOCKFILE_HPP
+#define SOCKFILE_HPP
+
+#include "jsocket.hpp"
+
+#ifdef DAFSSERVER_EXPORTS
+#define DAFSSERVER_API DECL_EXPORT
+#else
+#define DAFSSERVER_API DECL_IMPORT
+#endif
+
+#define DAFILESRV_METAINFOVERSION 2
+
+#define DAFILESRV_STREAMREAD_MINVERSION 22
+#define DAFILESRV_STREAMGENERAL_MINVERSION 25
+
+// RemoteFileServer throttling defaults
+#define DEFAULT_THREADLIMIT 100
+#define DEFAULT_THREADLIMITDELAYMS (60*1000)
+#define DEFAULT_ASYNCCOPYMAX 10
+
+#define DEFAULT_STDCMD_PARALLELREQUESTLIMIT 80
+#define DEFAULT_STDCMD_THROTTLEDELAYMS 1000
+#define DEFAULT_STDCMD_THROTTLECPULIMIT 85
+#define DEFAULT_STDCMD_THROTTLEQUEUELIMIT 1000
+
+#define DEFAULT_SLOWCMD_PARALLELREQUESTLIMIT 20
+#define DEFAULT_SLOWCMD_THROTTLEDELAYMS 5000
+#define DEFAULT_SLOWCMD_THROTTLECPULIMIT 75
+#define DEFAULT_SLOWCMD_THROTTLEQUEUELIMIT 1000
+
+
+enum RowServiceCfg
+{
+    rs_off,     // No dedicated row service, allows row service commands on std. dafilesrv port.
+    rs_on,      // Dedicated row service on own port accepting authorized signed connections only. Row service commands on std. dafilersv port will be refused.
+    rs_both,    // Dedicated row service on own port accepting authorized signed connections only. Still accepts unsigned connection on std. dafilesrv port
+    rs_onssl,   // Same as rs_on, but SSL
+    rs_bothssl  // Same as rs_only, but SSL
+};
+
+interface IRemoteFileServer : extends IInterface
+{
+    virtual void run(DAFSConnectCfg connectMethod, const SocketEndpoint &listenep, unsigned sslPort=0, const SocketEndpoint *rowServiceEp=nullptr, bool rowServiceSSL=false, bool rowServiceOnStdPort=true) = 0;
+    virtual void run(DAFSConnectCfg _connectMethod, ISocket *listenSocket, ISocket *secureSocket, ISocket *rowServiceSocket) = 0;
+    virtual void stop() = 0;
+    virtual unsigned idleTime() = 0; // in ms
+    virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;
+    virtual StringBuffer &getStats(StringBuffer &stats, bool reset) = 0;
+};
+
+interface IRemoteRowServer : extends IInterface
+{
+    virtual void run(unsigned port=0) = 0;
+    virtual void stop() = 0;
+    virtual unsigned idleTime() = 0; // in ms
+    virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;
+    virtual StringBuffer &getStats(StringBuffer &stats, bool reset) = 0;
+};
+
+interface IKeyManager;
+interface IDelayedFile;
+interface IDaFsConnection;
+
+extern DAFSSERVER_API const char *remoteServerVersionString();
+extern DAFSSERVER_API IRemoteFileServer * createRemoteFileServer(unsigned maxThreads=DEFAULT_THREADLIMIT, unsigned maxThreadsDelayMs=DEFAULT_THREADLIMITDELAYMS, unsigned maxAsyncCopy=DEFAULT_ASYNCCOPYMAX, IPropertyTree *keyPairInfo=nullptr);
+extern DAFSSERVER_API int setDaliServerTrace(byte flags);
+extern DAFSSERVER_API bool enableDaliServerAuthentication(bool on);
+
+
+#endif

fs/dafsserver/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>fs/dafsserver</title>
+
+    <para>
+        The fs/dafsserver directory contains the sources for the fs/dafsserver library.
+    </para>
+</section>

fs/dafsstream/CMakeLists.txt

@@ -0,0 +1,66 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: dafsstream 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for dafsstream
+#####################################################
+
+project( dafsstream ) 
+
+set (    SRCS 
+         dafsstream.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/hrpc 
+         ${HPCC_SOURCE_DIR}/system/mp 
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/system/jhtree
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/testing/unittests
+         ${HPCC_SOURCE_DIR}/rtl/include
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/ecl/hql
+         ${HPCC_SOURCE_DIR}/common/deftype
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DDAFSCLIENT_EXPORTS )
+
+HPCC_ADD_LIBRARY( dafsstream SHARED ${SRCS}  )
+install ( TARGETS dafsstream RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+target_link_libraries ( dafsstream 
+    eclrtl
+    jlib
+    jhtree 
+    hql   
+    mp
+    dalibase
+    remote
+    dafsclient
+    )

common/remote/dafsstream.cpp

@@ -37,7 +37,8 @@
 #include "rtldynfield.hpp"
 #include "eclhelper_dyn.hpp"
 
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
+
 #include "dafscommon.hpp"
 #include "dafsstream.hpp"
 
@@ -188,7 +189,7 @@ public:
                 ClusterPartDiskMapSpec mspec;
                 mspec.defaultCopies = replicated?DFD_DefaultCopies:DFD_NoCopies;
 
-                fileDesc.setown(queryRemoteCreateFileDescriptorCB()(nullptr));
+                fileDesc.setown(createFileDescriptor());
                 fileDesc->setDefaultDir(dir.str());
                 fileDesc->setNumParts(numParts);
                 fileDesc->setPartMask(partMask);
@@ -197,7 +198,7 @@ public:
             }
             case 2: // serialized compact IFileDescriptor
             {
-                fileDesc.setown(queryRemoteCreateFileDescriptorCB()(fileInfo));
+                fileDesc.setown(deserializeFileDescriptorTree(fileInfo));
                 break;
             }
         }

common/remote/dafsstream.hpp

@@ -18,14 +18,15 @@
 #ifndef DAFSCLIENT_HPP
 #define DAFSCLIENT_HPP
 
-#ifdef REMOTE_EXPORTS
-#define REMOTE_API DECL_EXPORT
+#ifdef DAFSCLIENT_EXPORTS
+#define DAFSCLIENT_API DECL_EXPORT
 #else
-#define REMOTE_API DECL_IMPORT
+#define DAFSCLIENT_API DECL_IMPORT
 #endif
 
 #include "seclib.hpp"
 
+
 interface IOutputMetaData;
 namespace dafsstream
 {
@@ -51,14 +52,14 @@ interface IDaFsException : extends IException
 {
 };
 
-interface REMOTE_API IDFUFilePartBase : extends IInterface
+interface DAFSCLIENT_API IDFUFilePartBase : extends IInterface
 {
     virtual void start() = 0;
     virtual void finalize() = 0;
     virtual IOutputMetaData *queryMeta() const = 0;
 };
 
-interface REMOTE_API IDFUFilePartReader : extends IDFUFilePartBase
+interface DAFSCLIENT_API IDFUFilePartReader : extends IDFUFilePartBase
 {
     virtual const void *nextRow(size32_t &sz) = 0;
     virtual const void *getRows(size32_t min, size32_t &got) = 0; // will read at least min, and returned data will contain whole rows only
@@ -70,13 +71,13 @@ interface REMOTE_API IDFUFilePartReader : extends IDFUFilePartBase
     virtual void addVirtualFieldMapping(const char *fieldName, const char *fieldValue) = 0;
 };
 
-interface REMOTE_API IDFUFilePartWriter : extends IDFUFilePartBase
+interface DAFSCLIENT_API IDFUFilePartWriter : extends IDFUFilePartBase
 {
     virtual void write(size32_t sz, const void *rowData) = 0; // NB: can be multiple rows
 };
 
 
-interface REMOTE_API IDFUFileAccessExt : extends IInterface
+interface DAFSCLIENT_API IDFUFileAccessExt : extends IInterface
 {
     virtual IOutputMetaData *queryMeta() const = 0;
     virtual IFileDescriptor &queryFileDescriptor() const = 0;
@@ -85,7 +86,7 @@ interface REMOTE_API IDFUFileAccessExt : extends IInterface
 };
 
 enum DFUFileOption { dfo_null=0, dfo_compressedRemoteStreams=1 }; // NB: will be used in bit field
-interface REMOTE_API IDFUFileAccess : extends IInterface
+interface DAFSCLIENT_API IDFUFileAccess : extends IInterface
 {
     virtual const char *queryName() const = 0;
     virtual const char *queryFileId() const = 0;
@@ -129,10 +130,10 @@ interface REMOTE_API IDFUFileAccess : extends IInterface
 };
 
 // NB: fileId, supplied/only needed by older esp's at publish time
-REMOTE_API IDFUFileAccess *createDFUFileAccess(const char *metaInfoBlobB64, const char *fileId=nullptr);
-REMOTE_API IRowWriter *createRowWriter(IDFUFilePartWriter *partWriter);
+DAFSCLIENT_API IDFUFileAccess *createDFUFileAccess(const char *metaInfoBlobB64, const char *fileId=nullptr);
+DAFSCLIENT_API IRowWriter *createRowWriter(IDFUFilePartWriter *partWriter);
 
-REMOTE_API void setDefaultCommCompression(const char *compType);
+DAFSCLIENT_API void setDefaultCommCompression(const char *compType);
 
 } // end of namespace dafsstream
 

fs/dafsstream/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>fs/dafsstream</title>
+
+    <para>
+        The fs/dafsstream directory contains the sources for the fs/dafsstream library.
+    </para>
+</section>

fs/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>fs</title>
+
+    <para>
+        The fs directory contains the subdirectories for dafilsrv related components and libraries
+    </para>
+</section>

roxie/ccd/CMakeLists.txt

@@ -63,7 +63,7 @@ set (   SRCS
 
 include_directories ( 
          .
-         ${HPCC_SOURCE_DIR}/common/remote
+         ${HPCC_SOURCE_DIR}/fs/dafsclient
          ${HPCC_SOURCE_DIR}/system/jhtree
          ${HPCC_SOURCE_DIR}/system/mp
          ${HPCC_SOURCE_DIR}/common/workunit
@@ -106,7 +106,7 @@ target_link_libraries ( ccd
          nbcd
          roxiemem 
          udplib 
-         remote 
+         dafsclient 
          eclrtl 
          dalibase 
          deftype 

system/include/modinit.h

@@ -64,7 +64,7 @@
 #define INIT_PRIORITY_DEFVALUE                  INIT_PRIORITY_STANDARD+0x300        // before hqlinternal
 #define INIT_PRIORITY_ENV_DALIENV               INIT_PRIORITY_STANDARD+100
 #define INIT_PRIORITY_ENV_ENVIRONMENT           INIT_PRIORITY_STANDARD+150
-#define INIT_PRIORITY_REMOTE_RMTFILE            INIT_PRIORITY_STANDARD+100
+#define INIT_PRIORITY_DAFSCLIENT_RMTFILE        INIT_PRIORITY_STANDARD+100
 #define INIT_PRIORITY_JHTREE_JHTREE             INIT_PRIORITY_STANDARD+150
 #define INIT_PRIORITY_ECLRTL_ECLRTL         INIT_PRIORITY_STANDARD+0x100
 #define INIT_PRIORITY_MP_MPTAG                  INIT_PRIORITY_STANDARD+100

testing/unittests/CMakeLists.txt

@@ -45,7 +45,7 @@ include_directories (
          ./../../system/jlib
          ./../../system/mp
          ./../../system/libbase58
-         ./../../common/remote
+         ./../../fs/dafsclient
          ./../../common/thorhelper
          ./../../dali/base
          ./../../system/security/shared
@@ -64,7 +64,7 @@ HPCC_ADD_EXECUTABLE ( unittests ${SRCS} )
 install ( TARGETS unittests RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries ( unittests
          jlib
-         remote
+         dafsclient
          dalibase
          deftype
          libbase58

testing/unittests/jlibtests.cpp

@@ -26,7 +26,7 @@
 #include "jfile.hpp"
 #include "jdebug.hpp"
 #include "jset.hpp"
-#include "sockfile.hpp"
+#include "rmtfile.hpp"
 #include "jqueue.hpp"
 #include "jregexp.hpp"
 

thorlcr/activities/activitymasters_lcr.cmake

@@ -75,7 +75,7 @@ include_directories (
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
          ./../thorutil 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/jhtree 
          ./../../system/mp 
          ./../master 
@@ -106,7 +106,7 @@ set_target_properties(activitymasters_lcr PROPERTIES
 install ( TARGETS activitymasters_lcr RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
 target_link_libraries ( activitymasters_lcr
          jlib
-         remote 
+         dafsclient 
          hql
          thorsort_lcr 
          jhtree 

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -31,7 +31,7 @@
 #include "eclhelper.hpp" // tmp for IHThor..Arg interfaces.
 
 #include "rmtfile.hpp"
-#include "sockfile.hpp"
+#include "rmtclient.hpp"
 
 #include "thormisc.hpp"
 #include "thmfilemanager.hpp"

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -19,11 +19,11 @@
 #include "jfile.hpp"
 #include "jtime.hpp"
 #include "jsort.hpp"
-#include "sockfile.hpp"
 
 #include "rtlkey.hpp"
 #include "jhtree.hpp"
 #include "rmtfile.hpp"
+#include "rmtclient.hpp"
 
 #include "thorstep.ipp"
 

thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp

@@ -31,8 +31,6 @@
 
 #include "jhtree.hpp"
 
-#include "sockfile.hpp"
-
 #include "thorxmlwrite.hpp"
 
 #include "thorport.hpp"

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -35,8 +35,6 @@
 
 #include "rtlcommon.hpp"
 
-#include "sockfile.hpp"
-
 #include "thorxmlwrite.hpp"
 
 #include "thorport.hpp"

thorlcr/graph/graphslave_lcr.cmake

@@ -29,7 +29,7 @@ set (    SRCS
 
 include_directories ( 
          ./../thorutil 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/jhtree 
          ./../../system/mp 
          ./../../common/workunit 
@@ -57,7 +57,7 @@ target_link_libraries ( graphslave_lcr
          eclrtl 
          deftype 
          thorhelper 
-         remote 
+         dafsclient 
          dalibase 
          environment 
          dllserver 

thorlcr/graph/thgraphslave.cpp

@@ -18,17 +18,18 @@
 #include "jlib.hpp"
 #include "jlzw.hpp"
 #include "jhtree.hpp"
+#include "rmtfile.hpp"
 #include "daclient.hpp"
 #include "commonext.hpp"
 #include "thorplugin.hpp"
 #include "thcodectx.hpp"
 #include "thmem.hpp"
 #include "thorport.hpp"
-#include "sockfile.hpp"
 #include "slwatchdog.hpp"
 #include "thgraphslave.hpp"
 #include "thcompressutil.hpp"
 #include "enginecontext.hpp"
+#include "rmtclient.hpp"
 
 //////////////////////////////////
 

thorlcr/master/CMakeLists.txt

@@ -32,7 +32,7 @@ set (    SRCS
 
 include_directories ( 
          ./../thorutil 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/jhtree 
          ./../../system/mp 
          ./../../common/workunit 
@@ -70,7 +70,7 @@ install ( TARGETS thormaster_lcr RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries (  thormaster_lcr 
          jlib
          jhtree 
-         remote 
+         dafsclient 
          dalibase 
          environment 
          dllserver 

thorlcr/slave/CMakeLists.txt

@@ -32,7 +32,7 @@ set (    SRCS
 
 include_directories ( 
          ./../thorutil 
-         ./../../common/remote 
+         ./../../fs/dafsclient 
          ./../../system/jhtree 
          ./../../system/mp 
          ./../../common/workunit 
@@ -66,7 +66,7 @@ HPCC_ADD_EXECUTABLE ( thorslave_lcr ${SRCS} )
 install ( TARGETS thorslave_lcr RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries (  thorslave_lcr 
          jlib
-         remote 
+         dafsclient 
          dalibase 
          environment 
          dllserver 

tools/backupnode/CMakeLists.txt

@@ -32,7 +32,7 @@ include_directories (
          ./../../system/include 
          ./../../system/jlib 
          ./../../system/mp
-         ./../../common/remote
+         ./../../fs/dafsclient
          ./../../dali/base
          ./../../system/jhtree
          ./../../rtl/include
@@ -48,7 +48,7 @@ install ( TARGETS backupnode RUNTIME DESTINATION ${EXEC_DIR} )
 target_link_libraries (  backupnode 
          jlib
          jhtree
-         remote
+         dafsclient
          mp
          dalibase
     )

tools/swapnode/swapnode.cmake

@@ -28,7 +28,7 @@ set (    SRCS
     )
 
 include_directories (
-         ./../../common/remote
+         ./../../fs/dafsclient
          ./../../system/mp
          ./../../system/include
          ./../../dali/base

tools/swapnode/swapnodelib.cmake

@@ -29,6 +29,7 @@ set (    SRCS
 
 include_directories (
          ./../../common/remote
+         ./../../fs/dafsclient
          ./../../system/mp
          ./../../system/include
          ./../../dali/base
@@ -45,6 +46,7 @@ install ( TARGETS swapnodelib RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATIO
 target_link_libraries ( swapnodelib
          jlib
          remote
+         dafsclient
          dalibase
          workunit
          environment

tools/testsocket/CMakeLists.txt

@@ -27,7 +27,7 @@ include_directories (
          ./../../system/jlib
          ./../../system/include 
          ./../../system/security/securesocket
-         ./../../common/remote
+         ./../../fs/dafsclient
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -36,7 +36,7 @@ set ( SRCS testsocket.cpp )
 HPCC_ADD_EXECUTABLE ( testsocket ${SRCS} )
 target_link_libraries ( testsocket
          jlib
-         remote
+         dafsclient
          )
 
 IF (USE_OPENSSL)

tools/testsocket/testsocket.cpp

@@ -26,7 +26,8 @@
 #include "jthread.hpp"
 #include "jfile.hpp"
 #include "securesocket.hpp"
-#include "sockfile.hpp"
+
+#include "rmtclient.hpp"
 
 bool abortEarly = false;
 bool forceHTTP = false;