HPCC-20340 DFUReadAccess + wsdfuaccess lib

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/environment/environment.cpp

@@ -22,6 +22,7 @@
 #include "jiter.ipp"
 #include "jmisc.hpp"
 #include "jencrypt.hpp"
+#include "jutil.hpp"
 
 #include "mpbase.hpp"
 #include "daclient.hpp"
@@ -30,6 +31,10 @@
 #include "dasds.hpp"
 #include "dalienv.hpp"
 
+#include <string>
+#include <unordered_map>
+#include <tuple>
+
 #define SDS_LOCK_TIMEOUT  30000
 #define DEFAULT_DROPZONE_INDEX      1
 #define DROPZONE_BY_MACHINE_SUFFIX  "-dropzoneByMachine-"
@@ -109,6 +114,15 @@ private:
     mutable Mutex safeCache;
     mutable bool dropZoneCacheBuilt;
     mutable bool machineCacheBuilt;
+    mutable bool clusterKeyNameCache;
+    StringBuffer fileAccessUrl;
+
+    struct KeyPairMapEntity
+    {
+        std::string publicKey, privateKey;
+    };
+    mutable std::unordered_map<std::string, KeyPairMapEntity> keyPairMap;
+    mutable std::unordered_map<std::string, std::string> keyClusterMap;
     StringBuffer xPath;
     mutable unsigned numOfMachines;
     mutable unsigned numOfDropZones;
@@ -122,6 +136,76 @@ private:
     mutable bool isDropZoneRestrictionLoaded = false;
     mutable bool dropZoneRestrictionEnabled = true;
 
+
+    void ensureClusterKeyMap() const // keyPairMap and keyClusterMap it alters is mutable
+    {
+        if (!clusterKeyNameCache)
+        {
+            StringBuffer keysDir;
+            envGetConfigurationDirectory("keys",nullptr, nullptr, keysDir);
+
+            Owned<IPropertyTreeIterator> keyPairIt = p->getElements("EnvSettings/Keys/KeyPair");
+            ForEach(*keyPairIt)
+            {
+                IPropertyTree &keyPair = keyPairIt->query();
+                const char *name = keyPair.queryProp("@name");
+                const char *publicKeyPath = keyPair.queryProp("@publicKey");
+                const char *privateKeyPath = keyPair.queryProp("@privateKey");
+                if (isEmptyString(name))
+                {
+                    WARNLOG("skipping invalid EnvSettings/Key/KeyPair entry, name not defined");
+                    continue;
+                }
+                if (isEmptyString(publicKeyPath) || isEmptyString(privateKeyPath))
+                {
+                    WARNLOG("skipping invalid EnvSettings/Key/KeyPair entry, name=%s", name);
+                    continue;
+                }
+                StringBuffer absPublicKeyPath, absPrivateKeyPath;
+                if (!isAbsolutePath(publicKeyPath))
+                {
+                    absPublicKeyPath.append(keysDir);
+                    addPathSepChar(absPublicKeyPath);
+                    absPublicKeyPath.append(publicKeyPath);
+                }
+                else
+                    absPublicKeyPath.append(publicKeyPath);
+                if (!isAbsolutePath(privateKeyPath))
+                {
+                    absPrivateKeyPath.append(keysDir);
+                    addPathSepChar(absPrivateKeyPath);
+                    absPrivateKeyPath.append(privateKeyPath);
+                }
+                else
+                    absPrivateKeyPath.append(privateKeyPath);
+
+                keyPairMap[name] = { absPublicKeyPath.str(), absPrivateKeyPath.str() };
+            }
+            Owned<IPropertyTreeIterator> clusterIter = p->getElements("EnvSettings/Keys/Cluster");
+            ForEach(*clusterIter)
+            {
+                IPropertyTree &cluster = clusterIter->query();
+                const char *clusterName = cluster.queryProp("@name");
+                if (isEmptyString(clusterName))
+                {
+                    WARNLOG("skipping EnvSettings/Keys/Cluster entry with no name");
+                    continue;
+                }
+                if (cluster.hasProp("@keyPairName"))
+                {
+                    const char *keyPairName = cluster.queryProp("@keyPairName");
+                    if (isEmptyString(keyPairName))
+                    {
+                        WARNLOG("skipping invalid EnvSettings/Key/Cluster entry, name=%s", clusterName);
+                        continue;
+                    }
+                    keyClusterMap[clusterName] = keyPairName;
+                }
+            }
+            clusterKeyNameCache = true;
+        }
+    }
+
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -165,6 +249,30 @@ public:
     unsigned getNumberOfDropZones() const { buildDropZoneCache(); return numOfDropZones; }
     IConstDropZoneInfo * getDropZoneByIndex(unsigned index) const;
     bool isDropZoneRestrictionEnabled() const;
+
+    virtual const char *getClusterKeyPairName(const char *cluster) const override
+    {
+        synchronized procedure(safeCache);
+        ensureClusterKeyMap();
+        return keyClusterMap[cluster].c_str();
+    }
+    virtual const char *getPublicKeyPath(const char *keyPairName) const override
+    {
+        synchronized procedure(safeCache);
+        ensureClusterKeyMap();
+        return keyPairMap[keyPairName].publicKey.c_str();
+    }
+    virtual const char *getPrivateKeyPath(const char *keyPairName) const override
+    {
+        synchronized procedure(safeCache);
+        ensureClusterKeyMap();
+        return keyPairMap[keyPairName].privateKey.c_str();
+    }
+    virtual const char *getFileAccessUrl() const
+    {
+        synchronized procedure(safeCache);
+        return fileAccessUrl.length() ? fileAccessUrl.str() : nullptr;
+    }
 };
 
 class CLockedEnvironment : implements IEnvironment, public CInterface
@@ -277,6 +385,14 @@ public:
             { return c->getDropZoneIterator(); }
     virtual bool isDropZoneRestrictionEnabled() const
             { return c->isDropZoneRestrictionEnabled(); }
+    virtual const char *getClusterKeyPairName(const char *cluster) const override
+            { return c->getClusterKeyPairName(cluster); }
+    virtual const char *getPublicKeyPath(const char *keyPairName) const override
+            { return c->getPublicKeyPath(keyPairName); }
+    virtual const char *getPrivateKeyPath(const char *keyPairName) const override
+            { return c->getPrivateKeyPath(keyPairName); }
+    virtual const char *getFileAccessUrl() const
+            { return c->getFileAccessUrl(); }
 };
 
 void CLockedEnvironment::commit()
@@ -1047,6 +1163,8 @@ void CLocalEnvironment::init()
     numOfMachines = 0;
     numOfDropZones = 0;
     isDropZoneRestrictionLoaded = false;
+    clusterKeyNameCache = false;
+    ::getFileAccessUrl(fileAccessUrl);
 }
 
 CLocalEnvironment::~CLocalEnvironment()
@@ -1508,6 +1626,8 @@ void CLocalEnvironment::clearCache()
         p.setown(conn->getRoot());
     }
     cache.kill();
+    keyClusterMap.clear();
+    keyPairMap.clear();
     init();
     resetPasswordsFromSDS();
 }

common/environment/environment.hpp

@@ -15,11 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-// *** Include file generated by HIDL Version 1.3 from environment.scm ***
-// *** Not to be hand edited (changes will be lost on re-generation) ***
-
-#ifndef environment_SCM_INCL
-#define environment_SCM_INCL
+#ifndef ENVIRONMENT_INCL
+#define ENVIRONMENT_INCL
 
 #include "jiface.hpp"
 
@@ -31,7 +28,7 @@
     #define ENVIRONMENT_API DECL_IMPORT
 #endif
 
-interface IPropertyTree;   // Not yet SCM-compliant
+interface IPropertyTree;   // Forward reference
 interface IEnvironment;    // Forward reference
 interface ISDSSubscription;// Forward reference
 
@@ -150,6 +147,10 @@ interface IConstEnvironment : extends IConstEnvBase
     virtual IConstDropZoneInfo * getDropZoneByAddressPath(const char * netaddress, const char *targetPath) const = 0;
     virtual IConstDropZoneInfoIterator * getDropZoneIterator() const = 0;
     virtual bool isDropZoneRestrictionEnabled() const = 0;
+    virtual const char *getClusterKeyPairName(const char *cluster) const = 0;
+    virtual const char *getPublicKeyPath(const char *keyPairName) const = 0;
+    virtual const char *getPrivateKeyPath(const char *keyPairName) const = 0;
+    virtual const char *getFileAccessUrl() const = 0;
 };
 
 
@@ -173,7 +174,6 @@ interface IEnvironmentFactory : extends IInterface
     virtual void validateCache() = 0;
 };
 
-
 class StringBuffer;
 extern "C" ENVIRONMENT_API IEnvironmentFactory * getEnvironmentFactory(bool update);
 extern "C" ENVIRONMENT_API void closeEnvironment();
@@ -181,5 +181,5 @@ extern "C" ENVIRONMENT_API void closeEnvironment();
 
 
 
-#endif // _environment_SCM_INCL
+#endif // _ENVIRONMENT_INCL
 //end

dali/base/dadfs.cpp

@@ -1303,7 +1303,7 @@ static void checkLogicalScope(const char *scopename,IUserDescriptor *user,bool r
         throw e;
 }
 
-static bool checkLogicalName(CDfsLogicalFileName &dlfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg)
+bool checkLogicalName(CDfsLogicalFileName &dlfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg)
 {
     bool ret = true;
     if (dlfn.isMulti()) { //is temporary superFile?
@@ -1334,6 +1334,14 @@ static bool checkLogicalName(CDfsLogicalFileName &dlfn,IUserDescriptor *user,boo
     return ret;
 }
 
+bool checkLogicalName(const char *lfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg)
+{
+    CDfsLogicalFileName dlfn;
+    dlfn.set(lfn);
+    return checkLogicalName(dlfn, user, readreq, createreq, allowquery, specialnotallowedmsg);
+}
+
+
 /*
  * This class removes all files marked for deletion during transactions.
  *

dali/base/dadfs.hpp

@@ -816,6 +816,8 @@ inline bool isPartTLK(IPartDescriptor *p) { return isPartTLK(p->queryProperties(
 
 extern da_decl void ensureFileScope(const CDfsLogicalFileName &dlfn, unsigned timeoutms=INFINITE);
 
+extern da_decl bool checkLogicalName(const char *lfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg);
+
 
 #endif
 

dali/base/dafdesc.cpp

@@ -3066,3 +3066,25 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
     }
     return res.getLink();
 }
+
+void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file)
+{
+    IPropertyTree *fileInfoTree = info.setPropTree("FileInfo", createPTree());
+    Owned<IPartDescriptorIterator> partIter = file.getIterator();
+    StringBuffer path, host;
+    ForEach(*partIter)
+    {
+        IPropertyTree *partTree = fileInfoTree->addPropTree("Part", createPTree());
+        IPartDescriptor &part = partIter->query();
+        unsigned numCopies = part.numCopies();
+        for (unsigned copy=0; copy<numCopies; copy++)
+        {
+            RemoteFilename rfn;
+            part.getFilename(copy, rfn);
+
+            IPropertyTree *copyTree = partTree->addPropTree("Copy", createPTree());
+            copyTree->setProp("@filePath", rfn.getLocalPath(path.clear()));
+            copyTree->setProp("@host", rfn.queryEndpoint().getUrlStr(host.clear()));
+        }
+    }
+}

dali/base/dafdesc.hpp

@@ -368,5 +368,7 @@ inline DFD_OS SepCharBaseOs(char c)
     return DFD_OSdefault;
 }
 
+extern da_decl void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file);
+
 
 #endif

esp/clients/CMakeLists.txt

@@ -15,3 +15,4 @@
 ################################################################################
 add_subdirectory (wsecl)
 add_subdirectory (WUManager)
+add_subdirectory (wsdfuaccess)

esp/clients/wsdfuaccess/CMakeLists.txt

@@ -0,0 +1,62 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: wsdfuaccess
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for wsdfuaccess
+#####################################################
+
+
+project( wsdfuaccess ) 
+
+include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
+
+set (    SRCS 
+         wsdfuaccess.cpp 
+         ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp
+    )
+
+include_directories ( 
+         ${HPCC_SOURCE_DIR}/system/include 
+         ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/system/security/shared 
+         ${HPCC_SOURCE_DIR}/system/security/securesocket 
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${HPCC_SOURCE_DIR}/system/mp
+         ${HPCC_SOURCE_DIR}/esp/bindings 
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/client 
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp 
+         ${HPCC_SOURCE_DIR}/system/jlib 
+         ${HPCC_SOURCE_DIR}/esp/platform 
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform 
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DWSDFUACCESS_EXPORTS )
+
+HPCC_ADD_LIBRARY( wsdfuaccess SHARED ${SRCS} )
+install ( TARGETS wsdfuaccess RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+target_link_libraries ( wsdfuaccess 
+         jlib
+         xmllib 
+         esphttp 
+         dalibase
+    )
+
+

esp/clients/wsdfuaccess/sourcedoc.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>esp/clients/wsdfuaccess</title>
+
+    <para>
+        The esp/clients/wsdfuaccess directory contains the sources for the esp/clients/wsdfuaccess library.
+       
+        The library provides SOAP access to the ESP DFUFileAccess service.
+        The service can be called with a logical filename, access rights and various other flags
+        (see ESPrequest DFUFileAccessRequest), to retreive file meta data with a security token that can
+        be used to access physical files on dafilesrv.
+    </para>
+</section>

esp/clients/wsdfuaccess/wsdfuaccess.cpp

@@ -0,0 +1,78 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "dautils.hpp"
+#include "seclib.hpp"
+#include "ws_dfu.hpp"
+
+#include "wsdfuaccess.hpp"
+
+namespace wsdfuaccess
+{
+
+CSecAccessType translateToCSecAccessAccessType(SecAccessFlags from)
+{
+    switch (from)
+    {
+        case SecAccess_Access:
+            return CSecAccessType_Access;
+        case SecAccess_Read:
+            return CSecAccessType_Read;
+        case SecAccess_Write:
+            return CSecAccessType_Write;
+        case SecAccess_Full:
+            return CSecAccessType_Full;
+        case SecAccess_None:
+        default:
+            return CSecAccessType_None;
+    }
+}
+
+bool getFileAccess(StringBuffer &metaInfo, const char *serviceUrl, const char *jobId, const char *logicalName, SecAccessFlags access, unsigned expirySecs, const char *user, const char *token)
+{
+    Owned<IClientWsDfu> dfuClient = createWsDfuClient();
+    dfuClient->addServiceUrl(serviceUrl);
+    dfuClient->setUsernameToken(user, token, "");
+
+    Owned<IClientDFUFileAccessRequest> dfuReq = dfuClient->createDFUFileAccessRequest();
+
+    CDfsLogicalFileName lfn;
+    lfn.set(logicalName);
+
+    StringBuffer cluster, lfnName;
+    lfn.getCluster(cluster);
+    lfn.get(lfnName); // remove cluster if present
+
+    dfuReq->setName(lfnName);
+    dfuReq->setCluster(cluster);
+    dfuReq->setExpirySeconds(expirySecs);
+    dfuReq->setAccessRole(CFileAccessRole_Engine);
+    dfuReq->setAccessType(translateToCSecAccessAccessType(access));
+    dfuReq->setJobId(jobId);
+
+    Owned<IClientDFUFileAccessResponse> dfuResp = dfuClient->DFUFileAccess(dfuReq);
+
+    const IMultiException *excep = &dfuResp->getExceptions(); // NB: warning despite getXX name, this does not Link
+    if (excep->ordinality() > 0)
+        throw LINK((IMultiException *)excep); // JCSMORE - const IException.. not caught in general..
+
+    metaInfo.append(dfuResp->getMetaInfoBlob());
+    return true;
+}
+
+} // namespace wsdfuaccess

esp/clients/wsdfuaccess/wsdfuaccess.hpp

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _WSDFUACCESS_HPP
+#define _WSDFUACCESS_HPP
+
+#ifndef WSDFUACCESS_API
+
+#ifdef WSDFUACCESS_EXPORTS
+#define WSDFUACCESS_API DECL_EXPORT
+#else
+#define WSDFUACCESS_API DECL_IMPORT
+#endif
+
+#endif
+
+
+class StringBuffer;
+
+namespace wsdfuaccess
+{
+
+WSDFUACCESS_API bool getFileAccess(StringBuffer &metaInfo, const char *serviceUrl, const char *jobId, const char *logicalName, SecAccessFlags access, unsigned expirySecs, const char *user, const char *token);
+
+} // end of namespace wsdfuaccess
+
+#endif // _WSDFUACCESS_HPP

esp/scm/ws_dfu.ecm

@@ -37,6 +37,26 @@ ESPenum DFUDefFileFormat : string
     def("def"),
 };
 
+ESPenum SecAccessType : string
+{
+    None("None"),
+    Access("Access"),
+    Read("Read"),
+    Write("Write"),
+    Full("Full")
+};
+
+/* The FileAccessRole differentiates what type of info is returned in the response
+ * "Token" will generally be used to refresh an expired token.
+ * "Engine" may not be required long term, but allows the service to tailor and hopefully optimize the reply with info. the engines need. 
+ */
+ESPenum FileAccessRole : string
+{
+    Token("Token"),       // Request for security token only, no plain text file information will be return in the response. May be used to refresh an expired token.
+    Engine("Engine"),     // For internal HPCC engines, plain text file meta info may differ from external clients (for now at least). 
+    External("External"), // For external clients (e.g. Spark), plain text less verbose than legacy format needed by engines (e.g. only needs locations, # parts only). 
+};
+
 ESPStruct SpaceItem
 {
     string Name;
@@ -812,7 +832,43 @@ ESPresponse [exceptions_inline, nil_remove] EraseHistoryResponse
     ESParray<ESPStruct History, Origin> History;
 };
 
+ESPrequest DFUFileAccessRequest
+{
+    string Name;
+    string Cluster;
+    int ExpirySeconds(60);
+    ESPenum FileAccessRole AccessRole;
+    ESPenum SecAccessType AccessType;
+    string JobId;
+    bool ReturnJsonTypeInfo(false);
+    bool ReturnBinTypeInfo(false);
+};
 
+// DFUPartLocations and DFUPartCopies part of DFUFileAccessResponse below
+ESPStruct DFUPartLocations
+{
+    int LocationIndex;
+    string Location;
+};
+
+ESPStruct DFUPartCopies
+{
+    int PartIndex;
+    ESParray<string> LocationIndexes;
+};
+
+ESPresponse [exceptions_inline] DFUFileAccessResponse
+{
+    string MetaInfoBlob;
+    string ExpiryTime;
+    // {NumParts, FilePartLocations, FileParts} depend on ReturnFileInfo in request
+    int NumParts;        // number of parts in logical file
+    ESParray<EspStruct DFUPartLocations> FilePartLocations;
+    ESParray<EspStruct DFUPartCopies> FileParts;
+    
+    binary RecordTypeInfoBin;   // optional
+    string RecordTypeInfoJson;  // optional
+};
 
 //  ===========================================================================
 ESPservice [
@@ -845,6 +901,8 @@ ESPservice [
     ESPmethod EraseHistory(EraseHistoryRequest, EraseHistoryResponse);
     ESPmethod DFURecordTypeInfo(DFURecordTypeInfoRequest, DFURecordTypeInfoResponse);
     ESPmethod EclRecordTypeInfo(EclRecordTypeInfoRequest, EclRecordTypeInfoResponse);
+
+    ESPmethod [auth_feature("DfuAccess:READ"), min_var("1.39")] DFUFileAccess(DFUFileAccessRequest, DFUFileAccessResponse);
 };
 
 SCMexportdef(WSDFU);

esp/services/ws_dfu/CMakeLists.txt

@@ -60,6 +60,7 @@ include_directories (
          ./../../../ecl/hql 
          ./../../../system/security/securesocket 
          ./../../../system/security/shared 
+         ./../../../system/security/cryptohelper 
          ./../../../system/include 
          ./../../../common/workunit 
          ./../../../common/remote 

esp/services/ws_dfu/ws_dfuService.cpp

@@ -58,6 +58,12 @@
 #include "package.h"
 #include "daaudit.hpp"
 
+#include "jflz.hpp"
+#include "digisign.hpp"
+
+using namespace cryptohelper;
+
+
 #define     Action_Delete           "Delete"
 #define     Action_AddtoSuperfile   "Add To Superfile"
 static const char* FEATURE_URL="DfuAccess";
@@ -82,6 +88,7 @@ const unsigned MAX_KEY_ROWS = 20;
 
 short days[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
 
+
 CThorNodeGroup* CThorNodeGroupCache::readNodeGroup(const char* _groupName)
 {
     Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
@@ -116,32 +123,32 @@ CThorNodeGroup* CThorNodeGroupCache::lookup(const char* groupName, unsigned time
 
 void CWsDfuEx::init(IPropertyTree *cfg, const char *process, const char *service)
 {
-    StringBuffer xpath;
 
     DBGLOG("Initializing %s service [process = %s]", service, process);
 
     espProcess.set(process);
 
-    xpath.appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/DefaultScope", process, service);
-    cfg->getProp(xpath.str(), defaultScope_);
+    VStringBuffer xpath("Software/EspProcess[@name=\"%s\"]", process);
+    IPropertyTree *processTree = cfg->queryPropTree(xpath);
+    if (!processTree)
+        throw MakeStringException(-1, "config not found for process %s", process);
 
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/User", process, service);
-    cfg->getProp(xpath.str(), user_);
+    xpath.clear().appendf("EspService[@name=\"%s\"]", service);
+    IPropertyTree *serviceTree = processTree->queryPropTree(xpath);
+    if (!serviceTree)
+        throw MakeStringException(-1, "config not found for service %s", service);
 
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/Password", process, service);
-    cfg->getProp(xpath.str(), password_);
+    serviceTree->getProp("DefaultScope", defaultScope_);
+    serviceTree->getProp("User", user_);
+    serviceTree->getProp("Password", password_);
 
     StringBuffer disableUppercaseTranslation;
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/DisableUppercaseTranslation", process, service);
-    cfg->getProp(xpath.str(), disableUppercaseTranslation);
+    serviceTree->getProp("DisableUppercaseTranslation", disableUppercaseTranslation);
 
     m_clusterName.clear();
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/ClusterName", process, service);
-    cfg->getProp(xpath.str(), m_clusterName);
+    serviceTree->getProp("ClusterName", m_clusterName);
 
-    Linked<IPropertyTree>  globals;
-    globals.set(cfg->queryPropTree(StringBuffer("Software/EspProcess[@name=\"").append(process).append("\"]/EspService[@name=\"").append(service).append("\"]").str()));
-    const char * plugins = globals->queryProp("Plugins/@path");
+    const char * plugins = serviceTree->queryProp("Plugins/@path");
     if (plugins)
         queryTransformerRegistry().addPlugins(plugins);
 
@@ -149,15 +156,12 @@ void CWsDfuEx::init(IPropertyTree *cfg, const char *process, const char *service
     if (streq(disableUppercaseTranslation.str(), "true"))
         m_disableUppercaseTranslation = true;
 
-    xpath.setf("Software/EspProcess[@name=\"%s\"]/@PageCacheTimeoutSeconds", process);
-    if (cfg->hasProp(xpath.str()))
-        setPageCacheTimeoutMilliSeconds(cfg->getPropInt(xpath.str()));
-    xpath.setf("Software/EspProcess[@name=\"%s\"]/@MaxPageCacheItems", process);
-    if (cfg->hasProp(xpath.str()))
-        setMaxPageCacheItems(cfg->getPropInt(xpath.str()));
+    if (processTree->hasProp("@PageCacheTimeoutSeconds"))
+        setPageCacheTimeoutMilliSeconds(processTree->getPropInt("@PageCacheTimeoutSeconds"));
+    if (processTree->hasProp("@MaxPageCacheItems"))
+        setMaxPageCacheItems(processTree->getPropInt("@MaxPageCacheItems"));
 
-    xpath.setf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/NodeGroupCacheMinutes", process, service);
-    int timeout = cfg->getPropInt(xpath.str(), -1);
+    int timeout = serviceTree->getPropInt("NodeGroupCacheMinutes", -1);
     if (timeout > -1)
         nodeGroupCacheTimeout = (unsigned) timeout*60*1000;
     else
@@ -169,6 +173,9 @@ void CWsDfuEx::init(IPropertyTree *cfg, const char *process, const char *service
 
     setDaliServixSocketCaching(true);
 
+    factory.setown(getEnvironmentFactory(true));
+    env.setown(factory->openEnvironment());
+    maxFileAccessExpirySeconds = serviceTree->getPropInt("@maxFileAccessExpirySeconds", defaultMaxFileAccessExpirySeconds);
 }
 
 bool CWsDfuEx::onDFUSearch(IEspContext &context, IEspDFUSearchRequest & req, IEspDFUSearchResponse & resp)
@@ -5885,4 +5892,253 @@ int CWsDfuEx::GetIndexData(IEspContext &context, bool bSchemaOnly, const char* i
     return iRet;
 }
 
+unsigned CWsDfuEx::getFilePartsInfo(IEspContext &context, IDistributedFile *df, const char *clusterName,
+    IArrayOf<IEspDFUPartLocations> &dfuPartLocations, IArrayOf<IEspDFUPartCopies> &dfuPartCopies)
+{
+    int nextLocationsIndex = 1; // NB: both LocationIndex and PartIndex are 1 based in response.
+    MapStringTo<int> locationMap;
+    Owned<IFileDescriptor> fdesc = df->getFileDescriptor(clusterName);
+    Owned<IPartDescriptorIterator> pi = fdesc->getIterator();
+    ForEach(*pi)
+    {
+        IPartDescriptor& part = pi->query();
+        unsigned partIndex = part.queryPartIndex();
+
+        StringArray partCopyLocationsIndexes;
+        for (unsigned int i=0; i<part.numCopies(); i++)
+        {
+            StringBuffer host, locationsIndexStr;
+            part.queryNode(i)->endpoint().getUrlStr(host);
+            int *locationsIndex = locationMap.getValue(host.str());
+            if (locationsIndex)
+            {
+                partCopyLocationsIndexes.append(locationsIndexStr.append(*locationsIndex).str());
+                continue;
+            }
+
+            Owned<IEspDFUPartLocations> partLocations = createDFUPartLocations();
+            partLocations->setLocationIndex(nextLocationsIndex);
+            partLocations->setLocation(host.str());
+            dfuPartLocations.append(*partLocations.getClear());
+
+            partCopyLocationsIndexes.append(locationsIndexStr.append(nextLocationsIndex).str());
+            locationMap.setValue(host.str(), nextLocationsIndex);
+            nextLocationsIndex++;
+        }
+
+        Owned<IEspDFUPartCopies> partCopies = createDFUPartCopies();
+        partCopies->setPartIndex(partIndex + 1);
+        partCopies->setLocationIndexes(partCopyLocationsIndexes);
+        dfuPartCopies.append(*partCopies.getClear());
+    }
+    return df->numParts();
+}
+
+static const char *securityInfoVersion="1";
+void CWsDfuEx::getFileMeta(StringBuffer &metaInfoStr, IDistributedFile &file, IUserDescriptor *user, CFileAccessRole role, const char *expiryTime, const char *keyPairName, IConstDFUFileAccessRequest &req)
+{
+    Owned<IPropertyTree> metaInfoEnvelope = createPTree();
+    Owned<IPropertyTree> metaInfo = createPTree();
+    const char *clusterName = req.getCluster(); // can be null
+    Owned<IFileDescriptor> fDesc = file.getFileDescriptor(clusterName);
+    extractFilePartInfo(*metaInfo, *fDesc);
+
+    MemoryBuffer metaInfoMb;
+
+    /* NB: If file access security is disabled in the environment, or on a per cluster basis
+     * keyPairName will be blank. In that case the meta data is returned in plain format.
+     * NB2: Dafilesrv's would also require file access security to be disabled in that case,
+     * otherwise they will be denied access.
+     * Should be part of the same configuration setup.
+     */
+#ifdef _USE_OPENSSL
+    if (!isEmptyString(keyPairName)) // without it, meta data is not encrypted
+    {
+        metaInfo->setProp("version", securityInfoVersion);
+        metaInfo->setProp("logicalFilename", file.queryLogicalName());
+        metaInfo->setProp("jobId", req.getJobId());
+        metaInfo->setProp("accessType", req.getAccessTypeAsString());
+        StringBuffer userStr;
+        if (user)
+            metaInfo->setProp("user", user->getUserName(userStr).str());
+        metaInfo->setProp("keyPairName", keyPairName);
+        metaInfo->setProp("expiryTime", expiryTime);
+
+        MemoryBuffer metaInfoBlob;
+        metaInfo->serialize(metaInfoBlob);
+
+        const char *privateKeyFName = env->getPrivateKeyPath(keyPairName);
+        Owned<CLoadedKey> privateKey = loadPrivateKeyFromFile(privateKeyFName, nullptr);
+        StringBuffer metaInfoSignature;
+        digiSign(metaInfoSignature, metaInfoBlob.length(), metaInfoBlob.bytes(), *privateKey);
+        metaInfoEnvelope->setProp("signature", metaInfoSignature);
+        metaInfoEnvelope->setPropBin("metaInfoBlob", metaInfoBlob.length(), metaInfoBlob.bytes());
+        metaInfoEnvelope->serialize(metaInfoMb.clear());
+    }
+    else
+#endif
+        metaInfo->serialize(metaInfoMb);
+
+    MemoryBuffer compressedMetaInfoMb;
+    fastLZCompressToBuffer(compressedMetaInfoMb, metaInfoMb.length(), metaInfoMb.bytes());
+    JBASE64_Encode(compressedMetaInfoMb.bytes(), compressedMetaInfoMb.length(), metaInfoStr, false);
+}
+
+StringBuffer &CWsDfuEx::getFileDafilesrvKeyName(StringBuffer &keyPairName, IDistributedFile &file)
+{
+    unsigned numClusters = file.numClusters();
+    for (unsigned c=0; c<numClusters; c++)
+    {
+        StringBuffer clusterName;
+        const char *cluster = file.getClusterName(c, clusterName.clear()).str();
+        const char *_keyPairName = env->getClusterKeyPairName(cluster);
+        if (0 == c)
+            keyPairName.set(_keyPairName);
+        else if (!strsame(keyPairName, _keyPairName))
+            throwStringExceptionV(0, "Configuration issue - file '%s' is on multiple clusters, keys for file access must match", file.queryLogicalName());
+    }
+
+    return keyPairName;
+}
+
+void CWsDfuEx::getFileAccess(IEspContext &context, IUserDescriptor *udesc, SecAccessFlags accessType, IEspDFUFileAccessRequest &req, IEspDFUFileAccessResponse &resp)
+{
+    bool writePermissions = (accessType == SecAccess_Write) || (accessType == SecAccess_Full);
+    bool readPermissions = true; // by implication
+
+    StringBuffer fileName(req.getName());
+    if (!isEmptyString(req.getCluster()))
+        fileName.append("@").append(req.getCluster());
+
+    checkLogicalName(fileName, udesc, readPermissions, writePermissions, false, nullptr);
+
+    switch (accessType)
+    {
+        case SecAccess_Access:
+        case SecAccess_Read:
+            break;
+        default:
+        {
+            // NB - no handling for write/full at moment
+            return;
+        }
+    }
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, udesc, false, false, true); // lock super-owners
+    if (!df)
+        throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.", req.getName());
+
+    CFileAccessRole role = req.getAccessRole();
+    switch (role)
+    {
+        case CFileAccessRole_Token:
+        {
+            break;
+        }
+        case CFileAccessRole_Engine:
+        {
+            /* JCSMORE - for now do nothing
+             * Ideally, would get the file tree here and add it to 'metaInfo' tree, i.e. outside of uncrypted secureInfo blob
+             * Then client could construct a IDistributeFile from it etc.
+             * However, the way the engines and IDistributedFile work at the moment, means that using this info
+             * at the client side would require a significant amount of refactoring of the IDistributedFile implementation.
+             * Not least because IDF allows updates via IPT -> Dali.
+             *
+             * So for now don't send anything, and rely on engine fetching the legacy way, i.e. direct from Dali, via lazy fetching etc.
+             */
+            break;
+        }
+        case CFileAccessRole_External:
+        {
+            IArrayOf<IEspDFUPartLocations> dfuPartLocations;
+            IArrayOf<IEspDFUPartCopies> dfuPartCopies;
+            resp.setNumParts(getFilePartsInfo(context, df, req.getCluster(), dfuPartLocations, dfuPartCopies));
+            resp.setFilePartLocations(dfuPartLocations);
+            resp.setFileParts(dfuPartCopies);
+            if (req.getReturnJsonTypeInfo() || req.getReturnJsonTypeInfo())
+            {
+                MemoryBuffer binLayout;
+                StringBuffer jsonLayout;
+                if (!getRecordFormatFromRtlType(binLayout, jsonLayout, df->queryAttributes(), req.getReturnJsonTypeInfo(), req.getReturnJsonTypeInfo()))
+                    getRecordFormatFromECL(binLayout, jsonLayout, df->queryAttributes(), req.getReturnJsonTypeInfo(), req.getReturnJsonTypeInfo());
+                if (req.getReturnJsonTypeInfo() && jsonLayout.length())
+                    resp.setRecordTypeInfoJson(jsonLayout.str());
+                if (req.getReturnBinTypeInfo() && binLayout.length())
+                    resp.setRecordTypeInfoBin(binLayout);
+            }
+            break;
+        }
+        default:
+            throwUnexpected();
+    }
+
+    // setup "expiryTime"
+    unsigned expirySecs = req.getExpirySeconds();
+    if (expirySecs > maxFileAccessExpirySeconds)
+        expirySecs = maxFileAccessExpirySeconds;
+    time_t now;
+    time(&now);
+    CDateTime expiryDt;
+    expiryDt.set(now + expirySecs);
+    StringBuffer expiryTimeStr;
+    expiryDt.getString(expiryTimeStr);
+
+    StringBuffer keyPairName;
+    getFileDafilesrvKeyName(keyPairName, *df);
+
+    StringBuffer metaInfo;
+    getFileMeta(metaInfo, *df, udesc, role, expiryTimeStr, keyPairName, req);
+    resp.setMetaInfoBlob(metaInfo);
+    resp.setExpiryTime(expiryTimeStr);
+}
+
+
+SecAccessFlags translateToSecAccessFlags(CSecAccessType from)
+{
+    switch (from)
+    {
+        case CSecAccessType_Access:
+            return SecAccess_Access;
+        case CSecAccessType_Read:
+            return SecAccess_Read;
+        case CSecAccessType_Write:
+            return SecAccess_Write;
+        case CSecAccessType_Full:
+            return SecAccess_Full;
+        case CSecAccessType_None:
+        default:
+            return SecAccess_None;
+    }
+}
+
+bool CWsDfuEx::onDFUFileAccess(IEspContext &context, IEspDFUFileAccessRequest &req, IEspDFUFileAccessResponse &resp)
+{
+    try
+    {
+        SecAccessFlags accessType = translateToSecAccessFlags(req.getAccessType());
+        if (SecAccess_None == accessType)
+            throw MakeStringException(ECLWATCH_DFU_ACCESS_DENIED, "onDFUFileAccess - Permission denied.");
+        else if (!context.validateFeatureAccess(FEATURE_URL, accessType, false))
+            throw MakeStringException(ECLWATCH_DFU_ACCESS_DENIED, "onDFUFileAccess - Permission denied.");
+
+        if (isEmptyString(req.getName()))
+             throw MakeStringException(ECLWATCH_INVALID_INPUT, "No Name defined.");
+
+        StringBuffer userID;
+        context.getUserID(userID);
+
+        Owned<IUserDescriptor> userDesc;
+        if (!userID.isEmpty())
+        {
+            userDesc.setown(createUserDescriptor());
+            userDesc->set(userID.str(), context.queryPassword(), context.querySignature());
+        }
+        getFileAccess(context, userDesc, accessType, req, resp);
+    }
+    catch (IException *e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
 //////////////////////HPCC Browser//////////////////////////

esp/services/ws_dfu/ws_dfuService.hpp

@@ -23,6 +23,7 @@
 #include "fileview.hpp"
 #include "fvrelate.hpp"
 #include "dadfs.hpp"
+#include "environment.hpp"
 #include <atomic>
 
 class CThorNodeGroup: public CInterface
@@ -136,6 +137,9 @@ private:
     unsigned nodeGroupCacheTimeout;
     Owned<CThorNodeGroupCache> thorNodeGroupCache;
     std::atomic<bool> m_daliDetached{false};
+    Owned<IEnvironmentFactory> factory;
+    Owned<IConstEnvironment> env;
+    static const unsigned defaultMaxFileAccessExpirySeconds=86400; // 24 hours
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -165,6 +169,7 @@ public:
     virtual bool onSuperfileAction(IEspContext &context, IEspSuperfileActionRequest &req, IEspSuperfileActionResponse &resp);
     virtual bool onListHistory(IEspContext &context, IEspListHistoryRequest &req, IEspListHistoryResponse &resp);
     virtual bool onEraseHistory(IEspContext &context, IEspEraseHistoryRequest &req, IEspEraseHistoryResponse &resp);
+    virtual bool onDFUFileAccess(IEspContext &context, IEspDFUFileAccessRequest &req, IEspDFUFileAccessResponse &resp);
 
 private:
     const char* getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix);
@@ -231,6 +236,12 @@ private:
     void queryFieldNames(IEspContext &context, const char *fileName, const char *cluster,
         unsigned __int64 fieldMask, StringArray &fieldNames);
     void parseFieldMask(unsigned __int64 fieldMask, unsigned &fieldCount, IntArray &fieldIndexArray);
+    unsigned getFilePartsInfo(IEspContext &context, IDistributedFile *df, const char *clusterName,
+        IArrayOf<IEspDFUPartLocations> &dfuPartLocations, IArrayOf<IEspDFUPartCopies> &dfuPartCopies);
+    StringBuffer &getFileDafilesrvKeyName(StringBuffer &keyPairName, IDistributedFile &file);
+    void getFileMeta(StringBuffer &metaInfo, IDistributedFile &file, IUserDescriptor *user, CFileAccessRole role, const char *expiryTime, const char *keyPairName, IConstDFUFileAccessRequest &req);
+    void getFileAccess(IEspContext &context, IUserDescriptor *udesc, SecAccessFlags accessType, IEspDFUFileAccessRequest &req, IEspDFUFileAccessResponse &resp);
+
     bool attachServiceToDali() override
     {
         m_daliDetached = false;
@@ -257,6 +268,7 @@ private:
     StringBuffer user_;
     StringBuffer password_;
     StringAttr   espProcess;
+    unsigned maxFileAccessExpirySeconds = defaultMaxFileAccessExpirySeconds;
 };
 
 

initfiles/componentfiles/configxml/esp.xsd.in

@@ -991,6 +991,13 @@
                  </xs:restriction>
                </xs:simpleType>
             </xs:attribute>
+            <xs:attribute name="MaxFileAccessExpirySeconds" type="xs:integer" use="optional" default="86400">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Maximum time a file access request is valid for</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -522,6 +522,7 @@
    <Category dir="$ENV{DESTDIR}${EXEC_PREFIX}/lib/[NAME]/queries/[INST]" name="query"/>
    <Category dir="$ENV{DESTDIR}${EXEC_PREFIX}/lock/[NAME]/[INST]" name="lock"/>
    <Category dir="$ENV{DESTDIR}${EXEC_PREFIX}/lib/[NAME]/hpcc-data4/[COMPONENT]" name="data4"/>
+   <Category dir="$ENV{DESTDIR}${EXEC_PREFIX}/lib/[NAME]/keys" name="keys"/>
   </Directories>
   <EclAgentProcess allowedPipePrograms="*"
                    build="_"

system/jlib/jutil.cpp

@@ -2540,6 +2540,30 @@ static IPropertyTree *getOSSdirTree()
     return NULL;
 }
 
+StringBuffer &getFileAccessUrl(StringBuffer &out)
+{
+    Owned<IPropertyTree> envtree = getHPCCEnvironment();
+    if (envtree)
+    {
+        IPropertyTree *secureFileAccessInfo = envtree->queryPropTree("EnvSettings/SecureFileAccess");
+        if (secureFileAccessInfo)
+        {
+            const char *protocol = secureFileAccessInfo->queryProp("@protocol");
+            const char *host = secureFileAccessInfo->queryProp("@host");
+            unsigned port = secureFileAccessInfo->getPropInt("@port", (unsigned)-1);
+            if (isEmptyString(protocol))
+                WARNLOG("Missing protocol from secure file access definition");
+            else if (isEmptyString(host))
+                WARNLOG("Missing host from secure file access definition");
+            else if ((unsigned)-1 == port)
+                WARNLOG("Missing port from secure file access definition");
+            else
+                out.appendf("%s://%s:%u/WsDfu", protocol, host, port);
+        }
+    }
+    return out;
+}
+
 bool getConfigurationDirectory(const IPropertyTree *useTree, const char *category, const char *component, const char *instance, StringBuffer &dirout)
 {
     Linked<const IPropertyTree> dirtree = useTree;

system/jlib/jutil.hpp

@@ -398,6 +398,9 @@ extern jlib_decl bool replaceConfigurationDirectoryEntry(const char *path,const
 
 extern jlib_decl const char *queryCurrentProcessPath();
 
+extern jlib_decl StringBuffer &getFileAccessUrl(StringBuffer &out);
+
+
 /**
  * Locate the 'package home' directory - normally /opt/HPCCSystems - by detecting the current executable's location
  *

system/security/cryptohelper/ske.hpp

@@ -23,6 +23,8 @@
 #include "cryptocommon.hpp"
 
 
+#include "pke.hpp"
+
 namespace cryptohelper
 {