HPCC-22682 Limit access to files marked as Restricted

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>

common/fileview2/fvdisksource.cpp

@@ -193,7 +193,7 @@ DiskDataSource::DiskDataSource(const char * _logicalName, IHqlExpression * _disk
         udesc->set(_username, _password);
     }
 
-    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get()));
+    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get(),false,false,false,nullptr,defaultPrivilegedUser));
 }
 
 

common/fileview2/fvidxsource.cpp

@@ -137,7 +137,7 @@ IndexDataSource::IndexDataSource(const char * _logicalName, IHqlExpression * _di
         udesc->set(_username, _password);
     }
 
-    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get()));
+    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get(),false,false,false,nullptr,defaultPrivilegedUser));
     filtered = false;
 }
 

common/fileview2/fvrelate.cpp

@@ -373,7 +373,7 @@ void ViewFileWeb::gatherWebFromPattern(const char * filenamePattern, const ViewG
     if (!localOptions.kind)
         localOptions.kind = S_LINK_RELATIONSHIP_KIND;
 
-    Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator(filenamePattern, false, udesc);
+    Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator(filenamePattern, false, udesc, defaultPrivilegedUser);
     if (iter->first())
     {
         do
@@ -426,7 +426,7 @@ ViewFile * ViewFileWeb::walkFile(const char * filename, IDistributedFile * alrea
         options.isExplicitFile = false;
     }
 
-    Owned<IDistributedFile> resolved = alreadyResolved ? LINK(alreadyResolved) : directory.lookup(filename,udesc,false,false,true); // lock super-owners
+    Owned<IDistributedFile> resolved = alreadyResolved ? LINK(alreadyResolved) : directory.lookup(filename,udesc,false,false,true,nullptr,defaultPrivilegedUser); // lock super-owners
     if (!resolved)
         return NULL;
 

common/fileview2/fvresultset.cpp

@@ -98,7 +98,7 @@ IFvDataSource * createFileDataSource(const char * logicalName, const char * clus
         udesc->set(username, password);
     }
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc.get());
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc.get(),false,false,false,nullptr,defaultPrivilegedUser);
     if (!df)
         throwError1(FVERR_CouldNotResolveX, logicalName);
     return createFileDataSource(df, logicalName, cluster, username, password);
@@ -2136,7 +2136,7 @@ IDistributedFile * CResultSetFactory::lookupLogicalName(const char * logicalName
         udesc->set(username, password);
     }
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc.get());
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc.get(),false,false,false,nullptr,defaultPrivilegedUser);
     if (!df)
         throwError1(FVERR_CouldNotResolveX, logicalName);
     return df.getClear();

common/thorhelper/thorcommon.hpp

@@ -649,4 +649,11 @@ extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &transl
 // Returns a ITranslator that gives access to a dynamic translator, keyed translator and the format used
 extern THORHELPER_API ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode);
 
+inline bool isActivityCodeSigned(IPropertyTree &graphNode)
+{
+    if (!isEmptyString(graphNode.queryProp("att[@name=\"signedBy\"]/@value")))
+        return true;
+    return false;
+}
+
 #endif // THORHELPER_HPP

common/workunit/referencedfilelist.cpp

@@ -338,7 +338,7 @@ void ReferencedFile::resolveLocal(const char *dstCluster, const char *srcCluster
         return;
     }
     reset();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user, false, false, false, nullptr, defaultPrivilegedUser);
     if(df)
         processLocalFileInfo(df, dstCluster, srcCluster, subfiles);
     else
@@ -392,7 +392,7 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
     reset();
     if (checkLocalFirst) //usually means we don't want to overwrite existing file info
     {
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user);
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user, false, false, false, nullptr, defaultPrivilegedUser);
         if(df)
         {
             processLocalFileInfo(df, dstCluster, NULL, subfiles);
@@ -471,7 +471,7 @@ void ReferencedFile::cloneSuperInfo(unsigned updateFlags, ReferencedFileList *li
             return;
 
         IDistributedFileDirectory &dir = queryDistributedFileDirectory();
-        Owned<IDistributedFile> df = dir.lookup(logicalName.str(), user);
+        Owned<IDistributedFile> df = dir.lookup(logicalName.str(), user, false, false, false, nullptr, defaultPrivilegedUser);
         if(df)
         {
             if (!(updateFlags & DALI_UPDATEF_SUPERFILES))

dali/base/dadfs.cpp

@@ -269,6 +269,8 @@ public:
             return str.append(": Lookup connection timeout: ").append(errstr);
         case DFSERR_FailedToDeleteFile:
             return str.append(": Failed to delete file: ").append(errstr);
+        case DFSERR_RestrictedFileAccessDenied:
+            return str.append(": Access to restricted file denied: ").append(errstr);
         }
         return str.append("Unknown DFS Exception");
     }
@@ -1044,8 +1046,8 @@ public:
 
     IDistributedFile *dolookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout);
 
-    IDistributedFile *lookup(const char *_logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout);
-    IDistributedFile *lookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout);
+    IDistributedFile *lookup(const char *_logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool privilegedUser, unsigned timeout) override;
+    IDistributedFile *lookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool privilegedUser, unsigned timeout) override;
 
     /* createNew always creates an unnamed unattached distributed file
      * The caller must associated it with a name and credentials when it is attached (attach())
@@ -1056,7 +1058,7 @@ public:
     IDistributedSuperFile *createNewSuperFile(IPropertyTree *tree, const char *optionalName=nullptr);
     void removeSuperFile(const char *_logicalname, bool delSubs, IUserDescriptor *user, IDistributedFileTransaction *transaction);
 
-    IDistributedFileIterator *getIterator(const char *wildname, bool includesuper,IUserDescriptor *user);
+    IDistributedFileIterator *getIterator(const char *wildname, bool includesuper,IUserDescriptor *user,bool isPrivilegedUser);
     IDFAttributesIterator *getDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive, bool includesuper,INode *foreigndali,unsigned foreigndalitimeout);
     IPropertyTreeIterator *getDFAttributesTreeIterator(const char *filters, DFUQResultField* localFilters, const char *localFilterBuf,
         IUserDescriptor *user, bool recursive, bool& allMatchingFilesReceived, INode *foreigndali,unsigned foreigndalitimeout);
@@ -1377,7 +1379,7 @@ public:
         for (;;)
         {
             // Transaction files have already been unlocked at this point, delete all remaining files
-            Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn, user, true, false, true, NULL, SDS_SUB_LOCK_TIMEOUT);
+            Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn, user, true, false, true, nullptr, defaultPrivilegedUser, SDS_SUB_LOCK_TIMEOUT);
             if (!file.get())
                 return;
             StringBuffer reason;
@@ -1672,7 +1674,7 @@ public:
             return LINK(ret);
         else
         {
-            ret = queryDistributedFileDirectory().lookup(name, udesc, false, false, false, this, timeout);
+            ret = queryDistributedFileDirectory().lookup(name, udesc, false, false, false, this, defaultPrivilegedUser, timeout);
             if (ret)
                 queryCreate(name, ret, true);
             return ret;
@@ -2497,11 +2499,12 @@ class CDistributedFileIterator: public CDistributedFileIteratorBase<IDistributed
     IDistributedFileDirectory *parent;
     Linked<IUserDescriptor> udesc;
     Linked<IDistributedFileTransaction> transaction;
+    bool isPrivilegedUser = false;
 
     bool set()
     {
         while (isValid()) {
-            cur.setown(parent->lookup(queryName(),udesc));
+            cur.setown(parent->lookup(queryName(),udesc, false, false, false, nullptr, isPrivilegedUser));
             if (cur)
                 return true;
             index++;
@@ -2510,8 +2513,8 @@ class CDistributedFileIterator: public CDistributedFileIteratorBase<IDistributed
     }
 
 public:
-    CDistributedFileIterator(IDistributedFileDirectory *_dir,const char *wildname,bool includesuper,IUserDescriptor *user,IDistributedFileTransaction *_transaction=NULL)
-        : transaction(_transaction)
+    CDistributedFileIterator(IDistributedFileDirectory *_dir,const char *wildname,bool includesuper,IUserDescriptor *user,bool _isPrivilegedUser, IDistributedFileTransaction *_transaction=NULL)
+        : isPrivilegedUser(_isPrivilegedUser), transaction(_transaction)
     {
         setUserDescriptor(udesc,user);
         if (!wildname||!*wildname)
@@ -4688,7 +4691,7 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
 
     CDistributedFilePartArray partscache;
     FileClusterInfoArray clusterscache;
-
+    bool containsRestrictedSubfile = false;
     /**
      * Adds a sub-file to a super-file within a transaction.
      */
@@ -5121,16 +5124,17 @@ protected:
                 if (!orderedSubFiles[i])
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: missing subfile part number %d of %d", logicalName.get(), i+1, n);
             }
+            containsRestrictedSubfile = false;
             // Now try to resolve them all (file/superfile)
             for (unsigned f=0; f<n; f++)
             {
                 IPropertyTree &sub = *(orderedSubFiles[f]);
                 sub.getProp("@name",subname.clear());
                 Owned<IDistributedFile> subfile;
-                subfile.setown(transaction?transaction->lookupFile(subname.str(),timeout):parent->lookup(subname.str(), udesc, false, false, false, transaction, timeout));
+                subfile.setown(transaction?transaction->lookupFile(subname.str(),timeout):parent->lookup(subname.str(), udesc, false, false, false, transaction, defaultPrivilegedUser, timeout));
                 if (!subfile.get())
                     subfile.setown(transaction?transaction->lookupSuperFile(subname.str(),timeout):parent->lookupSuperFile(subname.str(),udesc,transaction,timeout));
-
+                containsRestrictedSubfile = containsRestrictedSubfile || subfile->isRestrictedAccess();
                 // Some files are ok not to exist
                 if (!subfile.get())
                 {
@@ -6121,6 +6125,13 @@ public:
                 throw MakeStringException(-1, "The value of @numsubfiles (%d) is not equal to the number of SubFile items (%d)!",numSubfiles, subFileCount);
         }
     }
+    virtual bool isRestrictedAccess() override
+    {
+        // This ensures restriction applies even if superfile is unrestricted but subfiles are.
+        // However, this also means that the superfile will show as "Restricted" in ECL Watch and whenever the user tries to unset the flag
+        // it will appear to reset to Restricted.
+        return containsRestrictedSubfile;
+    }
 
 private:
     void doAddSubFile(IDistributedFile *_sub,bool before,const char *other,IDistributedFileTransactionExt *transaction) // takes ownership of sub
@@ -7618,11 +7629,11 @@ INamedGroupStore  &queryNamedGroupStore()
 
 // --------------------------------------------------------
 
-IDistributedFile *CDistributedFileDirectory::lookup(const char *_logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout)
+IDistributedFile *CDistributedFileDirectory::lookup(const char *_logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool privilegedUser, unsigned timeout)
 {
     CDfsLogicalFileName logicalname;
     logicalname.set(_logicalname);
-    return lookup(logicalname, user, writeattr, hold, lockSuperOwner, transaction, timeout);
+    return lookup(logicalname, user, writeattr, hold, lockSuperOwner, transaction, privilegedUser, timeout);
 }
 
 IDistributedFile *CDistributedFileDirectory::dolookup(CDfsLogicalFileName &_logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout)
@@ -7723,9 +7734,13 @@ IDistributedFile *CDistributedFileDirectory::dolookup(CDfsLogicalFileName &_logi
     return NULL;
 }
 
-IDistributedFile *CDistributedFileDirectory::lookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, unsigned timeout)
+IDistributedFile *CDistributedFileDirectory::lookup(CDfsLogicalFileName &logicalname, IUserDescriptor *user, bool writeattr, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool privilegedUser, unsigned timeout)
 {
-    return dolookup(logicalname, user, writeattr, hold, lockSuperOwner, transaction, timeout);
+    Owned <IDistributedFile>distributedFile = dolookup(logicalname, user, writeattr, hold, lockSuperOwner, transaction, timeout);
+    // Restricted access is currently designed to stop users viewing sensitive information. It is not designed to stop users deleting or overwriting existing restricted files
+    if (writeattr==false && distributedFile && distributedFile->isRestrictedAccess() && !privilegedUser)
+        throw new CDFS_Exception(DFSERR_RestrictedFileAccessDenied,logicalname.get());
+    return distributedFile.getClear();
 }
 
 IDistributedSuperFile *CDistributedFileDirectory::lookupSuperFile(const char *_logicalname,IUserDescriptor *user,IDistributedFileTransaction *transaction, unsigned timeout)
@@ -7764,7 +7779,9 @@ bool CDistributedFileDirectory::exists(const char *_logicalname,IUserDescriptor
     external = dlfn.isExternal();
     foreign = dlfn.isForeign();
     if (foreign) {
-        Owned<IDistributedFile> file = lookup(_logicalname, user, false, false, false, NULL, defaultTimeout);
+        // Restricted access is currently designed to stop users viewing sensitive information. Assuming privileged user rights to allow
+        // exists() operation to succeed regardless of user rights
+        Owned<IDistributedFile> file = lookup(_logicalname, user, false, false, false, NULL, defaultPrivilegedUser, defaultTimeout);
         if (file.get()==NULL)
             return false;
         if (file->querySuperFile()) {
@@ -7799,7 +7816,9 @@ bool CDistributedFileDirectory::exists(const char *_logicalname,IUserDescriptor
 
 bool CDistributedFileDirectory::existsPhysical(const char *_logicalname, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = lookup(_logicalname, user, false, false, false, NULL, defaultTimeout);
+    // Restricted access is currently designed to stop users viewing sensitive information. Assuming privileged user rights to allow
+    // existsPhysical() operation to succeed regardless of user rights
+    Owned<IDistributedFile> file = lookup(_logicalname, user, false, false, false, NULL, defaultPrivilegedUser, defaultTimeout);
     if (!file)
         return false;
     return file->existsPhysicalPartFiles(0);
@@ -8498,9 +8517,9 @@ void CDistributedFileDirectory::addEntry(CDfsLogicalFileName &dlfn,IPropertyTree
     }
 }
 
-IDistributedFileIterator *CDistributedFileDirectory::getIterator(const char *wildname, bool includesuper, IUserDescriptor *user)
+IDistributedFileIterator *CDistributedFileDirectory::getIterator(const char *wildname, bool includesuper, IUserDescriptor *user,bool isPrivilegedUser)
 {
-    return new CDistributedFileIterator(this,wildname,includesuper,user);
+    return new CDistributedFileIterator(this,wildname,includesuper,user,isPrivilegedUser);
 }
 
 GetFileClusterNamesType CDistributedFileDirectory::getFileClusterNames(const char *_logicalname,StringArray &out)
@@ -11177,8 +11196,8 @@ DistributedFileCompareResult CDistributedFileDirectory::fileCompare(const char *
     StringBuffer msg;
     try
     {
-        Owned<IDistributedFile> file1 = lookup(lfn1, user, false, false, false, NULL, defaultTimeout);
-        Owned<IDistributedFile> file2 = lookup(lfn2, user, false, false, false, NULL, defaultTimeout);
+        Owned<IDistributedFile> file1 = lookup(lfn1, user, false, false, false, NULL, defaultPrivilegedUser, defaultTimeout);
+        Owned<IDistributedFile> file2 = lookup(lfn2, user, false, false, false, NULL, defaultPrivilegedUser, defaultTimeout);
         if (!file1)
         {
             errstr.appendf("File %s not found",lfn1);
@@ -11325,7 +11344,7 @@ DistributedFileCompareResult CDistributedFileDirectory::fileCompare(const char *
 bool CDistributedFileDirectory::filePhysicalVerify(const char *lfn, IUserDescriptor *user, bool includecrc, StringBuffer &errstr)
 {
     bool differs = false;
-    Owned<IDistributedFile> file = lookup(lfn, user, false, false, false, NULL, defaultTimeout);
+    Owned<IDistributedFile> file = lookup(lfn, user, false, false, false, NULL, defaultPrivilegedUser, defaultTimeout);
     if (!file)
     {
         errstr.appendf("Could not find file: %s",lfn);
@@ -12872,7 +12891,7 @@ bool CDistributedFileDirectory::removePhysicalPartFiles(const char *logicalName,
  */
 extern da_decl void removeLogical(const char *fname, IUserDescriptor *user) {
     if (queryDistributedFileDirectory().exists(fname, user)) {
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fname, user, true);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fname, user, true, false, false, nullptr, defaultPrivilegedUser);
         CDistributedFile *f = QUERYINTERFACE(file.get(),CDistributedFile);
         assert(f);
         f->detachLogical();

dali/base/dadfs.hpp

@@ -556,26 +556,28 @@ interface IDistributedFileDirectory: extends IInterface
 {
     virtual IDistributedFile *lookup(   const char *logicalname,
                                         IUserDescriptor *user,
-                                        bool writeaccess=false,
-                                        bool hold = false,
-                                        bool lockSuperOwner = false,
-                                        IDistributedFileTransaction *transaction=NULL, // transaction only used for looking up superfile sub files
+                                        bool writeaccess,
+                                        bool hold,
+                                        bool lockSuperOwner,
+                                        IDistributedFileTransaction *transaction, // transaction only used for looking up superfile sub file
+                                        bool privilegedUser,
                                         unsigned timeout=INFINITE
                                     ) = 0;  // links, returns NULL if not found
 
     virtual IDistributedFile *lookup(   CDfsLogicalFileName &logicalname,
                                         IUserDescriptor *user,
-                                        bool writeaccess=false,
-                                        bool hold = false,
-                                        bool lockSuperOwner = false,
-                                        IDistributedFileTransaction *transaction=NULL, // transaction only used for looking up superfile sub files
+                                        bool writeaccess,
+                                        bool hold,
+                                        bool lockSuperOwner,
+                                        IDistributedFileTransaction *transaction, // transaction only used for looking up superfile sub files
+                                        bool privilegedUser,
                                         unsigned timeout=INFINITE
                                     ) = 0;  // links, returns NULL if not found
 
     virtual IDistributedFile *createNew(IFileDescriptor *desc) = 0;
     virtual IDistributedFile *createExternal(IFileDescriptor *desc, const char *name) = 0;
 
-    virtual IDistributedFileIterator *getIterator(const char *wildname, bool includesuper, IUserDescriptor *user) = 0;
+    virtual IDistributedFileIterator *getIterator(const char *wildname, bool includesuper, IUserDescriptor *user,bool isPrivilegedUser) = 0;
             // wildname is in form scope/name and may contain wild components for either
     virtual IDFAttributesIterator *getDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive=true, bool includesuper=false, INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
     virtual IPropertyTreeIterator *getDFAttributesTreeIterator(const char *filters, DFUQResultField* localFilters,
@@ -760,7 +762,8 @@ enum DistributedFileSystemError
     DFSERR_ClusterAlreadyExists,
     DFSERR_LookupConnectionTimout,       // only raised if timeout specified on lookup etc.
     DFSERR_FailedToDeleteFile,
-    DFSERR_PassIterateFilesLimit
+    DFSERR_PassIterateFilesLimit,
+    DFSERR_RestrictedFileAccessDenied
 };
 
 
@@ -837,6 +840,8 @@ extern da_decl void ensureFileScope(const CDfsLogicalFileName &dlfn, unsigned ti
 
 extern da_decl bool checkLogicalName(const char *lfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg);
 
+constexpr bool defaultPrivilegedUser = true;
+constexpr bool defaultNonPrivilegedUser = false;
 
 #endif
 

dali/base/dautils.cpp

@@ -3041,7 +3041,7 @@ public:
         return dfile.get(); 
     }
 
-    bool init(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs, bool write)
+    bool init(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs, bool write, bool isPrivilegedUser)
     {
         fileExists = false;
         if (!onlydfs)
@@ -3064,7 +3064,7 @@ public:
             if (gotlocal)
             {
                 if (!write && !onlylocal) // MORE - this means the dali access checks not happening... maybe that's ok?
-                    dfile.setown(queryDistributedFileDirectory().lookup(lfn,user,write)); // MORE - if dFile is not null then arguably exists should be true
+                    dfile.setown(queryDistributedFileDirectory().lookup(lfn,user,write,false,false,nullptr,isPrivilegedUser)); // MORE - if dFile is not null then arguably exists should be true
                 Owned<IFile> file = getPartFile(0,0);
                 if (file.get())
                 {
@@ -3088,7 +3088,7 @@ public:
             }
             else
             {
-                dfile.setown(queryDistributedFileDirectory().lookup(lfn,user,write));
+                dfile.setown(queryDistributedFileDirectory().lookup(lfn,user,write,false,false,nullptr,isPrivilegedUser));
                 if (dfile.get())
                     return true;
             }
@@ -3240,10 +3240,10 @@ public:
 
 };
 
-ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs, bool iswrite)
+ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs, bool iswrite, bool isPrivilegedUser)
 {
     Owned<CLocalOrDistributedFile> ret = new CLocalOrDistributedFile();
-    if (ret->init(fname,user,onlylocal,onlydfs,iswrite))
+    if (ret->init(fname,user,onlylocal,onlydfs,iswrite,isPrivilegedUser))
         return ret.getClear();
     return NULL;
 }

dali/base/dautils.hpp

@@ -440,7 +440,7 @@ interface ILocalOrDistributedFile: extends IInterface
     virtual bool isExternal() const = 0;
 };
 
-extern da_decl ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs,bool iswrite=false);
+extern da_decl ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs,bool iswrite, bool isPrivilegedUser);
 
 typedef __int64 ConnectionId;
 

dali/daliadmin/daliadmin.cpp

@@ -558,7 +558,7 @@ static void dfsfile(const char *lname,IUserDescriptor *userDesc, UnsignedArray *
         outln(str.str());
     }
     else {
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,userDesc);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,userDesc,false,false,false,nullptr,defaultPrivilegedUser);
         if (file) {
             Owned<IFileDescriptor> fdesc = file->getFileDescriptor();
             Owned<IPropertyTree> t = createPTree("File");
@@ -590,8 +590,8 @@ static void setdfspartattr(const char *lname, unsigned partNum, const char *attr
         throw MakeStringException(0, "External file not supported");
     if (lfn.isForeign()) 
         throw MakeStringException(0, "Foreign file not supported");
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname, userDesc);
-    if (nullptr == file.get())
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname, userDesc, false, false, false, nullptr, defaultPrivilegedUser);
+    if (!file)
         throw MakeStringException(0, "Could not find file: '%s'", lname);
     if (file->querySuperFile())
         throw MakeStringException(0, "Cannot be used on a superfile");
@@ -869,7 +869,7 @@ static void dfsLs(const char *name, const char *options, bool safe = false)
 
 static void dfsmap(const char *lname, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,false,nullptr,defaultPrivilegedUser);
     if (!file) {
         UERRLOG("File %s not found",lname);
         return;
@@ -902,7 +902,7 @@ static int dfsexists(const char *lname,IUserDescriptor *user)
 
 static void dfsparents(const char *lname, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,true);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,true,nullptr,defaultPrivilegedUser);
     if (file) {
         Owned<IDistributedSuperFileIterator> iter = file->getOwningSuperFiles();
         ForEach(*iter) 
@@ -916,7 +916,7 @@ static void dfsunlink(const char *lname, IUserDescriptor *user)
 {
     for (;;)
     {
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,true);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,true,nullptr,defaultPrivilegedUser);
         if (!file)
         {
             UERRLOG("File '%s' not found", lname);
@@ -1043,7 +1043,7 @@ public:
 static int dfsverify(const char *name,CDateTime *cutoff, IUserDescriptor *user)
 {
     static CIpTable dafilesrvips;
-    Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,user);
+    Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,user,false,false,false,nullptr,defaultPrivilegedUser);
     if (!file) {
         UERRLOG("VERIFY: cannot find %s",name);
         return 1;
@@ -1162,7 +1162,7 @@ static int dfsverify(const char *name,CDateTime *cutoff, IUserDescriptor *user)
 
 static void setprotect(const char *filename, const char *callerid, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(filename,user);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(filename,user,false,false,false,nullptr,defaultPrivilegedUser);
     file->setProtect(callerid,true);
 }
 
@@ -1170,7 +1170,7 @@ static void setprotect(const char *filename, const char *callerid, IUserDescript
 
 static void unprotect(const char *filename, const char *callerid, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(filename,user);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(filename,user,false,false,false,nullptr,defaultPrivilegedUser);
     file->setProtect((strcmp(callerid,"*")==0)?NULL:callerid,false);
 }
 //=============================================================================
@@ -1623,7 +1623,7 @@ static offset_t getCompressedSize(IDistributedFile *file)
 
 static void dfscompratio (const char *lname, IUserDescriptor *user)
 {
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lname,user,false,false,false,nullptr,defaultPrivilegedUser);
     StringBuffer out;
     out.appendf("File %s ",lname);
     if (file) {
@@ -1806,7 +1806,7 @@ static void normalizeFileNames(IUserDescriptor *user, const char *name)
         Owned<IDistributedFile> dFile;
         try
         {
-            dFile.setown(queryDistributedFileDirectory().lookup(dlfn, user, true, false, false, nullptr, 30000)); // 30 sec timeout
+            dFile.setown(queryDistributedFileDirectory().lookup(dlfn, user, true, false, false, nullptr, defaultPrivilegedUser, 30000)); // 30 sec timeout
             if (!dFile)
                 UWARNLOG("Could not find file lfn = %s", dlfn.get());
         }
@@ -1981,7 +1981,7 @@ static void holdlock(const char *logicalFile, const char *mode, IUserDescriptor
         throw MakeStringException(0,"Invalid mode: %s", mode);
 
     PROGLOG("Looking up file: %s, mode=%s", logicalFile, mode);
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(logicalFile, userDesc, write, false, false, NULL, 5000);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(logicalFile, userDesc, write, false, false, NULL, defaultPrivilegedUser, 5000);
     if (!file)
     {
         UERRLOG("File not found: %s", logicalFile);

dali/dalidiag/dalidiag.cpp

@@ -278,7 +278,7 @@ void dirParts(const char *ip,const char *dir)
 
 void partInfo(const char *name,unsigned copy)
 {
-    Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
+    Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER, false, false, false, nullptr, defaultPrivilegedUser);
     if (f) {
         Owned<IDistributedFilePartIterator> parts = f->getIterator();
         unsigned partno = 0;

dali/datest/datest.cpp

@@ -261,7 +261,7 @@ void Test_SuperFile2()
             StringBuffer name(TEST_SUB_FILE);
             name.append(i+1);
             addTestFile(name.str(),i+2);
-            Owned<IDistributedFile> sbfile = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
+            Owned<IDistributedFile> sbfile = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
             printf("adding size = %" I64F "d\n",sbfile->getFileSize(false,false));
             sfile->addSubFile(name);
             printf("sfile size = %" I64F "d\n",sfile->getFileSize(false,false));
@@ -277,7 +277,7 @@ void Test_SuperFile2()
             for (i = 0;i<3;i++) {
                 StringBuffer name(TEST_SUB_FILE);
                 name.append(i+1);
-                Owned<IDistributedFile> sbfile = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
+                Owned<IDistributedFile> sbfile = queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
                 printf("removing size = %" I64F "d\n",sbfile->getFileSize(false,false));
                 sfile->removeSubFile(name,false);
                 printf("sfile size = %" I64F "d\n",sfile->getFileSize(false,false));
@@ -291,7 +291,7 @@ void Test_SuperFile2()
 void Test_PartIter()
 {
     unsigned start = msTick();
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup("nhtest::file_name_ssn20030805",UNKNOWN_USER);
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup("nhtest::file_name_ssn20030805",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     Owned<IDistributedFilePartIterator> parts = file->getIterator();
     ForEach(*parts) {
         IDistributedFilePart & thisPart = parts->query(); 
@@ -517,19 +517,19 @@ void Test_DFS()
     dfile->attach("nigel::test::testfile3",UNKNOWN_USER);
     dfile->Release();
     fdesc->Release();
-    IDistributedFile *f = queryDistributedFileDirectory().lookup("nigel::test::testfile2",UNKNOWN_USER);
+    IDistributedFile *f = queryDistributedFileDirectory().lookup("nigel::test::testfile2",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     if (!f)
         printf("failed 1");
     ::Release(f);
-    f = queryDistributedFileDirectory().lookup("nigel::zest::testfile1",UNKNOWN_USER);
+    f = queryDistributedFileDirectory().lookup("nigel::zest::testfile1",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     assertex(!f);
-    f = queryDistributedFileDirectory().lookup("nigel::test::zestfile1",UNKNOWN_USER);
+    f = queryDistributedFileDirectory().lookup("nigel::test::zestfile1",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     assertex(!f);
-    f = queryDistributedFileDirectory().lookup("nigel::test::testfile1",UNKNOWN_USER);
+    f = queryDistributedFileDirectory().lookup("nigel::test::testfile1",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     if (!f)
         printf("failed 2 ");
     ::Release(f);
-    f = queryDistributedFileDirectory().lookup("nigel::test::testfile3",UNKNOWN_USER);
+    f = queryDistributedFileDirectory().lookup("nigel::test::testfile3",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     if (!f)
         printf("failed 3");
     StringBuffer str;
@@ -624,7 +624,7 @@ void Test_DFSU()
     dfile->attach("nigel::test::testfile3u",UNKNOWN_USER);
     dfile->Release();
     fdesc->Release();
-    IDistributedFile *f = queryDistributedFileDirectory().lookup("nigel::test::testfile2u",UNKNOWN_USER);
+    IDistributedFile *f = queryDistributedFileDirectory().lookup("nigel::test::testfile2u",UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     if (!f)
         printf("failed 1");
     StringBuffer str;
@@ -3331,8 +3331,8 @@ void testDaliLog(bool listener)
 
 void testlockprop(const char *lfn)
 {
-    Owned<IDistributedFile> f1 = queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER);
-    Owned<IDistributedFile> f2 = queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER);
+    Owned<IDistributedFile> f1 = queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
+    Owned<IDistributedFile> f2 = queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser);
     f1->lockProperties();
     f1->unlockProperties();
     printf("done\n");

dali/datest/dfuwutest.cpp

@@ -564,7 +564,7 @@ void testRoxieCopies()
         queryDistributedFileDirectory().removeEntry(fn.str(),UNKNOWN_USER);
         file->attach(fn.str(),UNKNOWN_USER);
         file.clear();
-        file.setown(queryDistributedFileDirectory().lookup(fn.str(),UNKNOWN_USER));
+        file.setown(queryDistributedFileDirectory().lookup(fn.str(),UNKNOWN_USER,false,false,false,nullptr,defaultNonPrivilegedUser));
         Owned<IFileDescriptor> fdesc4 = file->getFileDescriptor();
         printDesc(fdesc4);
     }

dali/daunittest/dautdfs.cpp

@@ -33,7 +33,6 @@
 #define DFSUTSCOPE  "test::dfsut"
 #define DFSUTDIR    "test/dfsut"
 
-
 class TTestDFS : public CPPUNIT_NS::TestFixture
 {
     CPPUNIT_TEST_SUITE(TTestDFS);
@@ -55,7 +54,7 @@ class TTestDFS : public CPPUNIT_NS::TestFixture
     {
         StringArray superfiles;
         StringArray files;
-        Owned<IDistributedFileIterator> iter = dfsdir->getIterator(DFSUTSCOPE "::*", true, UNKNOWN_USER);
+        Owned<IDistributedFileIterator> iter = dfsdir->getIterator(DFSUTSCOPE "::*", true, UNKNOWN_USER, defaultNonPrivilegedUser);
         ForEach(*iter) {
             IDistributedFile &file = iter->query();
             if (file.querySuperFile())
@@ -172,7 +171,7 @@ protected:
         Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
         file->attach(DFSUTSCOPE "::testfile1", UNKNOWN_USER);
         file.clear();
-        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile1", UNKNOWN_USER));
+        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile1", UNKNOWN_USER, false, false, false, nullptr, true));
         CPPUNIT_ASSERT(file.get()!=NULL);
         CPPUNIT_ASSERT(file->numParts()==1);
         CPPUNIT_ASSERT(file->numCopies(0)==2);
@@ -210,7 +209,7 @@ protected:
         file.setown(queryDistributedFileDirectory().createNew(fdesc));
         file->attach(DFSUTSCOPE "::testfile2", UNKNOWN_USER);
         file.clear();
-        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile2", UNKNOWN_USER));
+        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile2", UNKNOWN_USER, false, false, false, nullptr, true));
         CPPUNIT_ASSERT(file.get()!=NULL);
         CPPUNIT_ASSERT(file->numParts()==8);
         unsigned pi;
@@ -254,7 +253,7 @@ protected:
         file.setown(queryDistributedFileDirectory().createNew(fdesc));
         file->attach(DFSUTSCOPE "::testfile3", UNKNOWN_USER);
         file.clear();
-        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile3", UNKNOWN_USER));
+        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile3", UNKNOWN_USER, false, false, false, nullptr, true));
         CPPUNIT_ASSERT(file.get()!=NULL);
         CPPUNIT_ASSERT(file->numParts()==8);
         for (pi=0;pi<8;pi++) {
@@ -312,7 +311,7 @@ protected:
         file.setown(queryDistributedFileDirectory().createNew(fdesc));
         file->attach(DFSUTSCOPE "::testfile4", UNKNOWN_USER);
         file.clear();
-        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile4", UNKNOWN_USER));
+        file.setown(dfsdir->lookup(DFSUTSCOPE "::testfile4", UNKNOWN_USER, false, false, false, nullptr, true));
         CPPUNIT_ASSERT(file.get()!=NULL);
         CPPUNIT_ASSERT(file->numParts()==8);
         for (pi=0;pi<8;pi++) {

dali/dfu/dfurepl.cpp

@@ -258,7 +258,7 @@ struct ReplicateFileItem: extends CInterface
             OERRLOG(LOGPFX "Cannot replicate foreign file %s",lfn);
             return;
         }
-        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(dlfn,userdesc);
+        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(dlfn,userdesc,false,false,false,nullptr,defaultPrivilegedUser);
         if (!dfile) {
             UWARNLOG(LOGPFX "Cannot find file %s, perhaps deleted",lfn);
             return;
@@ -332,7 +332,7 @@ struct ReplicateFileItem: extends CInterface
         }
         bool abort = true;
         try {
-            dfile.setown(queryDistributedFileDirectory().lookup(dlfn,userdesc));
+            dfile.setown(queryDistributedFileDirectory().lookup(dlfn,userdesc,false,false,false,nullptr,defaultPrivilegedUser));
             if (dfile) {
                 CDateTime newfiledt;
                 dfile->getModificationTime(newfiledt);

dali/dfu/dfurun.cpp

@@ -943,7 +943,7 @@ public:
         }
 
         // first see if target exists (and remove if does and overwrite specified)
-        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(dlfn,ctx.user,true);
+        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(dlfn,ctx.user,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile) {
             if (!ctx.superoptions->getOverwrite())
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
@@ -1222,7 +1222,8 @@ public:
                         }
                     }
                     srcFile.setown(fdir.lookup(tmp.str(),userdesc,
-                            (cmd==DFUcmd_move)||(cmd==DFUcmd_rename)||((cmd==DFUcmd_copy)&&multiclusterinsert)));
+                            (cmd==DFUcmd_move)||(cmd==DFUcmd_rename)||((cmd==DFUcmd_copy)&&multiclusterinsert),
+                            false,false,nullptr,true));
 
                     if (!srcFile)
                         throw MakeStringException(-1,"Source file %s could not be found",tmp.str());
@@ -1354,7 +1355,7 @@ public:
                             }
                             else if (multiclustermerge)
                             {
-                                dstFile.setown(fdir.lookup(tmp.str(),userdesc,true));
+                                dstFile.setown(fdir.lookup(tmp.str(),userdesc,true,false,false,nullptr,defaultPrivilegedUser));
                                 if (!dstFile)
                                     throw MakeStringException(-1,"Destination for merge %s does not exist",tmp.str());
                                 StringBuffer err;
@@ -1363,7 +1364,7 @@ public:
                             }
                             else
                             {
-                                Owned<IDistributedFile> oldfile = fdir.lookup(tmp.str(),userdesc,true);
+                                Owned<IDistributedFile> oldfile = fdir.lookup(tmp.str(),userdesc,true,false,false,nullptr,defaultPrivilegedUser);
                                 if (oldfile)
                                 {
                                     StringBuffer reason;
@@ -1565,7 +1566,7 @@ public:
                     destination->getLogicalName(toname);
                     if (toname.length()) {
                         unsigned start = msTick();
-                        Owned<IDistributedFile> newfile = fdir.lookup(toname.str(),userdesc,true);
+                        Owned<IDistributedFile> newfile = fdir.lookup(toname.str(),userdesc,true,false,false,nullptr,defaultPrivilegedUser);
                         if (newfile) {
                             // check for rename into multicluster
                             CDfsLogicalFileName dstlfn;

dali/dfu/dfuutil.cpp

@@ -510,7 +510,7 @@ public:
         CDfsLogicalFileName dstlfn;
         if (!dstlfn.setValidate(destfilename,true))
             throw MakeStringException(-1,"Logical name %s invalid",destfilename);
-        Owned<IDistributedFile> dfile = fdir->lookup(dstlfn,userdesc,true);
+        Owned<IDistributedFile> dfile = fdir->lookup(dstlfn,userdesc,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile) {
             ClusterPartDiskMapSpec spec = spec1;
             const char * kind = ftree->queryProp("Attr/@kind");
@@ -532,7 +532,7 @@ public:
         CDfsLogicalFileName dstlfn;
         if (!dstlfn.setValidate(destfilename,true))
             throw MakeStringException(-1,"Logical name %s invalid",destfilename);
-        Owned<IDistributedFile> dfile = fdir->lookup(dstlfn,userdesc,true);
+        Owned<IDistributedFile> dfile = fdir->lookup(dstlfn,userdesc,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile) {
             ClusterPartDiskMapSpec spec = spec1;
             const char * kind = ftree->queryProp("Attr/@kind");
@@ -667,7 +667,7 @@ public:
         }
 
         // first see if target exists (and remove if does and overwrite specified)
-        Owned<IDistributedFile> dfile = fdir->lookup(dlfn,userdesc,true);
+        Owned<IDistributedFile> dfile = fdir->lookup(dlfn,userdesc,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile) {
             if (!checkOverwrite(DALI_UPDATEF_REPLACE_FILE))
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
@@ -743,7 +743,7 @@ public:
         }
 
         // first see if target exists (and remove if does and overwrite specified)
-        Owned<IDistributedFile> dfile = fdir->lookup(dlfn,userdesc,true);
+        Owned<IDistributedFile> dfile = fdir->lookup(dlfn,userdesc,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile) {
             if (!checkOverwrite(DALI_UPDATEF_REPLACE_FILE))
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
@@ -881,7 +881,7 @@ public:
         }
 
         //see if target already exists
-        Owned<IDistributedFile> dfile = fdir->lookup(dlfn, userdesc, true);
+        Owned<IDistributedFile> dfile = fdir->lookup(dlfn, userdesc, true, false, false, nullptr, defaultPrivilegedUser);
         if (dfile)
         {
             if (!checkOverwrite(DALI_UPDATEF_SUBFILE_MASK))
@@ -1018,7 +1018,7 @@ public:
 
     StringBuffer &getFileXML(const char *lfn, StringBuffer &out, IUserDescriptor *user)
     {
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn, user);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn, user, false, false, false, nullptr, defaultPrivilegedUser);
         if (!file) {
             INamedGroupStore  &grpstore= queryNamedGroupStore();
             Owned<IGroup> grp = grpstore.lookup(lfn);
@@ -1122,7 +1122,7 @@ public:
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getUrlStr(s).str());
         }
         // first see if target exists (and remove if does and overwrite specified)
-        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(lfn,user,true);
+        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(lfn,user,true,false,false,nullptr,defaultPrivilegedUser);
         if (dfile)
         {
             if (!overwrite)

dali/dfu/dfuwu.cpp

@@ -774,7 +774,7 @@ public:
                     parent->getPassword(password);
                 }
                 userdesc->set(username.str(),password.str());
-                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,userdesc);
+                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,userdesc,false,false,false,nullptr,defaultPrivilegedUser);
                 if (file)
                     return file->getFileDescriptor();
             }

dali/dfuXRefLib/dfuxreflib.cpp

@@ -709,7 +709,7 @@ struct CLogicalNameEntry: public CInterface
     bool remove(IUserDescriptor *user)
     {
         IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
-        Owned<IDistributedFile> file = fdir.lookup(lname.get(),user);
+        Owned<IDistributedFile> file = fdir.lookup(lname.get(),user, false, false, false, nullptr, defaultPrivilegedUser);
         if (!file)
             return false;
         file->detach();
@@ -1774,7 +1774,7 @@ class CXRefManager: public CXRefManagerBase
         ForEachItemIn(i,logicalnamelist) {
             CLogicalNameEntry &item = logicalnamelist.item(i);
             if (!item.done&&item.nummismatchedsizes) {
-                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(item.lname.get());
+                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(item.lname.get(), UNKNOWN_USER, false, false, false, nullptr, defaultPrivilegedUser);
                 if (file) {
                     outf("checking %s\n",item.lname.get());
                     Owned<IDistributedFilePartIterator> partiter = file->getIterator();
@@ -2272,7 +2272,7 @@ class CXRefManager: public CXRefManagerBase
             ForEachItemIn(i,logicalnamelist) {
                 CLogicalNameEntry &item = logicalnamelist.item(i);
                 if (item.unknowngrp||item.mismatchgrp.get()||item.missinggrp) {
-                    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(item.lname.get(),UNKNOWN_USER);
+                    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(item.lname.get(),UNKNOWN_USER, false, false, false, nullptr, defaultPrivilegedUser);
                     if (file) {
                         if (item.missinggrp)
                             outf("WARNING: Missing group for %s\n",item.lname.get());

dali/fuse/dafuse.cpp

@@ -517,7 +517,7 @@ class CFuseDaliDFS: public CFuseBase
                 stbuf->st_nlink = dci->scopes.ordinality()+2;
             }
             else {
-                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,user,false);
+                Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,user,false,false,false,nullptr,defaultPrivilegedUser);
                 if (file) {
                     stbuf->st_mode = S_IFREG | 0444;
                     stbuf->st_nlink = 1;
@@ -611,7 +611,7 @@ class CFuseDaliDFS: public CFuseBase
         if (!pathToLFN(path,lfn))
             return -ENOENT;
         try {
-            Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,user);
+            Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,user,false,false,false,nullptr,defaultPrivilegedUser);
             if (!file)
                 return -ENOENT;
             if((info->flags & 3) != O_RDONLY ) 
@@ -648,7 +648,7 @@ class CFuseDaliDFS: public CFuseBase
                 CDfsLogicalFileName lfn;
                 if (!pathToLFN(path,lfn))
                     return -ENOENT;
-                file.setown(queryDistributedFileDirectory().lookup(lfn,user));
+                file.setown(queryDistributedFileDirectory().lookup(lfn,user,false,false,false,nullptr,defaultPrivilegedUser));
                 if (!file)
                     return -ENOENT;
             }

dali/sasha/saverify.cpp

@@ -221,7 +221,7 @@ public:
 
     void verifyFile(const char *name,CDateTime *cutoff)
     {
-        Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
+        Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser);
         if (!file)
             return;
         IPropertyTree &fileprops = file->queryAttributes();
@@ -343,7 +343,7 @@ public:
             }
         }
         if (!stopped) {
-            file.setown(queryDistributedFileDirectory().lookup(name,UNKNOWN_USER));
+            file.setown(queryDistributedFileDirectory().lookup(name,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser));
             if (!file)
                 return;
             if (afor.ok) {

dali/sasha/saxref.cpp

@@ -1498,7 +1498,7 @@ public:
             }
             Owned<IDistributedFile> file;
             try {
-                file.setown(queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER));
+                file.setown(queryDistributedFileDirectory().lookup(lfn,UNKNOWN_USER,false,false,false,nullptr,defaultPrivilegedUser));
             }
             catch (IException *e) {
                 EXCLOG(e,"CNewXRefManager::listLost");

ecl/eclagent/agentctx.hpp

@@ -92,7 +92,7 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual IWorkUnit *updateWorkUnit() const = 0;
     virtual void unlockWorkUnit() = 0;
     
-    virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt=NULL, bool optional=false, bool noteRead=true, bool write=false, StringBuffer * expandedlfn=NULL) = 0;
+    virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt, bool optional, bool noteRead, bool write, StringBuffer * expandedlfn, bool isPrivilegedUser) = 0;
     virtual StringBuffer & getTempfileBase(StringBuffer & buff) = 0;
     virtual const char *noteTemporaryFile(const char *fname) = 0;
     virtual const char *noteTemporaryFilespec(const char *fname) = 0;

ecl/eclagent/eclagent.cpp

@@ -1378,7 +1378,7 @@ bool EclAgent::expandLogicalName(StringBuffer & fullname, const char * logicalNa
     return useScope;
 }
 
-ILocalOrDistributedFile *EclAgent::resolveLFN(const char *fname, const char *errorTxt, bool optional, bool noteRead, bool isWrite, StringBuffer * expandedlfn)
+ILocalOrDistributedFile *EclAgent::resolveLFN(const char *fname, const char *errorTxt, bool optional, bool noteRead, bool isWrite, StringBuffer * expandedlfn, bool isPrivilegedUser)
 {
     StringBuffer lfn;
     expandLogicalFilename(lfn, fname, queryWorkUnit(), resolveFilesLocally, false);
@@ -1398,7 +1398,7 @@ ILocalOrDistributedFile *EclAgent::resolveLFN(const char *fname, const char *err
     }
     if (expandedlfn)
         *expandedlfn = lfn;
-    Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(lfn.str(), queryUserDescriptor(), resolveFilesLocally, !resolveFilesLocally, isWrite);
+    Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(lfn.str(), queryUserDescriptor(), resolveFilesLocally, !resolveFilesLocally, isWrite, isPrivilegedUser);
     if (ldFile)
     {
         IDistributedFile * dFile = ldFile->queryDistributedFile();
@@ -1456,7 +1456,7 @@ bool EclAgent::fileExists(const char *name)
     StringBuffer lfn;
     expandLogicalName(lfn, name);
 
-    Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn.str(),queryUserDescriptor());
+    Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn.str(),queryUserDescriptor(), false, false, false, nullptr, defaultPrivilegedUser);
     if (f) 
         return true;
     return false;
@@ -2580,7 +2580,7 @@ unsigned __int64 EclAgent::getDatasetHash(const char * logicalName, unsigned __i
         return crc;
     }
 
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fullname.str(),queryUserDescriptor());
+    Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fullname.str(),queryUserDescriptor(), false, false, false, nullptr, defaultPrivilegedUser);
     if (file)
     {
         WorkunitUpdate wu = updateWorkUnit();
@@ -2905,7 +2905,7 @@ restart:     // If things change beneath us as we are deleting, repeat the proce
                 MilliSleep(PERSIST_LOCK_SLEEP + (getRandom()%PERSIST_LOCK_SLEEP));
                 persistLock.setown(getPersistReadLock(goer));
             }
-            Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(goer, queryUserDescriptor(), true);
+            Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(goer, queryUserDescriptor(), true, false, false, nullptr, defaultPrivilegedUser);
             if (!f)
                 goto restart; // Persist has been deleted since last checked - repeat the whole process
             const char *newAccessTime = f->queryAttributes().queryProp("@accessed");
@@ -2995,7 +2995,7 @@ char * EclAgent::getGroupName()
 
 char * EclAgent::queryIndexMetaData(char const * lfn, char const * xpath)
 {
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFN(lfn, "IndexMetaData");
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFN(lfn, "IndexMetaData", false, true, false, nullptr, defaultPrivilegedUser);
     IDistributedFile * dFile = ldFile->queryDistributedFile();
     if (!dFile)
         return NULL;
@@ -3084,7 +3084,7 @@ char *EclAgent::getFilePart(const char *lfn, bool create)
     }
     else
     {
-        Owned<ILocalOrDistributedFile> ldFile = resolveLFN(lfn, "l2p", false, false);
+        Owned<ILocalOrDistributedFile> ldFile = resolveLFN(lfn, "l2p", false, false, false, nullptr, defaultPrivilegedUser);
         if (!ldFile)
             return NULL;
         unsigned numParts = ldFile->numParts();

ecl/eclagent/eclagent.ipp

@@ -167,9 +167,9 @@ public:
     {
         ctx->unlockWorkUnit();
     }
-    virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt, bool optional, bool noteRead, bool write, StringBuffer * expandedlfn)
+    virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt, bool optional, bool noteRead, bool write, StringBuffer * expandedlfn, bool isPrivilegedUser)
     {
-        return ctx->resolveLFN(logicalName, errorTxt, optional, noteRead, write, expandedlfn);
+        return ctx->resolveLFN(logicalName, errorTxt, optional, noteRead, write, expandedlfn, isPrivilegedUser);
     }
     virtual StringBuffer & getTempfileBase(StringBuffer & buff)
     {
@@ -578,7 +578,7 @@ public:
     //virtual void logException(IEclException *e);  
     virtual char *resolveName(const char *in, char *out, unsigned outlen);
     virtual void logFileAccess(IDistributedFile * file, char const * component, char const * type);
-    virtual ILocalOrDistributedFile  *resolveLFN(const char *logicalName, const char *errorTxt=NULL, bool optional=false, bool noteRead=true, bool write=false, StringBuffer * expandedlfn=NULL);
+    virtual ILocalOrDistributedFile  *resolveLFN(const char *logicalName, const char *errorTxt, bool optional, bool noteRead, bool write, StringBuffer * expandedlfn, bool isPrivilegedUser);
 
     virtual void executeThorGraph(const char * graphName);
     virtual void executeGraph(const char * graphName, bool realThor, size32_t parentExtractSize, const void * parentExtract);
@@ -787,6 +787,7 @@ public:
     bool onlyUpdateIfChanged;
     bool alreadyUpdated;
     bool isEof;
+    bool isCodeSigned = false;
     unsigned whichBranch;
     CIArrayOf<EclGraphElement> branches;
     UnsignedArray branchIndexes;

ecl/eclagent/eclgraph.cpp

@@ -392,7 +392,7 @@ bool EclGraphElement::alreadyUpToDate(IAgentContext & agent)
         UNIMPLEMENTED;
     }
 
-    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(filename.get(), "Read", true, false, false);
+    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(filename.get(), "Read", true, false, false, nullptr, isCodeSigned);
     if (!ldFile)
         return false;
     IDistributedFile * f = ldFile->queryDistributedFile();
@@ -421,6 +421,8 @@ bool EclGraphElement::alreadyUpToDate(IAgentContext & agent)
 void EclGraphElement::createFromXGMML(ILoadedDllEntry * dll, IPropertyTree * _node)
 {
     node.set(_node);
+    if (node)
+        isCodeSigned = isActivityCodeSigned(*node);
     kind = (ThorActivityKind)node->getPropInt("att[@name=\"_kind\"]/@value", TAKnone);
     id = node->getPropInt("@id", 0);
 

ecl/eclcc/eclcc.cpp

@@ -2333,7 +2333,7 @@ IHqlExpression *EclCC::lookupDFSlayout(const char *filename, IErrorReceiver &err
         // Look up the file in Dali
         try
         {
-            Owned<IDistributedFile> dfsFile = queryDistributedFileDirectory().lookup(filename, udesc, false, false);
+            Owned<IDistributedFile> dfsFile = queryDistributedFileDirectory().lookup(filename, udesc, false, false, false, nullptr, defaultPrivilegedUser);
             if (dfsFile)
             {
                 const char *recordECL = dfsFile->queryAttributes().queryProp("ECL");

ecl/hthor/hthor.cpp

@@ -158,9 +158,9 @@ const void * CRowBuffer::next()
 }
 
 
-ILocalOrDistributedFile *resolveLFNFlat(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional)
+ILocalOrDistributedFile *resolveLFNFlat(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional, bool isPrivilegedUser)
 {
-    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(logicalName, errorTxt, optional);
+    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(logicalName, errorTxt, optional, true, false, nullptr, isPrivilegedUser);
     if (!ldFile)
         return nullptr;
     IDistributedFile *dFile = ldFile->queryDistributedFile();
@@ -444,7 +444,7 @@ void CHThorDiskWriteActivity::resolve()
     assertex(mangledHelperFileName.str());
     if((helper.getFlags() & (TDXtemporary | TDXjobtemp)) == 0)
     {
-        Owned<ILocalOrDistributedFile> f = agent.resolveLFN(mangledHelperFileName.str(),"Cannot write, invalid logical name",true,false,true,&lfn);
+        Owned<ILocalOrDistributedFile> f = agent.resolveLFN(mangledHelperFileName.str(),"Cannot write, invalid logical name",true,false,true,&lfn,defaultPrivilegedUser);
         if (f)
         {
             if (f->queryDistributedFile())
@@ -1050,7 +1050,7 @@ CHThorIndexWriteActivity::CHThorIndexWriteActivity(IAgentContext &_agent, unsign
     expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
     if (!agent.queryResolveFilesLocally())
     {
-        Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), true);
+        Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), true, false, false, nullptr, defaultNonPrivilegedUser);
 
         if (f)
         {
@@ -8084,12 +8084,13 @@ CHThorDiskReadBaseActivity::CHThorDiskReadBaseActivity(IAgentContext &_agent, un
     expectedDiskMeta = helper.queryDiskRecordSize();
     projectedDiskMeta = helper.queryProjectedDiskRecordSize();
     actualDiskMeta.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
-
+    isCodeSigned = false;
     if (_node)
     {
         const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layoutTranslation']/@value");
         if (recordTranslationModeHintText)
             recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+        isCodeSigned = isActivityCodeSigned(*_node);
     }
 }
 
@@ -8166,7 +8167,7 @@ void CHThorDiskReadBaseActivity::resolve()
     }
     else
     {
-        ldFile.setown(resolveLFNFlat(agent, mangledHelperFileName.str(), "Read", 0 != (helper.getFlags() & TDRoptional)));
+        ldFile.setown(resolveLFNFlat(agent, mangledHelperFileName.str(), "Read", 0 != (helper.getFlags() & TDRoptional), isCodeSigned));
         if ( mangledHelperFileName.charAt(0) == '~')
             logicalFileName.set(mangledHelperFileName.str()+1);
         else
@@ -10436,11 +10437,13 @@ CHThorNewDiskReadBaseActivity::CHThorNewDiskReadBaseActivity(IAgentContext &_age
     expectedDiskMeta = helper.queryDiskRecordSize();
     projectedDiskMeta = helper.queryProjectedDiskRecordSize();
     formatOptions.setown(createPTree());
+    isCodeSigned = false;
     if (_node)
     {
         const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layoutTranslation']/@value");
         if (recordTranslationModeHintText)
             recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+        isCodeSigned = isActivityCodeSigned(*_node);
     }
 
     PropertyTreeXmlWriter writer(formatOptions);
@@ -10524,7 +10527,7 @@ void CHThorNewDiskReadBaseActivity::resolveFile()
     }
     else
     {
-        ldFile.setown(resolveLFNFlat(agent, mangledHelperFileName.str(), "Read", 0 != (helper.getFlags() & TDRoptional)));
+        ldFile.setown(resolveLFNFlat(agent, mangledHelperFileName.str(), "Read", 0 != (helper.getFlags() & TDRoptional), isCodeSigned));
         if ( mangledHelperFileName.charAt(0) == '~')
             logicalFileName = mangledHelperFileName.str()+1;
         else

ecl/hthor/hthor.ipp

@@ -2254,6 +2254,7 @@ protected:
     MemoryAttr encryptionkey;
     bool persistent;
     bool grouped;
+    bool isCodeSigned = false;
     enum ReadType:byte { rt_unknown, rt_binary, rt_csv, rt_xml } readType = rt_unknown;
     RecordTranslationMode recordTranslationModeHint = RecordTranslationMode::Unspecified;
     unsigned __int64 stopAfter = 0;
@@ -2958,7 +2959,7 @@ protected:
     bool finishedParts = false;
     unsigned __int64 stopAfter = 0;
     unsigned __int64 offsetOfPart = 0;
-
+    bool isCodeSigned = false;
     void close();
     void resolveFile();
     virtual void verifyRecordFormatCrc();
@@ -3059,7 +3060,7 @@ extern HTHOR_API IHThorActivity * create ## NAME ## Activity(IAgentContext &_age
 extern HTHOR_API IHThorActivity * create ## NAME ## Activity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThor ## NAME ## Arg &arg, ThorActivityKind kind, EXTRATYPE extra) \
 {   return new CHThor ## NAME ##Activity(_agent, _activityId, _subgraphId, arg, kind, extra); }
 
-extern ILocalOrDistributedFile *resolveLFNFlat(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional);
+extern ILocalOrDistributedFile *resolveLFNFlat(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional, bool privilegedUser);
 
 #endif // HTHOR_IPP_INCL
 

ecl/hthor/hthorkey.cpp

@@ -1073,9 +1073,9 @@ IInputSteppingMeta * CHThorIndexReadActivity::querySteppingMeta()
 }
 
 
-ILocalOrDistributedFile *resolveLFNIndex(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional, bool noteRead=true, bool write=false)
+ILocalOrDistributedFile *resolveLFNIndex(IAgentContext &agent, const char *logicalName, const char *errorTxt, bool optional, bool noteRead, bool write, bool isPrivilegedUser)
 {
-    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(logicalName, errorTxt, optional, noteRead, write);
+    Owned<ILocalOrDistributedFile> ldFile = agent.resolveLFN(logicalName, errorTxt, optional, noteRead, write, nullptr, isPrivilegedUser);
     if (!ldFile)
         return nullptr;
     IDistributedFile *dFile = ldFile->queryDistributedFile();
@@ -1089,7 +1089,7 @@ extern HTHOR_API IHThorActivity *createIndexReadActivity(IAgentContext &_agent,
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(arg.getFileName());
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexRead", 0 != (arg.getFlags() & TIRoptional));
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexRead", 0 != (arg.getFlags() & TIRoptional),true, false, defaultPrivilegedUser);
     Linked<IDistributedFile> dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
     if (!dFile)
     {
@@ -1271,7 +1271,7 @@ extern HTHOR_API IHThorActivity *createIndexNormalizeActivity(IAgentContext &_ag
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(arg.getFileName());
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexNormalize", 0 != (arg.getFlags() & TIRoptional),true,true);
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexNormalize", 0 != (arg.getFlags() & TIRoptional),true,true,defaultPrivilegedUser);
     Linked<IDistributedFile> dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
     if (!dFile)
     {
@@ -1388,7 +1388,7 @@ extern HTHOR_API IHThorActivity *createIndexAggregateActivity(IAgentContext &_ag
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(arg.getFileName());
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexAggregate", 0 != (arg.getFlags() & TIRoptional));
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexAggregate", 0 != (arg.getFlags() & TIRoptional), true, false, defaultPrivilegedUser);
     Linked<IDistributedFile> dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
     if (!dFile)
     {
@@ -1544,7 +1544,7 @@ extern HTHOR_API IHThorActivity *createIndexCountActivity(IAgentContext &_agent,
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(arg.getFileName());
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexCount", 0 != (arg.getFlags() & TIRoptional));
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexCount", 0 != (arg.getFlags() & TIRoptional), true, false, defaultPrivilegedUser);
     Linked<IDistributedFile> dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
     if (!dFile)
     {
@@ -1655,7 +1655,7 @@ extern HTHOR_API IHThorActivity *createIndexGroupAggregateActivity(IAgentContext
 {
     // A logical filename for the key should refer to a single physical file - either the TLK or a monolithic key
     OwnedRoxieString lfn(arg.getFileName());
-    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexGroupAggregate", 0 != (arg.getFlags() & TIRoptional));
+    Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(_agent, lfn, "IndexGroupAggregate", 0 != (arg.getFlags() & TIRoptional), true, false, defaultPrivilegedUser);
     Linked<IDistributedFile> dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
     if (!dFile)
     {
@@ -2136,11 +2136,13 @@ class CHThorThreadedActivityBase : public CHThorActivityBase, implements IThread
     };
 
 public:
-    CHThorThreadedActivityBase (IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, IHThorFetchContext &_fetch, ThorActivityKind _kind, IRecordSize *diskSize)
+    CHThorThreadedActivityBase (IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, IHThorFetchContext &_fetch, ThorActivityKind _kind, IRecordSize *diskSize, IPropertyTree *_node)
         : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), fetch(_fetch)
     {
         exception = NULL;
         rowLimit = 0;
+        if (_node)
+            isCodeSigned = isActivityCodeSigned(*_node);
     }
 
     virtual ~CHThorThreadedActivityBase ()
@@ -2207,6 +2209,7 @@ protected:
     Owned<InputHandler> inputThread;
     unsigned numParts;
     unsigned __int64 rowLimit;
+    bool isCodeSigned = false;
     Owned<IThreadPool> threadPool;
     CriticalSection pendingCrit;
     IException *exception;
@@ -2267,7 +2270,7 @@ public:
         OwnedRoxieString lfn(fetch.getFileName());
         if (lfn.get())
         {
-            Owned<ILocalOrDistributedFile> ldFile = resolveLFNFlat(agent, lfn, "Fetch", 0 != (fetch.getFetchFlags() & FFdatafileoptional));
+            Owned<ILocalOrDistributedFile> ldFile = resolveLFNFlat(agent, lfn, "Fetch", 0 != (fetch.getFetchFlags() & FFdatafileoptional), isCodeSigned);
             IDistributedFile * dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
             if(dFile)
             {
@@ -2294,7 +2297,7 @@ class CHThorFetchActivityBase : public CHThorThreadedActivityBase, public IFetch
 {
 public:
     CHThorFetchActivityBase(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorArg &_arg, IHThorFetchContext &_fetch, ThorActivityKind _kind, IRecordSize *diskSize, IPropertyTree *_node)
-      : CHThorThreadedActivityBase (_agent, _activityId, _subgraphId, _arg, _fetch, _kind, diskSize)
+      : CHThorThreadedActivityBase (_agent, _activityId, _subgraphId, _arg, _fetch, _kind, diskSize, _node)
     {
         pendingSeq = 0;
         signalSeq = 0;
@@ -2583,7 +2586,7 @@ public:
         ICsvParameters * csvInfo = _arg.queryCsvParameters();
 
         OwnedRoxieString lfn(fetch.getFileName());
-        Owned<ILocalOrDistributedFile> ldFile = resolveLFNFlat(agent, lfn, "CsvFetch", 0 != (_arg.getFetchFlags() & FFdatafileoptional));
+        Owned<ILocalOrDistributedFile> ldFile = resolveLFNFlat(agent, lfn, "CsvFetch", 0 != (_arg.getFetchFlags() & FFdatafileoptional), isCodeSigned);
         IDistributedFile * dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
         const char * quotes = NULL;
         const char * separators = NULL;
@@ -3481,9 +3484,10 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
     Owned<IOutputMetaData> actualDiskMeta;           // only one disk layout is permitted
     Owned<const IDynamicTransform> translator;
     RecordTranslationMode recordTranslationModeHint = RecordTranslationMode::Unspecified;
+    bool isCodeSigned = false;
 public:
     CHThorKeyedJoinActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorKeyedJoinArg &_arg, ThorActivityKind _kind, IPropertyTree *_node)
-        : CHThorThreadedActivityBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind, _arg.queryDiskRecordSize()), helper(_arg)
+        : CHThorThreadedActivityBase(_agent, _activityId, _subgraphId, _arg, _arg, _kind, _arg.queryDiskRecordSize(), _node), helper(_arg)
     {
         atomic_set(&prefiltered, 0);
         atomic_set(&postfiltered, 0);
@@ -3496,6 +3500,7 @@ public:
             const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layoutTranslation']/@value");
             if (recordTranslationModeHintText)
                 recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+            isCodeSigned = isActivityCodeSigned(*_node);
         }
     }
 
@@ -4023,7 +4028,7 @@ public:
     virtual void start()
     {
         OwnedRoxieString lfn(helper.getIndexFileName());
-        Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(agent, lfn, "KeyedJoin", 0 != (helper.getJoinFlags() & JFindexoptional));
+        Owned<ILocalOrDistributedFile> ldFile = resolveLFNIndex(agent, lfn, "KeyedJoin", 0 != (helper.getJoinFlags() & JFindexoptional), true, false, isCodeSigned);
         dFile = ldFile ? ldFile->queryDistributedFile() : NULL;
         if (dFile)
         {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1105,7 +1105,7 @@ int CWsDfuEx::superfileAction(IEspContext &context, const char* action, const ch
 
     if (!autocreatesuper)
     {//a file lock created by the lookup() will be released after '}'
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(superfile, userdesc.get(), true);
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(superfile, userdesc.get(), true, false, false, nullptr, defaultPrivilegedUser);
         if (existingSuperfile)
         {
             if (!df)
@@ -1257,7 +1257,7 @@ bool doRemoveFileFromSuperfiles(const char *lfn, IUserDescriptor *userdesc, Stri
     StringArray emptySuperFiles;
     IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
     {
-        Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true);
+        Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true, false, false, nullptr, defaultPrivilegedUser);
         if(!df)
             return false;
         Owned<IDistributedSuperFileIterator> supers = df->getOwningSuperFiles();
@@ -1316,7 +1316,7 @@ DeleteActionResult doDeleteFile(const char *fn, IUserDescriptor *userdesc, Strin
     {
         IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
         {
-            Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true);
+            Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true, false, false, nullptr, defaultPrivilegedUser);
             if(!df)
             {
                 PROGLOG("CWsDfuEx::DFUDeleteFiles: %s not found", lfn);
@@ -1495,7 +1495,7 @@ bool CWsDfuEx::onDFUArrayAction(IEspContext &context, IEspDFUArrayActionRequest
 
             try
             {
-                Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(curfile, userdesc.get(), true);
+                Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(curfile, userdesc.get(), true, false, false, nullptr, defaultPrivilegedUser);
                 if (df)
                 {
                     if (subfiles.length() > 0)
@@ -1614,7 +1614,7 @@ IHqlExpression * getEclRecordDefinition(const char * ecl)
 
 IHqlExpression * getEclRecordDefinition(IUserDescriptor* udesc, const char* FileName)
 {
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(FileName, udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(FileName, udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if(!df)
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",FileName);
     if(!df->queryAttributes().hasProp("ECL"))
@@ -1696,7 +1696,7 @@ bool CWsDfuEx::onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRe
             userdesc.setown(createUserDescriptor());
             userdesc->set(userId, context.queryPassword(), context.querySignature());
         }
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userdesc);
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userdesc, false, false, false, nullptr, defaultPrivilegedUser);
         if (!df)
             throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",fileName);
 
@@ -1772,7 +1772,7 @@ void CWsDfuEx::getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuf
 
 bool CWsDfuEx::checkFileContent(IEspContext &context, IUserDescriptor* udesc, const char * logicalName, const char * cluster)
 {
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (!df)
         return false;
 
@@ -1971,7 +1971,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
             return;
     }
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(name, udesc, false, false, true); // lock super-owners
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(name, udesc, false, false, true, nullptr, defaultPrivilegedUser); // lock super-owners
     if(!df)
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",name);
 
@@ -4942,7 +4942,7 @@ bool CWsDfuEx::onListHistory(IEspContext &context, IEspListHistoryRequest &req,
 
         MemoryBuffer xmlmap;
         IArrayOf<IEspHistory> arrHistory;
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(req.getName(),userdesc.get());
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(req.getName(),userdesc.get(), false, false, false, nullptr, defaultPrivilegedUser);
         if (file)
         {
             IPropertyTree *history = file->queryHistory();
@@ -4990,7 +4990,7 @@ bool CWsDfuEx::onEraseHistory(IEspContext &context, IEspEraseHistoryRequest &req
 
         MemoryBuffer xmlmap;
         IArrayOf<IEspHistory> arrHistory;
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(req.getName(),userdesc.get());
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(req.getName(),userdesc.get(),false,false,false,nullptr,defaultPrivilegedUser);
         if (file)
         {
             IPropertyTree *history = file->queryHistory();
@@ -5597,7 +5597,7 @@ int CWsDfuEx::GetIndexData(IEspContext &context, bool bSchemaOnly, const char* i
     {
         userdesc.setown(createUserDescriptor());
         userdesc->set(username.str(), context.queryPassword(), context.querySignature());
-        df.setown(queryDistributedFileDirectory().lookup(indexName, userdesc));
+        df.setown(queryDistributedFileDirectory().lookup(indexName, userdesc, false, false, false, nullptr, defaultPrivilegedUser));
         if(!df)
             throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Could not find file %s.", indexName);
 
@@ -5909,7 +5909,7 @@ void CWsDfuEx::dFUFileAccessCommon(IEspContext &context, const CDfsLogicalFileNa
 
     checkLogicalName(fileName, userDesc, true, false, false, nullptr); // check for read permissions
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userDesc, false, false, true, nullptr, lockTimeoutMs); // lock super-owners
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userDesc, false, false, true, nullptr, defaultPrivilegedUser, lockTimeoutMs); // lock super-owners
     if (!df)
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file '%s'.", fileName.str());
 
@@ -6436,7 +6436,7 @@ bool CWsDfuEx::onDFUFilePublish(IEspContext &context, IEspDFUFilePublishRequest
             userDesc->set(userId.str(), context.queryPassword(), context.querySignature());
             fileDesc->queryProperties().setProp("@owner", userId);
         }
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(newFileName, userDesc, false, false, true, nullptr, req.getLockTimeoutMs());
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(newFileName, userDesc, false, false, true, nullptr, defaultPrivilegedUser, req.getLockTimeoutMs());
         if (df)
         {
             if (!req.getOverwrite())

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -199,7 +199,7 @@ void CWsDfuXRefEx::readLostFileQueryResult(IEspContext &context, StringBuffer &b
 
         try
         {
-            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userDesc, false, false, false, NULL, 0);
+            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userDesc, false, false, false, NULL, defaultPrivilegedUser, 0);
             if(df)
                 item.addPropInt64("Size", queryDistributedFileSystem().getSize(df));
         }

esp/services/ws_fs/ws_fsBinding.cpp

@@ -131,7 +131,7 @@ int CFileSpraySoapBindingEx::onGetInstantQuery(IEspContext &context, CHttpReques
                     {
                         if (stricmp(method, "CopyInput") == 0)
                         {
-                            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(sourceLogicalFile.str(), userdesc.get());
+                            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(sourceLogicalFile.str(), userdesc.get(), false, false, false, nullptr, defaultPrivilegedUser);
                             if(!df)
                             {
                                 throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Could not find file %s.",sourceLogicalFile.str());

esp/services/ws_fs/ws_fsService.cpp

@@ -645,7 +645,7 @@ StringBuffer& getNodeGroupFromLFN(StringBuffer& nodeGroup, const char* lfn, cons
         udesc->set(username, passwd);
     }
 
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn, udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn, udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (!df)
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST, "Failed to find file: %s", lfn);
     return df->getClusterGroupName(0, nodeGroup);
@@ -2513,7 +2513,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
             logicalName.setForeign(ep,false);
         }
 
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(logicalName, udesc);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(logicalName, udesc, false, false, false, nullptr, defaultPrivilegedUser);
         if (!file)
             throw MakeStringException(ECLWATCH_FILE_NOT_EXIST, "Failed to find file: %s", logicalName.get());
 

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -105,7 +105,7 @@ const unsigned roxieQueryRoxieTimeOut = 60000;
 
 bool isFileKnownOnCluster(const char *logicalname, IConstWUClusterInfo *clusterInfo, IUserDescriptor* userdesc)
 {
-    Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true);
+    Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true, false, false, nullptr, defaultPrivilegedUser);
     if (dst)
     {
         SCMStringBuffer processName;

esp/services/ws_sql/SQL2ECL/HPCCFileCache.cpp

@@ -133,7 +133,7 @@ bool HPCCFileCache::updateHpccFileDescription(const char * filename, const char
 
         //queryDistributedFileDirectory returns singleton
         IDistributedFileDirectory & dfd = queryDistributedFileDirectory();
-        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc);
+        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc, false, false, false, nullptr, defaultPrivilegedUser);
 
         if(!df)
             return false;
@@ -171,7 +171,7 @@ HPCCFile * HPCCFileCache::fetchHpccFileByName(const char * filename, const char
 
         //queryDistributedFileDirectory returns singleton
         IDistributedFileDirectory & dfd = queryDistributedFileDirectory();
-        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc);
+        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc, false, false, false, nullptr, defaultPrivilegedUser);
 
         if(!df)
             throw MakeStringException(-1,"Cannot find file %s.",filename);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -128,7 +128,7 @@ StringBuffer &getWuidFromLogicalFileName(IEspContext &context, const char *logic
 {
     Owned<IUserDescriptor> userdesc = createUserDescriptor();
     userdesc->set(context.queryUserId(), context.queryPassword(), context.querySignature());
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, userdesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, userdesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (!df)
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",logicalName);
     return wuid.append(df->queryAttributes().queryProp("@workunit"));
@@ -1341,7 +1341,7 @@ IDistributedFile* WsWuInfo::getLogicalFileData(IEspContext& context, const char*
     context.getUserID(username);
     Owned<IUserDescriptor> userdesc(createUserDescriptor());
     userdesc->set(username.str(), context.queryPassword(), context.querySignature());
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, userdesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, userdesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (!df)
         return NULL;
 

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -226,7 +226,7 @@ bool copyWULogicalFiles(IEspContext &context, IConstWorkUnit &cw, const char *cl
                         foreign.append(*info.getClear());
                     else
                     {
-                        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalname, udesc);
+                        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalname, udesc, false, false, false, nullptr, defaultPrivilegedUser);
                         if(!df)
                             notFound.append(*info.getClear());
                         else if (df->findCluster(cluster)!=NotFound)

esp/smc/SMCLib/LogicFileWrapper.cpp

@@ -41,7 +41,7 @@ LogicFileWrapper::~LogicFileWrapper()
 void LogicFileWrapper::FindClusterName(const char* logicalName, StringBuffer& returnCluster, IUserDescriptor* udesc)
 {
     try {
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc) ;
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc, false, false, false, nullptr, defaultPrivilegedUser) ;
         if(!df)
             throw MakeStringException(-1,"Could not find logical file");
         df->getClusterName(0,returnCluster);    // ** TBD other cluster
@@ -71,7 +71,7 @@ bool LogicFileWrapper::doDeleteFile(const char* logicalName,const char *cluster,
     {
         IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
         {
-            Owned<IDistributedFile> df = fdir.lookup(cname.str(), udesc, true) ;
+            Owned<IDistributedFile> df = fdir.lookup(cname.str(), udesc, true, false, false, nullptr, defaultPrivilegedUser) ;
             if(!df)
             {
                 returnStr.appendf("<Message><Value>File %s not found</Value></Message>", cname.str());
@@ -98,7 +98,7 @@ bool LogicFileWrapper::doCompressFile(const char* name,StringBuffer& returnStr,
 {
     try
     {
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(name, udesc) ;
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(name, udesc, false, false, false, nullptr, defaultPrivilegedUser) ;
         if(!df)
             return false;
 

plugins/fileservices/fileservices.cpp

@@ -448,7 +448,7 @@ FILESERVICES_API bool FILESERVICES_CALL fsFileValidate(ICodeContext *ctx, const
     constructLogicalName(ctx, name, lfn);
 
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (df)
     {
         Owned<IDistributedFilePartIterator> partIter = df->getIterator();
@@ -489,7 +489,7 @@ FILESERVICES_API void FILESERVICES_CALL fsSetReadOnly(ICodeContext *ctx, const c
 
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
     Owned<IException> error;
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, true);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, true, false, false, nullptr, defaultPrivilegedUser);
     if (df)
     {
         LOG(MCauditInfo, "Set ReadOnly:  %s", name);
@@ -1856,7 +1856,7 @@ FILESERVICES_API void FILESERVICES_CALL fsSetFileDescription(ICodeContext *ctx,
     constructLogicalName(ctx, logicalfilename, lfn);
 
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (df) {
         DistributedFilePropertyLock lock(df);
         lock.queryAttributes().setProp("@description",value);
@@ -1871,7 +1871,7 @@ FILESERVICES_API char *  FILESERVICES_CALL fsGetFileDescription(ICodeContext *ct
     constructLogicalName(ctx, logicalfilename, lfn);
 
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, false, false, false, nullptr, defaultPrivilegedUser);
     if (!df)
         throw MakeStringException(0, "GetFileDescription: Could not locate file %s", lfn.str());
     const char * ret = df->queryAttributes().queryProp("@description");
@@ -2033,7 +2033,7 @@ FILESERVICES_API void FILESERVICES_CALL fsLogicalFileSuperOwners(ICodeContext *c
     }
     else {
         Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc,false,false,true); // lock super-owners
+        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc,false,false,true, nullptr, defaultPrivilegedUser); // lock super-owners
         if (df) {
             Owned<IDistributedSuperFileIterator> iter = df->getOwningSuperFiles();
             ForEach(*iter) {
@@ -2383,7 +2383,7 @@ FILESERVICES_API void FILESERVICES_CALL fsSetColumnMapping(ICodeContext * ctx,co
 {
     StringBuffer lfn;
     constructLogicalName(ctx, filename, lfn);
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),ctx->queryUserDescriptor(),true);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),ctx->queryUserDescriptor(),true,false,false,nullptr,defaultPrivilegedUser);
     if (df)
         df->setColumnMapping(mapping);
     else
@@ -2394,7 +2394,7 @@ FILESERVICES_API char *  FILESERVICES_CALL fsfGetColumnMapping(ICodeContext * ct
 {
     StringBuffer lfn;
     constructLogicalName(ctx, filename, lfn);
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),ctx->queryUserDescriptor(),true);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),ctx->queryUserDescriptor(),true,false,false,nullptr,defaultPrivilegedUser);
     if (df) {
         StringBuffer mapping;
         df->getColumnMapping(mapping);
@@ -2552,7 +2552,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfGetLogicalFileAttribute(ICodeContex
     StringBuffer lfn;
     constructLogicalName(ctx, _lfn, lfn);
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, false,false, false, nullptr, defaultPrivilegedUser);
     StringBuffer ret;
     if (df) {
         if (strcmp(attrname,"ECL")==0)
@@ -2604,7 +2604,7 @@ FILESERVICES_API void FILESERVICES_CALL fsProtectLogicalFile(ICodeContext * ctx,
     StringBuffer lfn;
     constructLogicalName(ctx, _lfn, lfn);
     Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
-    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc);
+    Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(lfn.str(),udesc, false, false, false, nullptr, defaultPrivilegedUser);
     StringBuffer ret;
     if (df) {
         StringBuffer u("user:");

roxie/ccd/ccdactivities.cpp

@@ -994,11 +994,12 @@ public:
     {
         Owned<IHThorDiskReadBaseArg> helper = (IHThorDiskReadBaseArg *) helperFactory();
         bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
+        bool isCodeSigned = isActivityCodeSigned(_graphNode);
         if (!variableFileName)
         {
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
             OwnedRoxieString fileName(helper->getFileName());
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
             if (datafile)
             {
                 unsigned channel = queryFactory.queryChannel();
@@ -2416,9 +2417,10 @@ public:
         bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0);
         if (!variableFileName)
         {
+            bool isCodeSigned = isActivityCodeSigned(graphNode);
             bool isOpt = (helper->getFlags() & TIRoptional) != 0;
             OwnedRoxieString indexName(helper->getFileName());
-            datafile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
+            datafile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isCodeSigned));
             if (datafile)
             {
                 translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
@@ -3599,9 +3601,10 @@ public:
         bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableFileName)
         {
+            bool isCodeSigned = isActivityCodeSigned(_graphNode);
             bool isOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
             OwnedRoxieString fname(helper->getFileName());
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fname, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fname, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
             if (datafile)
             {
                 unsigned expectedCrc = helper->getDiskFormatCrc();
@@ -3996,9 +3999,10 @@ public:
         expectedMeta = helper->queryIndexRecordSize();
         if (!variableFileName)
         {
+            bool isCodeSigned = isActivityCodeSigned(_graphNode);
             bool isOpt = (helper->getJoinFlags() & JFindexoptional) != 0;
             OwnedRoxieString indexFileName(helper->getIndexFileName());
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
             if (datafile)
             {
                 translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
@@ -4334,9 +4338,10 @@ public:
         bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableFileName)
         {
+            bool isCodeSigned = isActivityCodeSigned(_graphNode);
             bool isOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
             OwnedRoxieString fileName(helper->getFileName());
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
             if (datafile)
             {
                 unsigned expectedCrc = helper->getDiskFormatCrc();
@@ -4677,19 +4682,20 @@ public:
             ThorActivityKind kind = getActivityKind(_graphNode);
             if (kind != TAKdiskwrite && kind != TAKspillwrite && kind != TAKindexwrite && kind != TAKpiperead && kind != TAKpipewrite)
             {
+                bool isCodeSigned = isActivityCodeSigned(_graphNode);
                 const char *fileName = queryNodeFileName(_graphNode, kind);
                 const char *indexName = queryNodeIndexName(_graphNode, kind);
                 if (indexName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
-                    indexfile.setown(_queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+                    indexfile.setown(_queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
                     if (indexfile)
                         keyArray.setown(indexfile->getKeyArray(isOpt, queryFactory.queryChannel()));
                 }
                 if (fileName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
-                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isCodeSigned));
                     if (datafile)
                     {
                         fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));

roxie/ccd/ccdcontext.cpp

@@ -441,7 +441,7 @@ private:
 
     inline bool fileExists(const char *lfn)
     {
-        Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, queryUserDescriptor());
+        Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, queryUserDescriptor(), false, false, false, nullptr, defaultPrivilegedUser);
         if (f)
             return true;
         return false;
@@ -735,7 +735,7 @@ private:
                     MilliSleep(PERSIST_LOCK_SLEEP + (getRandom()%PERSIST_LOCK_SLEEP));
                     persistLock.setown(getPersistReadLock(goer));
                 }
-                Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(goer, queryUserDescriptor(), true);
+                Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(goer, queryUserDescriptor(), true, false, false, nullptr, defaultPrivilegedUser);
                 if (!f)
                     goto restart; // Persist has been deleted since last checked - repeat the whole process
                 const char *newAccessTime = f->queryAttributes().queryProp("@accessed");
@@ -2487,22 +2487,22 @@ public:
         return NULL;
     }
 
-    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt)
+    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser)
     {
         CDateTime cacheDate; // Note - this is empty meaning we don't know...
         return querySlaveDynamicFileCache()->lookupDynamicFile(*this, filename, cacheDate, 0, header, isOpt, false);
     }
 
-    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters)
+    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser)
     {
         throwUnexpected(); // only support writing on the server
     }
 
-    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal)
+    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser)
     {
         // On a slave, we need to request info using our own header (not the one passed in) and need to get global rather than just local info
         // (possibly we could get just local if the channel matches but not sure there is any point)
-        Owned<const IResolvedFile> dFile = resolveLFN(lfn, isOpt);
+        Owned<const IResolvedFile> dFile = resolveLFN(lfn, isOpt, isPrivilegedUser);
         if (dFile)
         {
             MemoryBuffer mb;
@@ -3614,7 +3614,7 @@ public:
         }
     }
 
-    virtual const IResolvedFile *resolveLFN(const char *fileName, bool isOpt)
+    virtual const IResolvedFile *resolveLFN(const char *fileName, bool isOpt, bool isPrivilegedUser)
     {
         CriticalBlock b(contextCrit);
         StringBuffer expandedName;
@@ -3622,7 +3622,7 @@ public:
         Linked<const IResolvedFile> ret = fileCache.getValue(expandedName);
         if (!ret)
         {
-            ret.setown(factory->queryPackage().lookupFileName(fileName, isOpt, false, false, workUnit, false));
+            ret.setown(factory->queryPackage().lookupFileName(fileName, isOpt, false, false, workUnit, false, isPrivilegedUser));
             if (ret)
             {
                 IResolvedFile *add = const_cast<IResolvedFile *>(ret.get());
@@ -3632,9 +3632,9 @@ public:
         return ret.getClear();
     }
 
-    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters)
+    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser)
     {
-        return factory->queryPackage().createFileName(filename, overwrite, extend, clusters, workUnit);
+        return factory->queryPackage().createFileName(filename, overwrite, extend, clusters, workUnit, isPrivilegedUser);
     }
 
     virtual void endGraph(unsigned __int64 startTimeStamp, cycle_t startCycles, bool aborting) override
@@ -3643,9 +3643,9 @@ public:
         CRoxieContextBase::endGraph(startTimeStamp, startCycles, aborting);
     }
 
-    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal)
+    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser)
     {
-        Owned<const IResolvedFile> dFile = resolveLFN(lfn, isOpt);
+        Owned<const IResolvedFile> dFile = resolveLFN(lfn, isOpt, isPrivilegedUser);
         if (dFile)
         {
             MemoryBuffer mb;
@@ -3893,7 +3893,7 @@ public:
     {
         StringBuffer fullname;
         expandLogicalFilename(fullname, logicalName, workUnit, false, false);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fullname.str(),queryUserDescriptor());
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fullname.str(),queryUserDescriptor(),false,false,false,nullptr,defaultPrivilegedUser);
         if (file)
         {
             WorkunitUpdate wu = updateWorkUnit();

roxie/ccd/ccdcontext.hpp

@@ -54,9 +54,9 @@ interface IRoxieSlaveContext : extends IRoxieContextLogger
     virtual const QueryOptions &queryOptions() const = 0;
     virtual void addSlavesReplyLen(unsigned len) = 0;
     virtual const char *queryAuthToken() = 0;
-    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt) = 0;
-    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters) = 0;
-    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal) = 0;
+    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser) = 0;
+    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser) = 0;
+    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser) = 0;
     virtual IActivityGraph * getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity) = 0;
     virtual IProbeManager *queryProbeManager() const = 0;
     virtual IDebuggableContext *queryDebugContext() const = 0;

roxie/ccd/ccddali.cpp

@@ -487,7 +487,7 @@ public:
         return ret.getClear();
     }
 
-    IFileDescriptor *checkClonedFromRemote(const char *_lfn, IFileDescriptor *fdesc, bool cacheIt)
+    IFileDescriptor *checkClonedFromRemote(const char *_lfn, IFileDescriptor *fdesc, bool cacheIt, bool isPrivilegedUser)
     {
         // NOTE - we rely on the fact that  queryNamedGroupStore().lookup caches results,to avoid excessive load on remote dali
         if (_lfn && !strnicmp(_lfn, "foreign", 7)) //if need to support dali hopping should add each remote location
@@ -518,7 +518,7 @@ public:
             {
                 if (traceLevel > 1)
                     DBGLOG("checkClonedFromRemote: Resolving %s in legacy mode", _lfn);
-                Owned<IDistributedFile> cloneFile = resolveLFN(foreignLfn, cacheIt, false);
+                Owned<IDistributedFile> cloneFile = resolveLFN(foreignLfn, cacheIt, false, isPrivilegedUser);
                 if (cloneFile)
                 {
                     Owned<IFileDescriptor> cloneFDesc = cloneFile->getFileDescriptor();
@@ -538,14 +538,14 @@ public:
         return NULL;
     }
 
-    virtual IDistributedFile *resolveLFN(const char *logicalName, bool cacheIt, bool writeAccess)
+    virtual IDistributedFile *resolveLFN(const char *logicalName, bool cacheIt, bool writeAccess, bool isPrivilegedUser)
     {
         if (isConnected)
         {
             unsigned start = msTick();
             CDfsLogicalFileName lfn;
             lfn.set(logicalName);
-            Owned<IDistributedFile> dfsFile = queryDistributedFileDirectory().lookup(lfn, userdesc.get(), writeAccess, cacheIt);
+            Owned<IDistributedFile> dfsFile = queryDistributedFileDirectory().lookup(lfn, userdesc.get(), writeAccess, cacheIt,false,nullptr,isPrivilegedUser);
             if (dfsFile)
             {
                 IDistributedSuperFile *super = dfsFile->querySuperFile();

roxie/ccd/ccddali.hpp

@@ -39,8 +39,8 @@ interface IRoxieDaliHelper : extends IInterface
 {
     virtual void commitCache() = 0;
     virtual bool connected() const = 0;
-    virtual IFileDescriptor *checkClonedFromRemote(const char *id, IFileDescriptor *fdesc, bool cacheIt) = 0;
-    virtual IDistributedFile *resolveLFN(const char *filename, bool cacheIt, bool writeAccess) = 0;
+    virtual IFileDescriptor *checkClonedFromRemote(const char *id, IFileDescriptor *fdesc, bool cacheIt, bool isPrivilegedUser) = 0;
+    virtual IDistributedFile *resolveLFN(const char *filename, bool cacheIt, bool writeAccess, bool isPrivilegedUser) = 0;
     virtual IFileDescriptor *resolveCachedLFN(const char *filename) = 0;
     virtual IConstWorkUnit *attachWorkunit(const char *wuid, ILoadedDllEntry *source) = 0;
     virtual IPropertyTree *getQuerySet(const char *id) = 0;

roxie/ccd/ccdfile.cpp

@@ -1930,7 +1930,7 @@ public:
                     Owned<IFileDescriptor> fDesc = sub.getFileDescriptor();
                     Owned<IFileDescriptor> remoteFDesc;
                     if (daliHelper)
-                        remoteFDesc.setown(daliHelper->checkClonedFromRemote(sub.queryLogicalName(), fDesc, cacheIt));
+                        remoteFDesc.setown(daliHelper->checkClonedFromRemote(sub.queryLogicalName(), fDesc, cacheIt, defaultPrivilegedUser));
                     subDFiles.append(OLINK(sub));
                     addFile(sub.queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
                 }
@@ -1949,7 +1949,7 @@ public:
                 Owned<IFileDescriptor> fDesc = dFile->getFileDescriptor();
                 Owned<IFileDescriptor> remoteFDesc;
                 if (daliHelper)
-                    remoteFDesc.setown(daliHelper->checkClonedFromRemote(_lfn, fDesc, cacheIt));
+                    remoteFDesc.setown(daliHelper->checkClonedFromRemote(_lfn, fDesc, cacheIt, defaultPrivilegedUser));
                 addFile(dFile->queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
             }
         }
@@ -2483,6 +2483,10 @@ public:
         else
             return false;
     }
+    virtual bool isRestrictedAccess() const override
+    {
+        return (dFile && dFile->isRestrictedAccess());
+    }
 };
 
 

roxie/ccd/ccdfile.hpp

@@ -112,6 +112,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
     virtual bool exists() const = 0;
     virtual bool isSuperFile() const = 0;
     virtual bool isKey() const = 0;
+    virtual bool isRestrictedAccess() const = 0;
 };
 
 interface IResolvedFileCreator : extends IResolvedFile

roxie/ccd/ccdquery.cpp

@@ -1160,7 +1160,8 @@ public:
                                     if (indexName)
                                     {
                                         bool isOpt = pretendAllOpt || node.getPropBool("att[@name='_isIndexOpt']/@value");
-                                        const IResolvedFile *indexFile = package.lookupFileName(indexName, isOpt, true, true, wu, true);
+                                        bool isCodeSigned = isActivityCodeSigned(node);
+                                        const IResolvedFile *indexFile = package.lookupFileName(indexName, isOpt, true, true, wu, true, isCodeSigned);
                                         if (indexFile)
                                         {
                                             hashValue = indexFile->addHash64(hashValue);
@@ -1173,8 +1174,9 @@ public:
                                     {
                                         if (!node.getPropBool("att[@name='_isSpill']/@value") && !node.getPropBool("att[@name='_isSpillGlobal']/@value"))
                                         {
+                                            bool isCodeSigned = isActivityCodeSigned(node);
                                             bool isOpt = pretendAllOpt || node.getPropBool("att[@name='_isOpt']/@value");
-                                            const IResolvedFile *dataFile = package.lookupFileName(fileName, isOpt, true, true, wu, true);
+                                            const IResolvedFile *dataFile = package.lookupFileName(fileName, isOpt, true, true, wu, true, isCodeSigned);
                                             if (dataFile)
                                             {
                                                 hashValue = dataFile->addHash64(hashValue);

roxie/ccd/ccdserver.cpp

@@ -299,17 +299,17 @@ public:
     {
         return ctx->queryAuthToken();
     }
-    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt)
+    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser)
     {
-        return ctx->resolveLFN(filename, isOpt);
+        return ctx->resolveLFN(filename, isOpt, isPrivilegedUser);
     }
-    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters)
+    virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser)
     {
-        return ctx->createLFN(filename, overwrite, extend, clusters);
+        return ctx->createLFN(filename, overwrite, extend, clusters, isPrivilegedUser);
     }
-    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal)
+    virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser)
     {
-        ctx->onFileCallback(header, lfn, isOpt, isLocal);
+        ctx->onFileCallback(header, lfn, isOpt, isLocal, isPrivilegedUser);
     }
     virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity)
     {
@@ -417,8 +417,8 @@ protected:
     bool optStableInput = true; // is the input forced to ordered?
     bool optUnstableInput = false;  // is the input forced to unordered?
     bool optUnordered = false; // is the output specified as unordered?
+    bool isCodeSigned = false;
     unsigned heapFlags;
-
     mutable RelaxedAtomic<__int64> processed = {0};
     mutable RelaxedAtomic<__int64> started = {0};
 
@@ -432,6 +432,7 @@ public:
         optParallel = _graphNode.getPropInt("att[@name='parallel']/@value", 0);
         optUnordered = !_graphNode.getPropBool("att[@name='ordered']/@value", true);
         heapFlags = _graphNode.getPropInt("hint[@name='heapflags']/@value", _queryFactory.queryOptions().heapFlags);
+        isCodeSigned = ::isActivityCodeSigned(_graphNode);
     }
     
     ~CRoxieServerActivityFactoryBase()
@@ -611,6 +612,10 @@ public:
     {
         return (roxiemem::RoxieHeapFlags)heapFlags;
     }
+    virtual bool isActivityCodeSigned() const
+    {
+        return isCodeSigned;
+    }
 };
 
 class CRoxieServerMultiInputInfo
@@ -1020,17 +1025,17 @@ public:
         }
     }
 
-    const IResolvedFile *resolveLFNIndex(const char *filename, bool isOpt)
+    const IResolvedFile *resolveLFNIndex(const char *filename, bool isOpt, bool isPrivilegedUser)
     {
-        const IResolvedFile *ret = resolveLFN(filename, isOpt);
+        const IResolvedFile *ret = resolveLFN(filename, isOpt, isPrivilegedUser);
         if (ret && !ret->isKey())
             throw MakeStringException(0, "Attempting to read flat file as an index: %s", filename);
         return ret;
     }
 
-    const IResolvedFile *resolveLFNFlat(const char *filename, bool isOpt)
+    const IResolvedFile *resolveLFNFlat(const char *filename, bool isOpt, bool isPrivilegedUser)
     {
-        const IResolvedFile *ret = resolveLFN(filename, isOpt);
+        const IResolvedFile *ret = resolveLFN(filename, isOpt, isPrivilegedUser);
         if (ret && ret->isKey())
             throw MakeStringException(0, "Attempting to read index as a flat file: %s", filename);
         return ret;
@@ -1258,9 +1263,9 @@ public:
         return false;
     }
 
-    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt)
+    virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser)
     {
-        return ctx->resolveLFN(filename, isOpt);
+        return ctx->resolveLFN(filename, isOpt, isPrivilegedUser);
     }
 
     virtual const IResolvedFile *queryVarFileInfo() const
@@ -4850,7 +4855,7 @@ public:
                                 StringBuffer s;
                                 activity.queryLogCtx().CTXLOG("Callback on query %s file %s", header.toString(s).str(),(const char *) lfn);
                             }
-                            activity.queryContext()->onFileCallback(header, lfn, isOpt, isLocal);
+                            activity.queryContext()->onFileCallback(header, lfn, isOpt, isLocal, defaultPrivilegedUser);
                         }
                         else
                             throwUnexpected();
@@ -11708,7 +11713,8 @@ protected:
             else
                 clusters.append(".");
         }
-        writer.setown(ctx->createLFN(rawLogicalName, overwrite, extend, clusters));
+        // Using defaultPrivilegedUser as restricted files applies to limits reading of file (not writing of files)
+        writer.setown(ctx->createLFN(rawLogicalName, overwrite, extend, clusters, defaultPrivilegedUser));
         // MORE - need to check somewhere that single part if it's an existing file or an external one...
     }
 
@@ -12181,7 +12187,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
         else
             clusters.append(".");
         OwnedRoxieString fname(helper.getFileName());
-        writer.setown(ctx->createLFN(fname, overwrite, false, clusters)); // MORE - if there's a workunit, use if for scope.
+        writer.setown(ctx->createLFN(fname, overwrite, false, clusters, defaultPrivilegedUser)); // MORE - if there's a workunit, use if for scope.
         filename.set(writer->queryFile()->queryFilename());
         if (writer->queryFile()->exists())
         {
@@ -21906,7 +21912,7 @@ public:
             if (variableFileName)
             {
                 OwnedRoxieString fileName(helper.getFileName());
-                varFileInfo.setown(resolveLFNFlat(fileName, isOpt));
+                varFileInfo.setown(resolveLFNFlat(fileName, isOpt, factory->isActivityCodeSigned()));
                 numParts = 0;
                 if (varFileInfo)
                     numParts = varFileInfo->getNumParts();
@@ -22826,7 +22832,7 @@ public:
         {
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
             OwnedRoxieString fileName(helper->getFileName());
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
             bool isSimple = (datafile && datafile->getNumParts()==1 && !_queryFactory.queryOptions().disableLocalOptimizations);
             if (isLocal || isSimple)
             {
@@ -22907,7 +22913,7 @@ public:
             if ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) == 0)
             {
                 OwnedRoxieString fileName(helper->getFileName());
-                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true);
+                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                 if (temp)
                     addXrefFileInfo(reply, temp);
             }
@@ -22955,7 +22961,7 @@ public:
         {
             bool isOpt = (flags & TIRoptional) != 0;
             OwnedRoxieString indexName(indexHelper->getFileName());
-            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
+            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
             if (indexfile)
             {
                 unsigned projectedCrc = indexHelper->getProjectedFormatCrc();
@@ -22995,7 +23001,7 @@ public:
             if ((indexHelper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) == 0)
             {
                 OwnedRoxieString indexName(indexHelper->getFileName());
-                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true);
+                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                 if (temp)
                     addXrefFileInfo(reply, temp);
             }
@@ -23046,7 +23052,7 @@ protected:
     void setVariableFileInfo()
     {
         OwnedRoxieString indexName(indexHelper.getFileName());
-        varFileInfo.setown(resolveLFNIndex(indexName, isOpt));
+        varFileInfo.setown(resolveLFNIndex(indexName, isOpt, defaultPrivilegedUser));
         if (varFileInfo)
         {
             unsigned projectedCrc = indexHelper.getProjectedFormatCrc();
@@ -23717,7 +23723,7 @@ class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, impleme
     void setVariableFileInfo()
     {
         OwnedRoxieString indexName(indexHelper.getFileName());
-        varFileInfo.setown(resolveLFNIndex(indexName, isOpt));
+        varFileInfo.setown(resolveLFNIndex(indexName, isOpt, defaultPrivilegedUser));
         unsigned projectedCrc = indexHelper.getProjectedFormatCrc();
         unsigned expectedCrc = indexHelper.getDiskFormatCrc();
         IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
@@ -24909,7 +24915,7 @@ public:
         if (variableFileName)
         {
             OwnedRoxieString fname(helper.getFileName());
-            varFileInfo.setown(resolveLFNFlat(fname, isOpt));
+            varFileInfo.setown(resolveLFNFlat(fname, isOpt, defaultPrivilegedUser));
             if (varFileInfo)
                 map.setown(varFileInfo->getFileMap());
         }
@@ -25031,7 +25037,7 @@ public:
             datafile.setown(_queryFactory.queryPackage().lookupFileName(fname,
                                                                         (helper->getFetchFlags() & FFdatafileoptional) != 0,
                                                                         true, true,
-                                                                        _queryFactory.queryWorkUnit(), true));
+                                                                        _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
             if (datafile)
                 map.setown(datafile->getFileMap());
         }
@@ -25053,7 +25059,7 @@ public:
             if ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) == 0)
             {
                 OwnedRoxieString fileName(helper->getFileName());
-                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true);
+                Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                 if (temp)
                     addXrefFileInfo(reply, temp);
             }
@@ -25098,14 +25104,14 @@ public:
                 if (indexName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
-                    indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
+                    indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                     if (indexfile)
                         keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
                 }
                 if (fileName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
-                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
+                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                     if (datafile)
                     {
                         if (isLocal)
@@ -25139,13 +25145,13 @@ public:
             Owned<const IResolvedFile> temp;
             if (fileName.length())
             {
-                temp.setown(queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true));
+                temp.setown(queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                 if (temp)
                     addXrefFileInfo(reply, temp);
             }
             if (indexName.length())
             {
-                temp.setown(queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true));
+                temp.setown(queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                 if (temp)
                     addXrefFileInfo(reply, temp);
             }
@@ -25648,7 +25654,7 @@ public:
         else if (variableIndexFileName)
         {
             OwnedRoxieString indexFileName(helper.getIndexFileName());
-            varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0));
+            varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0, factory->isActivityCodeSigned()));
             if (varFileInfo)
             {
                 unsigned expectedCrc = helper.getIndexFormatCrc();
@@ -26347,7 +26353,7 @@ public:
         {
             bool isFetchOpt = (helper.getFetchFlags() & FFdatafileoptional) != 0;
             OwnedRoxieString fname(helper.getFileName());
-            varFetchFileInfo.setown(resolveLFNFlat(fname, isFetchOpt));
+            varFetchFileInfo.setown(resolveLFNFlat(fname, isFetchOpt, factory->isActivityCodeSigned()));
             if (varFetchFileInfo)
             {
                 // Note - we don't need to do any translation on the disk part of full-keyed joins
@@ -26502,7 +26508,7 @@ public:
         else if (variableIndexFileName)
         {
             OwnedRoxieString indexFileName(helper.getIndexFileName());
-            varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0));
+            varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0, factory->isActivityCodeSigned()));
             if (varFileInfo)
             {
                 unsigned expectedCrc = helper.getIndexFormatCrc();
@@ -26743,7 +26749,7 @@ public:
         {
             bool isOpt = (joinFlags & JFindexoptional) != 0;
             OwnedRoxieString indexFileName(helper->getIndexFileName());
-            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
+            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
             if (indexfile)
             {
                 unsigned expectedCrc = helper->getIndexFormatCrc();
@@ -26767,7 +26773,7 @@ public:
         if (!isHalfKeyed && !variableFetchFileName)
         {
             bool isFetchOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode, _kind), isFetchOpt, true, true, _queryFactory.queryWorkUnit(), true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode, _kind), isFetchOpt, true, true, _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
             if (datafile)
             {
                 if (isLocal)  // Not sure this works
@@ -26790,7 +26796,7 @@ public:
                 headId, keySet, keyTranslators, indexReadMeta, joinFlags, isSimple, isLocal);
         else
             return new CRoxieServerKeyedJoinActivity(_ctx, this, _probeManager,
-                headId, keySet, keyTranslators, indexReadMeta, 
+                headId, keySet, keyTranslators, indexReadMeta,
                 tailId, map, joinFlags, isLocal); // MORE - not sure local variant actually works - should be using files not map, for example?
     }
 

roxie/ccd/ccdserver.hpp

@@ -181,7 +181,7 @@ interface IRoxieServerActivity : extends IActivityBase
 // Roxie server-side caching
     virtual IRoxieServerSideCache *queryServerSideCache() const = 0;
 // Dynamic file support
-    virtual const IResolvedFile *resolveLFN(const char *fileName, bool isOpt) = 0;
+    virtual const IResolvedFile *resolveLFN(const char *fileName, bool isOpt, bool isPrivilegedUser) = 0;
     virtual const IResolvedFile *queryVarFileInfo() const = 0;
 //misc
     virtual IRoxieSlaveContext *queryContext() = 0;
@@ -226,6 +226,7 @@ interface IRoxieServerActivityFactory : extends IActivityFactory
     virtual const StatisticsMapping &queryStatsMapping() const = 0;
     virtual bool isInputOrdered(bool consumerOrdered, unsigned idx) const = 0;
     virtual roxiemem::RoxieHeapFlags getHeapFlags() const = 0;
+    virtual bool isActivityCodeSigned() const = 0;
 };
 interface IGraphResult : public IInterface
 {

roxie/ccd/ccdstate.cpp

@@ -451,7 +451,7 @@ protected:
     }
 
     // Use dali to resolve subfile into physical file info
-    static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool resolveLocal)
+    static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool resolveLocal, bool isPrivilegedUser)
     {
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
         if (traceLevel > 9)
@@ -474,7 +474,7 @@ protected:
             {
                 if (daliHelper->connected())
                 {
-                    Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheResult, writeAccess);
+                    Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheResult, writeAccess, isPrivilegedUser);
                     if (dFile)
                         result = createResolvedFile(fileName, NULL, dFile.getClear(), daliHelper, !useCache, cacheResult, writeAccess);
                 }
@@ -485,7 +485,7 @@ protected:
                     if (fd)
                     {
                         Owned <IResolvedFileCreator> creator = createResolvedFile(fileName, NULL, false);
-                        Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheResult);
+                        Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheResult, isPrivilegedUser);
                         creator->addSubFile(fd.getClear(), remoteFDesc.getClear());
                         result = creator.getClear();
                     }
@@ -524,15 +524,15 @@ protected:
     }
 
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
-    IResolvedFile *lookupExpandedFileName(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool checkCompulsory) const
+    IResolvedFile *lookupExpandedFileName(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool checkCompulsory, bool isPrivilegedUser) const
     {
-        IResolvedFile *result = lookupFile(fileName, useCache, cacheResult, writeAccess, alwaysCreate);
+        IResolvedFile *result = lookupFile(fileName, useCache, cacheResult, writeAccess, alwaysCreate, isPrivilegedUser);
         if (!result && (!checkCompulsory || !isCompulsory()))
-            result = resolveLFNusingDaliOrLocal(fileName, useCache, cacheResult, writeAccess, alwaysCreate, resolveLocally());
+            result = resolveLFNusingDaliOrLocal(fileName, useCache, cacheResult, writeAccess, alwaysCreate, resolveLocally(), isPrivilegedUser);
         return result;
     }
 
-    IResolvedFile *lookupFile(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate) const
+    IResolvedFile *lookupFile(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool isPrivilegedUser) const
     {
         // Order of resolution: 
         // 1. Files named in package
@@ -562,7 +562,7 @@ protected:
                     }
                     if (traceLevel > 9)
                         DBGLOG("Looking up subfile %s", subFileName.str());
-                    Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, useCache, cacheResult, false, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
+                    Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, useCache, cacheResult, false, false, false, isPrivilegedUser);  // NOTE - overwriting a superfile does NOT require write access to subfiles
                     if (subFileInfo)
                     {
                         if (!super)
@@ -588,7 +588,7 @@ protected:
             const CRoxiePackageNode *basePackage = getBaseNode(i);
             if (!basePackage)
                 continue;
-            IResolvedFile *result = basePackage->lookupFile(fileName, useCache, cacheResult, writeAccess, alwaysCreate);
+            IResolvedFile *result = basePackage->lookupFile(fileName, useCache, cacheResult, writeAccess, alwaysCreate, isPrivilegedUser);
             if (result)
                 return result;
         }
@@ -609,6 +609,7 @@ protected:
         {
             // Look through all files and resolve them now
             Owned<IPropertyTreeIterator> supers = node->getElements("SuperFile");
+            const bool isCodeSigned = isActivityCodeSigned(*node);
             ForEach(*supers)
             {
                 IPropertyTree &super = supers->query();
@@ -617,7 +618,7 @@ protected:
                 {
                     try
                     {
-                        const IResolvedFile *resolved = lookupFileName(name, false, true, true, NULL, true);
+                        const IResolvedFile *resolved = lookupFileName(name, false, true, true, NULL, true, isCodeSigned);
                         if (resolved)
                         {
                             files.append(*const_cast<IResolvedFile *>(resolved));
@@ -674,14 +675,14 @@ public:
         return lookupElements(xpath.str(), "MemIndex");
     }
 
-    virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool useCache, bool cacheResult, IConstWorkUnit *wu, bool ignoreForeignPrefix) const
+    virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool useCache, bool cacheResult, IConstWorkUnit *wu, bool ignoreForeignPrefix, bool isPrivilegedUser) const
     {
         StringBuffer fileName;
         expandLogicalFilename(fileName, _fileName, wu, false, ignoreForeignPrefix);
         if (traceLevel > 5)
             DBGLOG("lookupFileName %s", fileName.str());
 
-        const IResolvedFile *result = lookupExpandedFileName(fileName, useCache, cacheResult, false, false, true);
+        const IResolvedFile *result = lookupExpandedFileName(fileName, useCache, cacheResult, false, false, true, isPrivilegedUser);
         if (!result)
         {
             StringBuffer compulsoryMsg;
@@ -695,13 +696,13 @@ public:
         return result;
     }
 
-    virtual IRoxieWriteHandler *createFileName(const char *_fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const
+    virtual IRoxieWriteHandler *createFileName(const char *_fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu, bool isPrivilegedUser) const
     {
         StringBuffer fileName;
         expandLogicalFilename(fileName, _fileName, wu, false, false);
-        Owned<IResolvedFile> resolved = lookupFile(fileName, false, false, true, true);
+        Owned<IResolvedFile> resolved = lookupFile(fileName, false, false, true, true, isPrivilegedUser);
         if (!resolved)
-            resolved.setown(resolveLFNusingDaliOrLocal(fileName, false, false, true, true, resolveLocally()));
+            resolved.setown(resolveLFNusingDaliOrLocal(fileName, false, false, true, true, resolveLocally(), isPrivilegedUser));
         if (resolved)
         {
             if (resolved->exists())
@@ -730,7 +731,7 @@ public:
         else if (daliHelper)
             user = daliHelper->queryUserDescriptor();//predeployed query mode
 
-        Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, user, onlyLocal, onlyDFS, true);
+        Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, user, onlyLocal, onlyDFS, true, isPrivilegedUser);
         if (!ldFile)
             throw MakeStringException(ROXIE_FILE_ERROR, "Cannot write %s", fileName.str());
         return createRoxieWriteHandler(daliHelper, ldFile.getClear(), clusters);

roxie/ccd/ccdstate.hpp

@@ -54,9 +54,9 @@ extern const IRoxiePackageMap &queryEmptyRoxiePackageMap();
 interface IRoxiePackage : public IHpccPackage
 {
     // Lookup information in package to resolve existing logical file name
-    virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool useCache, bool cacheResults, IConstWorkUnit *wu, bool ignoreForeignPrefix) const = 0;
+    virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool useCache, bool cacheResults, IConstWorkUnit *wu, bool ignoreForeignPrefix, bool isPrivilegedUser) const = 0;
     // Lookup information in package to create new logical file name
-    virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
+    virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu, bool isPrivilegedUser) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
     // Set the hash to be used for this package

testing/unittests/dalitests.cpp

@@ -209,7 +209,7 @@ void checkFiles(const char *fn)
     if (fn) {
         checker.title(1,fn);
         try {
-            Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fn,user);
+            Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fn,user,false,false,false,nullptr,defaultNonPrivilegedUser);
             if (!file)
                 printf("file '%s' not found\n",fn);
             else
@@ -223,7 +223,7 @@ void checkFiles(const char *fn)
         }
     }
     else {
-        Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator("*",false,user);
+        Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator("*",false,user,defaultNonPrivilegedUser);
         unsigned i=0;
         unsigned ss = msTick();
         ForEach(*iter) {
@@ -608,7 +608,7 @@ public:
         for (i=0;i<100;i++) {
             s.clear().append("daregress::test").append(t);
             ASSERT(dir.exists(s.str(),user) && "Could not find sub-file");
-            Owned<IDistributedFile> dfile = dir.lookup(s.str(), user);
+            Owned<IDistributedFile> dfile = dir.lookup(s.str(), user, false, false, false, nullptr, false);
             ASSERT(dfile && "Could not find sub-file");
             offset_t totsz = 0;
             n = 11;
@@ -644,7 +644,7 @@ public:
         __int64 crctot = 0;
         unsigned np = 0;
         unsigned totrows = 0;
-        Owned<IDistributedFileIterator> fiter = dir.getIterator("daregress::*",false, user);
+        Owned<IDistributedFileIterator> fiter = dir.getIterator("daregress::*",false, user, defaultNonPrivilegedUser);
         Owned<IDistributedFilePartIterator> piter;
         ForEach(*fiter) {
             piter.setown(fiter->query().getIterator());
@@ -709,7 +709,7 @@ public:
         t = 39;
         for (i=0;i<100;i++) {
             ASSERT(!dir.exists(s.str(),user) && "Found dir after deletion");
-            Owned<IDistributedFile> dfile = dir.lookup(s.str(), user);
+            Owned<IDistributedFile> dfile = dir.lookup(s.str(), user, false, false, false, nullptr, false);
             ASSERT(!dfile && "Found file after deletion");
             t = (t+39)%100;
         }
@@ -1629,7 +1629,7 @@ public:
          * the super files, this should _not_ cause an issue, as no single super file will contain
          * mismatched subfiles.
         */
-        Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, timeout);
+        Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, false, timeout);
         assertex(sub1);
         sub1->lockProperties();
         sub1->queryAttributes().setPropBool("@local", true);
@@ -1711,7 +1711,7 @@ public:
             ASSERT(strcmp(sfile3->querySubFile(0).queryLogicalName(), "regress::trans::sub2") == 0 && "promote failed, wrong name for sub2");
             ASSERT(outlinked.length() == 1 && "promote failed, outlinked expected only one item");
             ASSERT(strcmp(outlinked.popGet(), "regress::trans::sub1") == 0 && "promote failed, outlinked expected to be sub1");
-            Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, timeout);
+            Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, false, timeout);
             ASSERT(sub1.get() && "promote failed, sub1 was physically deleted");
         }
 
@@ -1733,9 +1733,9 @@ public:
             ASSERT(strcmp(sfile3->querySubFile(0).queryLogicalName(), "regress::trans::sub3") == 0 && "promote failed, wrong name for sub3");
             ASSERT(outlinked.length() == 1 && "promote failed, outlinked expected only one item");
             ASSERT(strcmp(outlinked.popGet(), "regress::trans::sub2") == 0 && "promote failed, outlinked expected to be sub2");
-            Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, timeout);
+            Owned<IDistributedFile> sub1 = dir.lookup("regress::trans::sub1", user, false, false, false, NULL, false, timeout);
             ASSERT(sub1.get() && "promote failed, sub1 was physically deleted");
-            Owned<IDistributedFile> sub2 = dir.lookup("regress::trans::sub2", user, false, false, false, NULL, timeout);
+            Owned<IDistributedFile> sub2 = dir.lookup("regress::trans::sub2", user, false, false, false, NULL, false, timeout);
             ASSERT(sub2.get() && "promote failed, sub2 was physically deleted");
         }
     }
@@ -1764,16 +1764,16 @@ public:
         for (i=0;i<file->numClusters();i++)
             PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
         file.clear();
-        file.setown(queryDistributedFileDirectory().lookup("test::testfile1",user));
+        file.setown(queryDistributedFileDirectory().lookup("test::testfile1",user,false,false,false,nullptr,defaultNonPrivilegedUser));
         for (i=0;i<file->numClusters();i++)
             PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
         file.clear();
-        file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",user));
+        file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",user,false,false,false,nullptr,defaultNonPrivilegedUser));
         for (i=0;i<file->numClusters();i++)
             PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
         file.clear();
         removeLogical("test::testfile1@testgrp2", user);
-        file.setown(queryDistributedFileDirectory().lookup("test::testfile1",user));
+        file.setown(queryDistributedFileDirectory().lookup("test::testfile1",user,false,false,false,nullptr,defaultNonPrivilegedUser));
         for (i=0;i<file->numClusters();i++)
             PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
     }
@@ -2295,7 +2295,7 @@ public:
         }
         virtual void threadmain() override
         {
-            Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fileName, NULL);
+            Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fileName,nullptr,false,false,false,nullptr,defaultNonPrivilegedUser);
 
             if (!file)
             {

thorlcr/activities/fetch/thfetch.cpp

@@ -51,7 +51,7 @@ public:
     {
         CMasterActivity::init();
         OwnedRoxieString fname(helper->getFileName());
-        Owned<IDistributedFile> fetchFile = queryThorFileManager().lookup(container.queryJob(), fname, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true);
+        Owned<IDistributedFile> fetchFile = queryThorFileManager().lookup(container.queryJob(), fname, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), false, container.activityIsCodeSigned());
         if (fetchFile)
         {
             if (isFileKey(fetchFile))

thorlcr/activities/hashdistrib/thhashdistrib.cpp

@@ -123,7 +123,7 @@ public:
         StringBuffer scoped;
         OwnedRoxieString indexFileName(helper->getIndexFileName());
         queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped);
-        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), indexFileName);
+        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, false, false, container.activityIsCodeSigned());
         if (!file)
             throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
         if (0 == file->numParts())

thorlcr/activities/indexread/thindexread.cpp

@@ -229,7 +229,7 @@ public:
         StringBuffer expandedFileName;
         queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName);
         fileName.set(expandedFileName);
-        Owned<IDistributedFile> index = queryThorFileManager().lookup(container.queryJob(), helperFileName, false, 0 != (TIRoptional & indexBaseHelper->getFlags()), true);
+        Owned<IDistributedFile> index = queryThorFileManager().lookup(container.queryJob(), helperFileName, false, 0 != (TIRoptional & indexBaseHelper->getFlags()), true, container.activityIsCodeSigned());
         if (index)
         {
             checkFileType(this, index, "key", true);

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -144,7 +144,7 @@ public:
         {
             assertex(!isLocal);
             buildTlk = false;
-            Owned<IDistributedFile> _f = queryThorFileManager().lookup(container.queryJob(), diName);
+            Owned<IDistributedFile> _f = queryThorFileManager().lookup(container.queryJob(), diName, false, false, false, container.activityIsCodeSigned());
             checkFormatCrc(this, _f, helper->getFormatCrc(), nullptr, helper->getFormatCrc(), nullptr, true);
             IDistributedFile *f = _f->querySuperFile();
             if (!f) f = _f;
@@ -217,7 +217,7 @@ public:
                     assertex(diName.get());
                     IPartDescriptor *tlkDesc = fileDesc->queryPart(fileDesc->numParts()-1);
                     tlkDesc->serialize(dst);
-                    Owned<IDistributedFile> _f = queryThorFileManager().lookup(container.queryJob(), diName);
+                    Owned<IDistributedFile> _f = queryThorFileManager().lookup(container.queryJob(), diName, false, false, false, container.activityIsCodeSigned());
                     IDistributedFile *f = _f->querySuperFile();
                     if (!f) f = _f;
                     Owned<IDistributedFilePart> existingTlk = f->getPart(f->numParts()-1);
@@ -333,7 +333,7 @@ public:
         if (0 == (TIWvarfilename & helper->getFlags()))
         {
             OwnedRoxieString fname(helper->getFileName());
-            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true);
+            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true, false, container.activityIsCodeSigned());
             if (file)
             {
                 if (0 == (TIWoverwrite & helper->getFlags()))

thorlcr/activities/keydiff/thkeydiff.cpp

@@ -56,10 +56,10 @@ public:
         queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
         outputName.set(expandedFileName);
 
-        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
+        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName,false, false, false, container.activityIsCodeSigned()));
         if (!isFileKey(originalIndexFile))
             throw MakeActivityException(this, 0, "Attempting to read flat file as an index: %s", originalHelperName.get());
-        newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedHelperName));
+        newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedHelperName, false, false, false, container.activityIsCodeSigned()));
         if (!isFileKey(newIndexFile))
             throw MakeActivityException(this, 0, "Attempting to read flat file as an index: %s", updatedHelperName.get());
         if (originalIndexFile->numParts() != newIndexFile->numParts())
@@ -216,7 +216,7 @@ public:
         {
             if (KDPvaroutputname & helper->getFlags()) return;
             OwnedRoxieString outputHelperName(helper->getOutputName());
-            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputHelperName, false, true);
+            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputHelperName, false, true, false, container.activityIsCodeSigned());
             if (file)
                 throw MakeActivityException(this, TE_OverwriteNotSpecified, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", file->queryLogicalName());
         }

thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp

@@ -77,7 +77,7 @@ public:
         CMasterActivity::init();
         OwnedRoxieString indexFileName(helper->getIndexFileName());
         Owned<IDistributedFile> dataFile;
-        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
+        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true, container.activityIsCodeSigned());
 
         unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
         if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
@@ -225,7 +225,7 @@ public:
                     OwnedRoxieString fetchFilename(helper->getFileName());
                     if (fetchFilename)
                     {
-                        dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
+                        dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true, container.activityIsCodeSigned()));
                         if (dataFile)
                         {
                             if (isFileKey(dataFile))

thorlcr/activities/keyedjoin/thkeyedjoin.cpp

@@ -276,7 +276,7 @@ public:
         totalIndexParts = 0;
 
         Owned<IDistributedFile> dataFile;
-        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
+        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true, container.activityIsCodeSigned());
         if (indexFile)
         {
             if (!isFileKey(indexFile))
@@ -287,7 +287,7 @@ public:
                 OwnedRoxieString fetchFilename(helper->getFileName());
                 if (fetchFilename)
                 {
-                    dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
+                    dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true, container.activityIsCodeSigned()));
                     if (dataFile)
                     {
                         if (isFileKey(dataFile))

thorlcr/activities/keypatch/thkeypatch.cpp

@@ -55,10 +55,10 @@ public:
         queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
         outputName.set(expandedFileName);
 
-        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
+        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName, false, false, false, container.activityIsCodeSigned()));
         if (!isFileKey(originalIndexFile))
             throw MakeActivityException(this, 0, "Attempting to read flat file as an index: %s", originalHelperName.get());
-        patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName));
+        patchFile.setown(queryThorFileManager().lookup(container.queryJob(), patchHelperName, false, false, false, container.activityIsCodeSigned()));
         if (isFileKey(patchFile))
             throw MakeActivityException(this, 0, "Attempting to read index as a patch file: %s", patchHelperName.get());
         
@@ -186,7 +186,7 @@ public:
         if (0==(KDPoverwrite & helper->getFlags()))
         {
             if (KDPvaroutputname & helper->getFlags()) return;
-            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputName, false, true);
+            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputName, false, true, false, container.activityIsCodeSigned());
             if (file)
                 throw MakeActivityException(this, TE_OverwriteNotSpecified, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", file->queryLogicalName());
         }

thorlcr/activities/msort/thmsort.cpp

@@ -106,7 +106,7 @@ protected:
         OwnedRoxieString cosortlogname(helper->getSortedFilename());
         if (cosortlogname&&*cosortlogname)
         {
-            Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
+            Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname, false, false, false, container.activityIsCodeSigned());
             if (isFileKey(coSortFile))
                 throw MakeActivityException(this, 0, "Attempting to read index as a flat file: %s", cosortlogname.get());
             addReadFile(coSortFile);

thorlcr/activities/thdiskbase.cpp

@@ -46,7 +46,7 @@ void CDiskReadMasterBase::init()
     fileName.set(expandedFileName);
     reInit = 0 != (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename));
 
-    Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true);
+    Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, 0 != ((TDXtemporary|TDXjobtemp) & helper->getFlags()), 0 != (TDRoptional & helper->getFlags()), true, container.activityIsCodeSigned());
     if (file)
     {
         if (file->isExternal() && (helper->getFlags() & TDXcompress))
@@ -155,7 +155,7 @@ void CWriteMasterBase::init()
     if (diskHelperBase->getFlags() & TDWextend)
     {
         assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
-        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, false, true);
+        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), helperFileName, false, true, false, container.activityIsCodeSigned());
         if (file.get())
         {
             fileDesc.setown(file->getFileDescriptor());
@@ -362,7 +362,7 @@ void CWriteMasterBase::preStart(size32_t parentExtractSz, const byte *parentExtr
         if (0 == ((TDXvarfilename|TDXtemporary|TDXjobtemp) & diskHelperBase->getFlags()))
         {
             OwnedRoxieString fname(diskHelperBase->getFileName());
-            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true);
+            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true, false, container.activityIsCodeSigned());
             if (file)
             {
                 if (0 == ((TDWextend+TDWoverwrite) & diskHelperBase->getFlags()))

thorlcr/graph/thgraph.cpp

@@ -335,7 +335,7 @@ CGraphElementBase::CGraphElementBase(CGraphBase &_owner, IPropertyTree &_xgmml)
     ownerId = xgmml->getPropInt("att[@name=\"_parentActivity\"]/@value", 0);
     onCreateCalled = prepared = haveCreateCtx = nullAct = false;
     onlyUpdateIfChanged = xgmml->getPropBool("att[@name=\"_updateIfChanged\"]/@value", false);
-
+    isCodeSigned = isActivityCodeSigned(_xgmml);
     StringBuffer helperName("fAc");
     xgmml->getProp("@id", helperName);
     helperFactory = (EclHelperFactory) queryJob().queryDllEntry().getEntry(helperName.str());

thorlcr/graph/thgraph.hpp

@@ -276,6 +276,7 @@ protected:
     MemoryBuffer createCtxMb, startCtxMb;
     bool haveCreateCtx;
     unsigned maxCores;
+    bool isCodeSigned = false;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -372,6 +373,7 @@ public:
     virtual CActivityBase *factory() { return factory(getKind()); }
     virtual CActivityBase *factorySet(ThorActivityKind kind) { CActivityBase *_activity = factory(kind); activity.setown(_activity); return _activity; }
     virtual ICodeContext *queryCodeContext();
+    virtual bool activityIsCodeSigned() const { return isCodeSigned; }
 };
 
 typedef CIArrayOf<CGraphElementBase> CGraphElementArray;

thorlcr/graph/thgraphmaster.cpp

@@ -599,7 +599,7 @@ bool CMasterGraphElement::checkUpdate()
     if (doCheckUpdate)
     {
         StringAttr lfn;
-        Owned<IDistributedFile> file = queryThorFileManager().lookup(queryJob(), filename, temporary, true);
+        Owned<IDistributedFile> file = queryThorFileManager().lookup(queryJob(), filename, temporary, true, false, defaultPrivilegedUser);
         if (file)
         {
             IPropertyTree &props = file->queryAttributes();
@@ -1174,7 +1174,7 @@ public:
     virtual unsigned __int64 getDatasetHash(const char * name, unsigned __int64 hash) override
     {
         unsigned checkSum = 0;
-        Owned<IDistributedFile> iDfsFile = queryThorFileManager().lookup(jobChannel.queryJob(), name, false, true, false); // NB: do not update accessed
+        Owned<IDistributedFile> iDfsFile = queryThorFileManager().lookup(jobChannel.queryJob(), name, false, true, false, defaultPrivilegedUser); // NB: do not update accessed
         if (iDfsFile.get())
         {
             if (iDfsFile->getFileCheckSum(checkSum))
@@ -1505,7 +1505,7 @@ void CJobMaster::initNodeDUCache()
         Owned<IPropertyTreeIterator> fileIter = &workunit->getFileIterator();
         ForEach (*fileIter)
         {
-            Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(fileIter->query().queryProp("@name"), userDesc);
+            Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(fileIter->query().queryProp("@name"), userDesc, false, false, false, nullptr, defaultPrivilegedUser);
             if (f)
             {
                 unsigned n = f->numParts();

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -61,7 +61,7 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager
         // ii) It publishes immediately not when done!
         // iii) it is not removing existing physicals first
         // If we really want this, it should replicate when done somehow, and only publish at end.
-        Owned<IDistributedFile> file = lookup(job, logicalName, false, true);
+        Owned<IDistributedFile> file = lookup(job, logicalName, false, true, false, defaultPrivilegedUser);
         StringBuffer scopedName;
         addScope(job, logicalName, scopedName);
         if (group) // publishing
@@ -309,19 +309,19 @@ public:
         LOG(MCauditInfo,"%s",outs.str());
     }
 
-    IDistributedFile *timedLookup(CJobBase &job, CDfsLogicalFileName &lfn, bool write, unsigned timeout=INFINITE)
+    IDistributedFile *timedLookup(CJobBase &job, CDfsLogicalFileName &lfn, bool write, bool privilegedUser=false, unsigned timeout=INFINITE)
     {
         VStringBuffer blockedMsg("lock file '%s' for %s access", lfn.get(), write ? "WRITE" : "READ");
-        auto func = [&job, &lfn, &write](unsigned timeout) { return queryDistributedFileDirectory().lookup(lfn, job.queryUserDescriptor(), write, false, false, nullptr, timeout); };
+        auto func = [&job, &lfn, write, privilegedUser](unsigned timeout) { return queryDistributedFileDirectory().lookup(lfn, job.queryUserDescriptor(), write, false, false, nullptr, privilegedUser, timeout); };
         return blockReportFunc<IDistributedFile *>(job, func, timeout, blockedMsg);
     }
-    IDistributedFile *timedLookup(CJobBase &job, const char *logicalName, bool write, unsigned timeout=INFINITE)
+    IDistributedFile *timedLookup(CJobBase &job, const char *logicalName, bool write, bool privilegedUser=false, unsigned timeout=INFINITE)
     {
         CDfsLogicalFileName lfn;
         lfn.set(logicalName);
-        return timedLookup(job, lfn, write, timeout);
+        return timedLookup(job, lfn, write, privilegedUser, timeout);
     }
-    IDistributedFile *lookup(CJobBase &job, const char *logicalName, bool temporary=false, bool optional=false, bool reportOptional=false, bool updateAccessed=true)
+    IDistributedFile *lookup(CJobBase &job, const char *logicalName, bool temporary, bool optional, bool reportOptional, bool privilegedUser, bool updateAccessed=true)
     {
         StringBuffer scopedName;
         bool paused = false;
@@ -341,7 +341,7 @@ public:
         if (fileMapping)
             return &fileMapping->get();
 
-        Owned<IDistributedFile> file = timedLookup(job, scopedName.str(), false, job.queryMaxLfnBlockTimeMins() * 60000);
+        Owned<IDistributedFile> file = timedLookup(job, scopedName.str(), false, privilegedUser, job.queryMaxLfnBlockTimeMins() * 60000);
         if (file && 0 == file->numParts())
         {
             if (file->querySuperFile())
@@ -390,7 +390,7 @@ public:
                 throw MakeStringException(99, "Cannot publish %s, invalid logical name", logicalName);
             if (dlfn.isForeign())
                 throw MakeStringException(99, "Cannot publish to a foreign Dali: %s", logicalName);
-            efile.setown(timedLookup(job, dlfn, true, job.queryMaxLfnBlockTimeMins() * 60000));
+            efile.setown(timedLookup(job, dlfn, true, true, job.queryMaxLfnBlockTimeMins() * 60000));
             if (efile)
             {
                 if (!extend && !overwriteok)
@@ -420,7 +420,7 @@ public:
                 if (found)
                 {
                     workunit->releaseFile(logicalName);
-                    Owned<IDistributedFile> f = timedLookup(job, dlfn, false, job.queryMaxLfnBlockTimeMins() * 60000);
+                    Owned<IDistributedFile> f = timedLookup(job, dlfn, false, true, job.queryMaxLfnBlockTimeMins() * 60000);
                     if (f)
                     {
                         unsigned p, parts = f->numParts();
@@ -447,7 +447,7 @@ public:
                 }
                 remove(job, *efile, job.queryMaxLfnBlockTimeMins() * 60000);
                 efile.clear();
-                efile.setown(timedLookup(job, dlfn, true, job.queryMaxLfnBlockTimeMins() * 60000));
+                efile.setown(timedLookup(job, dlfn, true, true, job.queryMaxLfnBlockTimeMins() * 60000));
                 if (!efile.get())
                 {
                     ForEachItemIn(c, clusters)
@@ -604,7 +604,7 @@ public:
 
     unsigned __int64 getFileOffset(CJobBase &job, const char *logicalName, unsigned partno)
     {
-        Owned<IDistributedFile> file = lookup(job, logicalName, false);
+        Owned<IDistributedFile> file = lookup(job, logicalName, false, false, false, defaultPrivilegedUser);
         StringBuffer scopedName;
         addScope(job, logicalName, scopedName);
         if (!file)

thorlcr/mfilemanager/thmfilemanager.hpp

@@ -41,7 +41,7 @@ interface IDistributedFile;
 interface IThorFileManager : extends IInterface
 {
     virtual void noteFileRead(CJobBase &job, IDistributedFile *file, bool extended=false) = 0;
-    virtual IDistributedFile *lookup(CJobBase &job, const char *logicalName, bool temporary=false, bool optional=false, bool reportOptional=false, bool updateAccessed=true) = 0;
+    virtual IDistributedFile *lookup(CJobBase &job, const char *logicalName, bool temporary, bool optional, bool reportOptional, bool privilegedUser, bool updateAccessed=true) = 0;
     virtual IFileDescriptor *create(CJobBase &job, const char *logicalName, StringArray &groupNames, IArrayOf<IGroup> &groups, bool overwriteok, unsigned helperFlags=0, bool nonLocalIndex=false, unsigned restrictedWidth=0) = 0;
     virtual void publish(CJobBase &job, const char *logicalName, IFileDescriptor &file, Owned<IDistributedFile> *publishedFile=NULL) = 0;
     virtual void clearCacheEntry(const char *name) = 0;