Merge remote-tracking branch 'origin/closedown-4.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/dllserver.cpp

@@ -33,6 +33,8 @@
 #define DEFAULT_SERVER_ROOTDIR          "/c$/dllserver"
 #endif
 
+#define CONNECTION_TIMEOUT     30000
+
 static Owned<IConstDomainInfo> hostDomain;
 
 IConstDomainInfo * getDomainFromIp(const char * ip)
@@ -224,7 +226,7 @@ void DllLocation::remove(bool removeFiles, bool removeDirectory)
 
     StringBuffer path;
     getPath(path, entryRoot->queryProp("@name"));
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
     Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements("location");
     ForEach(*iter)
     {
@@ -406,7 +408,7 @@ void DllEntry::remove(bool removeFiles, bool removeDirectory)
     {
         StringBuffer path;
         getPath(path, root->queryProp("@name"));
-        Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+        Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
         conn->close(true);
     }
 }
@@ -501,7 +503,7 @@ void DllServer::copyFileLocally(RemoteFilename & targetName, RemoteFilename & so
 
 IIterator * DllServer::createDllIterator()
 {
-    Owned<IRemoteConnection> conn = querySDS().connect("/GeneratedDlls", myProcessSession(), 0, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect("/GeneratedDlls", myProcessSession(), 0, CONNECTION_TIMEOUT);
     IPropertyTree * root = conn->queryRoot();
     return conn ? (IIterator *)new DllIterator(root, root->getElements("GeneratedDll"), rootDir) : (IIterator *)new CNullIterator;
 }
@@ -510,7 +512,7 @@ DllEntry * DllServer::doGetEntry(const char * name)
 {
     StringBuffer path;
     getPath(path, name);
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, CONNECTION_TIMEOUT);
     if (conn)
         return new DllEntry(conn->queryRoot(), rootDir, NULL);
     return NULL;
@@ -523,7 +525,7 @@ IDllEntry * DllServer::createEntry(IPropertyTree *owner, IPropertyTree *entry)
 
 void DllServer::doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath)
 {
-    Owned<IRemoteConnection> lock = querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> lock = querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
 
     RemoteFilename dllRemote;
     StringBuffer ipText, dllText;
@@ -533,7 +535,7 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char *
 
     StringBuffer path;
     getPath(path, name);
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
     if (conn)
     {
         //check the entry doesn't exist already....
@@ -548,11 +550,11 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char *
     }
     else
     {
-        conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+        conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
         if (!conn)
         {
-            ::Release(querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
-            conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+            ::Release(querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
+            conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
         }
 
         IPropertyTree * entry = conn->queryRoot();

docs/common/Version.xml

@@ -5,11 +5,11 @@
   <chapterinfo>
     <date id="DateVer">DEVELOPER NON-GENERATED VERSION</date>
 
-    <releaseinfo id="FooterInfo">© 2013 HPCC Systems. All rights
+    <releaseinfo id="FooterInfo">© 2014 HPCC Systems. All rights
     reserved</releaseinfo>
 
     <copyright id="Copyright">
-      <year>2013 HPCC Systems. All rights reserved</year>
+      <year>2014 HPCC Systems. All rights reserved</year>
     </copyright>
   </chapterinfo>
 
@@ -18,12 +18,12 @@
   <section>
     <title>Version Section</title>
 
-    <para id="bpara">This document contains no actual content, information, 
-    or other interesting material. In fact it is only here to
-    serve one purpose and that is to store the chapterinfo the above sections
-    that are being used by several other documents.</para>
+    <para id="bpara">This document contains no actual content, information, or
+    other interesting material. In fact it is only here to serve one purpose
+    and that is to store the chapterinfo the above sections that are being
+    used by several other documents.</para>
 
-    <para id="CHMVer">2013 Version 3.x</para>
+    <para id="CHMVer">2014 Version X.X</para>
 
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>
@@ -35,9 +35,10 @@
     <para>DateVer is the <emphasis role="bold">id</emphasis> we provided to be
     able to reference it.</para>
 
-    <para>All content herein is but a placeholder to maintain a valid document structure
-    that we can use bookinfo/date and other elements for other purposes.</para>
-    
+    <para>All content herein is but a placeholder to maintain a valid document
+    structure that we can use bookinfo/date and other elements for other
+    purposes.</para>
+
     <para>☺</para>
   </section>
 </chapter>

docs/common/Version.xml.in

@@ -3,13 +3,13 @@
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <chapter>
   <chapterinfo>
-    <date id="DateVer">2013 Version ${DOC_VERSION}</date>
+    <date id="DateVer">2014 Version ${DOC_VERSION}</date>
 
-    <releaseinfo id="FooterInfo">© 2013 HPCC Systems. All rights
+    <releaseinfo id="FooterInfo">© 2014 HPCC Systems. All rights
     reserved</releaseinfo>
 
     <copyright id="Copyright">
-      <year>2013 HPCC Systems. All rights reserved</year>
+      <year>2014 HPCC Systems. All rights reserved</year>
     </copyright>
   </chapterinfo>
 
@@ -23,7 +23,7 @@
     serve one purpose and that is to store the chapterinfo the above sections
     that are being used by several other documents.</para>
 
-    <para id="CHMVer">2013 Version ${DOC_VERSION}</para>
+    <para id="CHMVer">2014 Version ${DOC_VERSION}</para>
 
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>

ecl/hqlcpp/hqlttcpp.cpp

@@ -5856,7 +5856,6 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
             Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModePersist, queryDirectDependencies(setValue), conts, info.queryCluster());
             setWorkflowPersist(wf, persistName.str(), persistWfid, info.queryMaxPersistCopies());
 
-            Owned<IWorkflowItem> wfPersist = addWorkflowToWorkunit(persistWfid, WFTypeNormal, WFModeNormal, NULL);
             DependenciesUsed dependencies(false);
             UnsignedArray visited;
             extractDependentInputs(visited, dependencies, queryDirectDependencies(setValue));
@@ -5865,15 +5864,23 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
 
             HqlExprArray checkArgs;
             checkArgs.append(*createExprAttribute(_files_Atom, dependencies.tablesRead));
+            inheritDependencies(&checkArgs.item(0));
             if (dependencies.resultsRead.ordinality())
+            {
                 checkArgs.append(*createExprAttribute(_results_Atom, dependencies.resultsRead));
+                inheritDependencies(&checkArgs.item(1));
+            }
             checkArgs.append(*createAttribute(_codehash_Atom, LINK(codehash)));
             checkArgs.append(*createAttribute(namedAtom, LINK(info.storedName)));
             if (expr->isDataset())
                 checkArgs.append(*createAttribute(fileAtom));
             OwnedHqlExpr check = createValue(no_persist_check, makeVoidType(), checkArgs);
+            inheritDependencies(check);
+
             workflowOut->append(*createWorkflowItem(check, persistWfid, no_actionlist));
             workflowOut->append(*createWorkflowItem(setValue, wfid, no_persist));
+
+            Owned<IWorkflowItem> wfPersist = addWorkflowToWorkunit(persistWfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(check), NULL);
         }
         else
         {

ecl/regress/issue10513.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+string prefix := '' : stored('prefix');
+
+in := dataset(prefix + 'file.in', { string x }, thor, opt);
+
+p := in : persist('p');
+
+output(p);

ecllibrary/std/File.ecl

@@ -476,6 +476,9 @@ EXPORT ClearSuperFile(varstring lsuperfn, boolean del=false) :=
 EXPORT RemoveOwnedSubFiles(varstring lsuperfn, boolean del=false) :=
     lib_fileservices.FileServices.RemoveOwnedSubFiles(lsuperfn, del);
 
+EXPORT DeleteOwnedSubFiles(varstring lsuperfn) :=  // Obsolete, use RemoteOwnedSubFiles
+    lib_fileservices.FileServices.DeleteOwnedSubFiles(lsuperfn);
+
 EXPORT SwapSuperFile(varstring lsuperfn1, varstring lsuperfn2) :=
     lib_fileservices.FileServices.SwapSuperFile(lsuperfn1, lsuperfn2);
 

esp/files/scripts/DFUQueryWidget.js

@@ -297,6 +297,13 @@ define([
             });
             this.initWorkunitsGrid();
             this.selectChild(this.workunitsTab, true);
+
+            this.filter.on("clear", function (evt) {
+                context.refreshGrid();
+            });
+            this.filter.on("apply", function (evt) {
+                context.refreshGrid();
+            });
         },
 
         initTab: function() {

plugins/fileservices/fileservices.cpp

@@ -87,6 +87,7 @@ static const char * EclDefinition =
 "  RemoveSuperFile(const varstring lsuperfn,const varstring lfn,boolean del=false,boolean remcontents=false) : c,action,globalcontext,entrypoint='fsRemoveSuperFile'; \n"
 "  ClearSuperFile(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsClearSuperFile'; \n"
 "  RemoveOwnedSubFiles(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsRemoveOwnedSubFiles'; \n"
+"  DeleteOwnedSubFiles(const varstring lsuperfn) : c,action,globalcontext,entrypoint='fsDeleteOwnedSubFiles'; // Obsolete, use RemoveOwnedSubFiles\n"
 "  SwapSuperFile(const varstring lsuperfn1,const varstring lsuperfn2) : c,action,globalcontext,entrypoint='fsSwapSuperFile'; \n"
 "  ReplaceSuperFile(const varstring lsuperfn,const varstring lfn,const varstring bylfn) : c,action,globalcontext,entrypoint='fsReplaceSuperFile'; \n"
 "  FinishSuperFileTransaction(boolean rollback=false) : c,action,globalcontext,entrypoint='fsFinishSuperFileTransaction'; \n"
@@ -1246,6 +1247,11 @@ FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *gct
     fsRemoveSuperFile(gctx,lsuperfn,NULL,del);
 }
 
+FILESERVICES_API void FILESERVICES_CALL fsDeleteOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn) // Obsolete
+{
+    fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, false);
+}
+
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn, bool del)
 {
     fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, del);

plugins/fileservices/fileservices.hpp

@@ -69,6 +69,7 @@ FILESERVICES_API void FILESERVICES_CALL fsAddSuperFile(IGlobalCodeContext *ctx,
 FILESERVICES_API void FILESERVICES_CALL fsRemoveSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *_lfn,bool del=false,bool remcontents=false);
 FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
+FILESERVICES_API void FILESERVICES_CALL fsDeleteOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn); // Obsolete
 FILESERVICES_API void FILESERVICES_CALL fsSwapSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn1,const char *lsuperfn2);
 FILESERVICES_API void FILESERVICES_CALL fsReplaceSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *lfn,const char *bylfn);
 FILESERVICES_API void FILESERVICES_CALL fsFinishSuperFileTransaction(IGlobalCodeContext *ctx, bool rollback=false);

roxie/ccd/ccdfile.cpp

@@ -1598,7 +1598,7 @@ CRoxieFileCache * fileCache;
 class CResolvedFile : public CInterface, implements IResolvedFileCreator
 {
 protected:
-    const IRoxiePackage *cached;
+    IResolvedFileCache *cached;
     StringAttr lfn;
     StringAttr physicalName;
     Owned<IDistributedFile> dFile; // NULL on copies serialized to slaves. Note that this implies we keep a lock on dali file for the lifetime of this object.
@@ -2063,9 +2063,15 @@ public:
         addSubFile(fdesc.getClear(), NULL);
     }
 
-    virtual void setCache(const IRoxiePackage *cache)
+    virtual void setCache(IResolvedFileCache *cache)
     {
-        assertex (!cached);
+        if (cached)
+        {
+            if (cache==NULL)
+                cached->removeCache(this);
+            else
+                throwUnexpected();
+        }
         cached = cache;
     }
 

roxie/ccd/ccdfile.hpp

@@ -85,7 +85,7 @@ interface IDefRecordMeta;
 interface IFilePartMap;
 class TranslatorArray;
 interface IInMemoryIndexManager ;
-interface IRoxiePackage;
+interface IResolvedFileCache;
 
 interface IResolvedFile : extends ISimpleSuperFileEnquiry
 {
@@ -102,7 +102,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
 
     virtual const char *queryPhysicalName() const = 0; // Returns NULL unless in local file mode.
     virtual const char *queryFileName() const = 0;
-    virtual void setCache(const IRoxiePackage *cache) = 0;
+    virtual void setCache(IResolvedFileCache *cache) = 0;
     virtual bool isAlive() const = 0;
     virtual const IPropertyTree *queryProperties() const = 0;
 

roxie/ccd/ccdstate.cpp

@@ -182,43 +182,42 @@ public:
  * </PackageMaps>
  */
 
-class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
+class CResolvedFileCache : implements IResolvedFileCache
 {
-protected:
-    Owned<IRoxieDaliHelper> daliHelper;
-
-    mutable CriticalSection cacheLock;
-    mutable CopyMapStringToMyClass<IResolvedFile> fileCache;
-
-    virtual aindex_t getBaseCount() const = 0;
-    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+    CriticalSection cacheLock;
+    CopyMapStringToMyClass<IResolvedFile> files;
 
-    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+public:
+    // Retrieve number of files in cache
+    inline unsigned count() const
+    {
+        return files.count();
+    }
 
     // Add a filename and the corresponding IResolvedFile to the cache
-    void addCache(const char *filename, const IResolvedFile *file) const
+    virtual void addCache(const char *filename, const IResolvedFile *file)
     {
         CriticalBlock b(cacheLock);
         IResolvedFile *add = const_cast<IResolvedFile *>(file);
         add->setCache(this);
-        fileCache.setValue(filename, add);
+        files.setValue(filename, add);
     }
     // Remove an IResolvedFile from the cache
-    void removeCache(const IResolvedFile *file) const
+    virtual void removeCache(const IResolvedFile *file)
     {
         CriticalBlock b(cacheLock);
         // NOTE: it's theoretically possible for the final release to happen after a replacement has been inserted into hash table. 
         // So only remove from hash table if what we find there matches the item that is being deleted.
-        IResolvedFile *goer = fileCache.getValue(file->queryFileName());
+        IResolvedFile *goer = files.getValue(file->queryFileName());
         if (goer == file)
-            fileCache.remove(file->queryFileName());
+            files.remove(file->queryFileName());
         // You might want to remove files from the daliServer cache too, but it's not safe to do so here as there may be multiple package caches
     }
     // Lookup a filename in the cache
-    IResolvedFile *lookupCache(const char *filename) const
+    virtual IResolvedFile *lookupCache(const char *filename)
     {
         CriticalBlock b(cacheLock);
-        IResolvedFile *cache = fileCache.getValue(filename);
+        IResolvedFile *cache = files.getValue(filename);
         if (cache)
         {
             LINK(cache);
@@ -227,6 +226,18 @@ protected:
         }
         return NULL;
     }
+};
+
+class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
+{
+protected:
+    static CResolvedFileCache daliFiles;
+    mutable CResolvedFileCache fileCache;
+
+    virtual aindex_t getBaseCount() const = 0;
+    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+
+    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
 
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
@@ -246,16 +257,20 @@ protected:
     }
 
     // Use dali to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate) const
+    static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate)
     {
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
+        IResolvedFile* result = daliFiles.lookupCache(fileName);
+        if (result)
+            return result;
+        Owned<IRoxieDaliHelper> daliHelper = connectToDali();
         if (daliHelper)
         {
             if (daliHelper->connected())
             {
                 Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheIt, writeAccess);
                 if (dFile)
-                    return createResolvedFile(fileName, NULL, dFile.getClear(), daliHelper, cacheIt, writeAccess);
+                    result = createResolvedFile(fileName, NULL, dFile.getClear(), daliHelper, cacheIt, writeAccess);
             }
             else if (!writeAccess)  // If we need write access and expect a dali, but don't have one, we should probably fail
             {
@@ -263,19 +278,14 @@ protected:
                 Owned<IFileDescriptor> fd = daliHelper->resolveCachedLFN(fileName);
                 if (fd)
                 {
-                    Owned <IResolvedFileCreator> result = createResolvedFile(fileName, NULL, false);
+                    Owned <IResolvedFileCreator> creator = createResolvedFile(fileName, NULL, false);
                     Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheIt);
-                    result->addSubFile(fd.getClear(), remoteFDesc.getClear());
-                    return result.getClear();
+                    creator->addSubFile(fd.getClear(), remoteFDesc.getClear());
+                    result = creator.getClear();
                 }
             }
         }
-        return NULL;
-    }
-    // Use local package file's localFile info to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingLocal(const char *fileName, bool writeAccess, bool alwaysCreate) const
-    {
-        if (node && node->getPropBool("@localFiles"))
+        if (!result)
         {
             StringBuffer useName;
             if (strstr(fileName,"::"))
@@ -288,24 +298,33 @@ protected:
             bool exists = checkFileExists(useName);
             if (exists || alwaysCreate)
             {
-                Owned <IResolvedFileCreator> result = createResolvedFile(fileName, useName, false);
+                Owned <IResolvedFileCreator> creator = createResolvedFile(fileName, useName, false);
                 if (exists)
-                    result->addSubFile(useName);
-                return result.getClear();
+                    creator->addSubFile(useName);
+                result = creator.getClear();
             }
         }
-        return NULL;
+        if (result && cacheIt)
+            daliFiles.addCache(fileName, result);
+        return result;
     }
+
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
+    IResolvedFile *lookupExpandedFileName(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
+    {
+        IResolvedFile *result = lookupFile(fileName, cache, writeAccess, alwaysCreate);
+        if (!result)
+            result = resolveLFNusingDaliOrLocal(fileName, cache, writeAccess, alwaysCreate);
+        return result;
+    }
+
     IResolvedFile *lookupFile(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
     {
         // Order of resolution: 
         // 1. Files named in package
-        // 2. If dali lookup enabled, dali
-        // 3. If local file system lookup enabled, local file system?
-        // 4. Files named in bases
+        // 2. Files named in bases
 
-        IResolvedFile* result = lookupCache(fileName);
+        IResolvedFile* result = fileCache.lookupCache(fileName);
         if (result)
             return result;
 
@@ -320,7 +339,7 @@ protected:
             {
                 StringBuffer subFileName;
                 subFileInfo->getSubFileName(idx, subFileName);
-                Owned<const IResolvedFile> subFileInfo = lookupFile(subFileName, cache, writeAccess, alwaysCreate);
+                Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, cache, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
                 if (subFileInfo)
                 {
                     if (!super)
@@ -329,18 +348,14 @@ protected:
                 }
             }
             if (super && cache)
-                addCache(fileName, super);
+                fileCache.addCache(fileName, super);
             return super.getClear();
         }
         result = resolveLFNusingPackage(fileName);
-        if (!result)
-            result = resolveLFNusingDali(fileName, cache, writeAccess, alwaysCreate);
-        if (!result)
-            result = resolveLFNusingLocal(fileName, writeAccess, alwaysCreate);
         if (result)
         {
             if (cache)
-                addCache(fileName, result);
+                fileCache.addCache(fileName, result);
             return result;
         }
         aindex_t count = getBaseCount();
@@ -366,7 +381,6 @@ public:
 
     CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     {
-        daliHelper.setown(connectToDali()); // MORE - should make this conditional?
         if (!fileNameServiceDali.length())
             node->setPropBool("@localFiles", true);
     }
@@ -392,7 +406,7 @@ public:
         if (traceLevel > 5)
             DBGLOG("lookupFileName %s", fileName.str());
 
-        const IResolvedFile *result = lookupFile(fileName, cache, false, false);
+        const IResolvedFile *result = lookupExpandedFileName(fileName, cache, false, false);
         if (!result)
         {
             if (!opt)
@@ -408,6 +422,8 @@ public:
         StringBuffer fileName;
         expandLogicalFilename(fileName, _fileName, wu, false);
         Owned<IResolvedFile> resolved = lookupFile(fileName, false, true, true);
+        if (!resolved)
+            resolved.setown(resolveLFNusingDaliOrLocal(fileName, false, true, true));
         if (resolved)
         {
             if (resolved->exists())
@@ -416,13 +432,13 @@ public:
                     throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", resolved->queryFileName());
                 if (extend)
                     UNIMPLEMENTED; // How does extend fit in with the clusterwritemanager stuff? They can't specify cluster and extend together...
-                removeCache(resolved);
-                resolved->remove();
+                resolved->setCache(NULL);
             }
             if (resolved->queryPhysicalName())
                 fileName.clear().append(resolved->queryPhysicalName());
             resolved.clear();
         }
+        Owned<IRoxieDaliHelper> daliHelper = connectToDali();
         bool disconnected = !daliHelper->connected();
         // MORE - not sure this is really the right test. If there SHOULD be a dali but is's unavailable, we should fail.
         Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, NULL, disconnected, !disconnected, true);
@@ -459,6 +475,8 @@ public:
     }
 };
 
+CResolvedFileCache CRoxiePackageNode::daliFiles;
+
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
 
 IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages)

roxie/ccd/ccdstate.hpp

@@ -60,8 +60,16 @@ interface IRoxiePackage : public IHpccPackage
     virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
-    // Remove a resolved file from the cache 
-    virtual void removeCache(const IResolvedFile *goer) const = 0; // note that this is const as cache is considered mutable
+};
+
+interface IResolvedFileCache
+{
+    // Add a filename and the corresponding IResolvedFile to the cache
+    virtual void addCache(const char *filename, const IResolvedFile *file) = 0;
+    // Lookup a filename in the cache
+    virtual IResolvedFile *lookupCache(const char *filename) = 0;
+    // Remove a resolved file from the cache
+    virtual void removeCache(const IResolvedFile *goer) = 0;
 };
 
 extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages);