Jelajahi Sumber

Merge remote-tracking branch 'origin/closedown-4.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 11 tahun lalu
induk
melakukan
4eba80bcaa

+ 11 - 9
common/dllserver/dllserver.cpp

@@ -33,6 +33,8 @@
 #define DEFAULT_SERVER_ROOTDIR          "/c$/dllserver"
 #define DEFAULT_SERVER_ROOTDIR          "/c$/dllserver"
 #endif
 #endif
 
 
+#define CONNECTION_TIMEOUT     30000
+
 static Owned<IConstDomainInfo> hostDomain;
 static Owned<IConstDomainInfo> hostDomain;
 
 
 IConstDomainInfo * getDomainFromIp(const char * ip)
 IConstDomainInfo * getDomainFromIp(const char * ip)
@@ -224,7 +226,7 @@ void DllLocation::remove(bool removeFiles, bool removeDirectory)
 
 
     StringBuffer path;
     StringBuffer path;
     getPath(path, entryRoot->queryProp("@name"));
     getPath(path, entryRoot->queryProp("@name"));
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
     Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements("location");
     Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements("location");
     ForEach(*iter)
     ForEach(*iter)
     {
     {
@@ -406,7 +408,7 @@ void DllEntry::remove(bool removeFiles, bool removeDirectory)
     {
     {
         StringBuffer path;
         StringBuffer path;
         getPath(path, root->queryProp("@name"));
         getPath(path, root->queryProp("@name"));
-        Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+        Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
         conn->close(true);
         conn->close(true);
     }
     }
 }
 }
@@ -501,7 +503,7 @@ void DllServer::copyFileLocally(RemoteFilename & targetName, RemoteFilename & so
 
 
 IIterator * DllServer::createDllIterator()
 IIterator * DllServer::createDllIterator()
 {
 {
-    Owned<IRemoteConnection> conn = querySDS().connect("/GeneratedDlls", myProcessSession(), 0, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect("/GeneratedDlls", myProcessSession(), 0, CONNECTION_TIMEOUT);
     IPropertyTree * root = conn->queryRoot();
     IPropertyTree * root = conn->queryRoot();
     return conn ? (IIterator *)new DllIterator(root, root->getElements("GeneratedDll"), rootDir) : (IIterator *)new CNullIterator;
     return conn ? (IIterator *)new DllIterator(root, root->getElements("GeneratedDll"), rootDir) : (IIterator *)new CNullIterator;
 }
 }
@@ -510,7 +512,7 @@ DllEntry * DllServer::doGetEntry(const char * name)
 {
 {
     StringBuffer path;
     StringBuffer path;
     getPath(path, name);
     getPath(path, name);
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, CONNECTION_TIMEOUT);
     if (conn)
     if (conn)
         return new DllEntry(conn->queryRoot(), rootDir, NULL);
         return new DllEntry(conn->queryRoot(), rootDir, NULL);
     return NULL;
     return NULL;
@@ -523,7 +525,7 @@ IDllEntry * DllServer::createEntry(IPropertyTree *owner, IPropertyTree *entry)
 
 
 void DllServer::doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath)
 void DllServer::doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath)
 {
 {
-    Owned<IRemoteConnection> lock = querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> lock = querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
 
 
     RemoteFilename dllRemote;
     RemoteFilename dllRemote;
     StringBuffer ipText, dllText;
     StringBuffer ipText, dllText;
@@ -533,7 +535,7 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char *
 
 
     StringBuffer path;
     StringBuffer path;
     getPath(path, name);
     getPath(path, name);
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, CONNECTION_TIMEOUT);
     if (conn)
     if (conn)
     {
     {
         //check the entry doesn't exist already....
         //check the entry doesn't exist already....
@@ -548,11 +550,11 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char *
     }
     }
     else
     else
     {
     {
-        conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+        conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
         if (!conn)
         if (!conn)
         {
         {
-            ::Release(querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
-            conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+            ::Release(querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
+            conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, CONNECTION_TIMEOUT));
         }
         }
 
 
         IPropertyTree * entry = conn->queryRoot();
         IPropertyTree * entry = conn->queryRoot();

+ 11 - 10
docs/common/Version.xml

@@ -5,11 +5,11 @@
   <chapterinfo>
   <chapterinfo>
     <date id="DateVer">DEVELOPER NON-GENERATED VERSION</date>
     <date id="DateVer">DEVELOPER NON-GENERATED VERSION</date>
 
 
-    <releaseinfo id="FooterInfo">© 2013 HPCC Systems. All rights
+    <releaseinfo id="FooterInfo">© 2014 HPCC Systems. All rights
     reserved</releaseinfo>
     reserved</releaseinfo>
 
 
     <copyright id="Copyright">
     <copyright id="Copyright">
-      <year>2013 HPCC Systems. All rights reserved</year>
+      <year>2014 HPCC Systems. All rights reserved</year>
     </copyright>
     </copyright>
   </chapterinfo>
   </chapterinfo>
 
 
@@ -18,12 +18,12 @@
   <section>
   <section>
     <title>Version Section</title>
     <title>Version Section</title>
 
 
-    <para id="bpara">This document contains no actual content, information, 
-    or other interesting material. In fact it is only here to
-    serve one purpose and that is to store the chapterinfo the above sections
-    that are being used by several other documents.</para>
+    <para id="bpara">This document contains no actual content, information, or
+    other interesting material. In fact it is only here to serve one purpose
+    and that is to store the chapterinfo the above sections that are being
+    used by several other documents.</para>
 
 
-    <para id="CHMVer">2013 Version 3.x</para>
+    <para id="CHMVer">2014 Version X.X</para>
 
 
     <para>The following line is the code to be put into the document you wish
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>
     to include the above version info in:</para>
@@ -35,9 +35,10 @@
     <para>DateVer is the <emphasis role="bold">id</emphasis> we provided to be
     <para>DateVer is the <emphasis role="bold">id</emphasis> we provided to be
     able to reference it.</para>
     able to reference it.</para>
 
 
-    <para>All content herein is but a placeholder to maintain a valid document structure
-    that we can use bookinfo/date and other elements for other purposes.</para>
-    
+    <para>All content herein is but a placeholder to maintain a valid document
+    structure that we can use bookinfo/date and other elements for other
+    purposes.</para>
+
     <para>☺</para>
     <para>☺</para>
   </section>
   </section>
 </chapter>
 </chapter>

+ 4 - 4
docs/common/Version.xml.in

@@ -3,13 +3,13 @@
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <chapter>
 <chapter>
   <chapterinfo>
   <chapterinfo>
-    <date id="DateVer">2013 Version ${DOC_VERSION}</date>
+    <date id="DateVer">2014 Version ${DOC_VERSION}</date>
 
 
-    <releaseinfo id="FooterInfo">© 2013 HPCC Systems. All rights
+    <releaseinfo id="FooterInfo">© 2014 HPCC Systems. All rights
     reserved</releaseinfo>
     reserved</releaseinfo>
 
 
     <copyright id="Copyright">
     <copyright id="Copyright">
-      <year>2013 HPCC Systems. All rights reserved</year>
+      <year>2014 HPCC Systems. All rights reserved</year>
     </copyright>
     </copyright>
   </chapterinfo>
   </chapterinfo>
 
 
@@ -23,7 +23,7 @@
     serve one purpose and that is to store the chapterinfo the above sections
     serve one purpose and that is to store the chapterinfo the above sections
     that are being used by several other documents.</para>
     that are being used by several other documents.</para>
 
 
-    <para id="CHMVer">2013 Version ${DOC_VERSION}</para>
+    <para id="CHMVer">2014 Version ${DOC_VERSION}</para>
 
 
     <para>The following line is the code to be put into the document you wish
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>
     to include the above version info in:</para>

+ 8 - 1
ecl/hqlcpp/hqlttcpp.cpp

@@ -5856,7 +5856,6 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
             Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModePersist, queryDirectDependencies(setValue), conts, info.queryCluster());
             Owned<IWorkflowItem> wf = addWorkflowToWorkunit(wfid, WFTypeNormal, WFModePersist, queryDirectDependencies(setValue), conts, info.queryCluster());
             setWorkflowPersist(wf, persistName.str(), persistWfid, info.queryMaxPersistCopies());
             setWorkflowPersist(wf, persistName.str(), persistWfid, info.queryMaxPersistCopies());
 
 
-            Owned<IWorkflowItem> wfPersist = addWorkflowToWorkunit(persistWfid, WFTypeNormal, WFModeNormal, NULL);
             DependenciesUsed dependencies(false);
             DependenciesUsed dependencies(false);
             UnsignedArray visited;
             UnsignedArray visited;
             extractDependentInputs(visited, dependencies, queryDirectDependencies(setValue));
             extractDependentInputs(visited, dependencies, queryDirectDependencies(setValue));
@@ -5865,15 +5864,23 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
 
 
             HqlExprArray checkArgs;
             HqlExprArray checkArgs;
             checkArgs.append(*createExprAttribute(_files_Atom, dependencies.tablesRead));
             checkArgs.append(*createExprAttribute(_files_Atom, dependencies.tablesRead));
+            inheritDependencies(&checkArgs.item(0));
             if (dependencies.resultsRead.ordinality())
             if (dependencies.resultsRead.ordinality())
+            {
                 checkArgs.append(*createExprAttribute(_results_Atom, dependencies.resultsRead));
                 checkArgs.append(*createExprAttribute(_results_Atom, dependencies.resultsRead));
+                inheritDependencies(&checkArgs.item(1));
+            }
             checkArgs.append(*createAttribute(_codehash_Atom, LINK(codehash)));
             checkArgs.append(*createAttribute(_codehash_Atom, LINK(codehash)));
             checkArgs.append(*createAttribute(namedAtom, LINK(info.storedName)));
             checkArgs.append(*createAttribute(namedAtom, LINK(info.storedName)));
             if (expr->isDataset())
             if (expr->isDataset())
                 checkArgs.append(*createAttribute(fileAtom));
                 checkArgs.append(*createAttribute(fileAtom));
             OwnedHqlExpr check = createValue(no_persist_check, makeVoidType(), checkArgs);
             OwnedHqlExpr check = createValue(no_persist_check, makeVoidType(), checkArgs);
+            inheritDependencies(check);
+
             workflowOut->append(*createWorkflowItem(check, persistWfid, no_actionlist));
             workflowOut->append(*createWorkflowItem(check, persistWfid, no_actionlist));
             workflowOut->append(*createWorkflowItem(setValue, wfid, no_persist));
             workflowOut->append(*createWorkflowItem(setValue, wfid, no_persist));
+
+            Owned<IWorkflowItem> wfPersist = addWorkflowToWorkunit(persistWfid, WFTypeNormal, WFModeNormal, queryDirectDependencies(check), NULL);
         }
         }
         else
         else
         {
         {

+ 24 - 0
ecl/regress/issue10513.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+string prefix := '' : stored('prefix');
+
+in := dataset(prefix + 'file.in', { string x }, thor, opt);
+
+p := in : persist('p');
+
+output(p);

+ 3 - 0
ecllibrary/std/File.ecl

@@ -476,6 +476,9 @@ EXPORT ClearSuperFile(varstring lsuperfn, boolean del=false) :=
 EXPORT RemoveOwnedSubFiles(varstring lsuperfn, boolean del=false) :=
 EXPORT RemoveOwnedSubFiles(varstring lsuperfn, boolean del=false) :=
     lib_fileservices.FileServices.RemoveOwnedSubFiles(lsuperfn, del);
     lib_fileservices.FileServices.RemoveOwnedSubFiles(lsuperfn, del);
 
 
+EXPORT DeleteOwnedSubFiles(varstring lsuperfn) :=  // Obsolete, use RemoteOwnedSubFiles
+    lib_fileservices.FileServices.DeleteOwnedSubFiles(lsuperfn);
+
 EXPORT SwapSuperFile(varstring lsuperfn1, varstring lsuperfn2) :=
 EXPORT SwapSuperFile(varstring lsuperfn1, varstring lsuperfn2) :=
     lib_fileservices.FileServices.SwapSuperFile(lsuperfn1, lsuperfn2);
     lib_fileservices.FileServices.SwapSuperFile(lsuperfn1, lsuperfn2);
 
 

+ 7 - 0
esp/files/scripts/DFUQueryWidget.js

@@ -297,6 +297,13 @@ define([
             });
             });
             this.initWorkunitsGrid();
             this.initWorkunitsGrid();
             this.selectChild(this.workunitsTab, true);
             this.selectChild(this.workunitsTab, true);
+
+            this.filter.on("clear", function (evt) {
+                context.refreshGrid();
+            });
+            this.filter.on("apply", function (evt) {
+                context.refreshGrid();
+            });
         },
         },
 
 
         initTab: function() {
         initTab: function() {

+ 6 - 0
plugins/fileservices/fileservices.cpp

@@ -87,6 +87,7 @@ static const char * EclDefinition =
 "  RemoveSuperFile(const varstring lsuperfn,const varstring lfn,boolean del=false,boolean remcontents=false) : c,action,globalcontext,entrypoint='fsRemoveSuperFile'; \n"
 "  RemoveSuperFile(const varstring lsuperfn,const varstring lfn,boolean del=false,boolean remcontents=false) : c,action,globalcontext,entrypoint='fsRemoveSuperFile'; \n"
 "  ClearSuperFile(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsClearSuperFile'; \n"
 "  ClearSuperFile(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsClearSuperFile'; \n"
 "  RemoveOwnedSubFiles(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsRemoveOwnedSubFiles'; \n"
 "  RemoveOwnedSubFiles(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsRemoveOwnedSubFiles'; \n"
+"  DeleteOwnedSubFiles(const varstring lsuperfn) : c,action,globalcontext,entrypoint='fsDeleteOwnedSubFiles'; // Obsolete, use RemoveOwnedSubFiles\n"
 "  SwapSuperFile(const varstring lsuperfn1,const varstring lsuperfn2) : c,action,globalcontext,entrypoint='fsSwapSuperFile'; \n"
 "  SwapSuperFile(const varstring lsuperfn1,const varstring lsuperfn2) : c,action,globalcontext,entrypoint='fsSwapSuperFile'; \n"
 "  ReplaceSuperFile(const varstring lsuperfn,const varstring lfn,const varstring bylfn) : c,action,globalcontext,entrypoint='fsReplaceSuperFile'; \n"
 "  ReplaceSuperFile(const varstring lsuperfn,const varstring lfn,const varstring bylfn) : c,action,globalcontext,entrypoint='fsReplaceSuperFile'; \n"
 "  FinishSuperFileTransaction(boolean rollback=false) : c,action,globalcontext,entrypoint='fsFinishSuperFileTransaction'; \n"
 "  FinishSuperFileTransaction(boolean rollback=false) : c,action,globalcontext,entrypoint='fsFinishSuperFileTransaction'; \n"
@@ -1246,6 +1247,11 @@ FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *gct
     fsRemoveSuperFile(gctx,lsuperfn,NULL,del);
     fsRemoveSuperFile(gctx,lsuperfn,NULL,del);
 }
 }
 
 
+FILESERVICES_API void FILESERVICES_CALL fsDeleteOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn) // Obsolete
+{
+    fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, false);
+}
+
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn, bool del)
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn, bool del)
 {
 {
     fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, del);
     fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, del);

+ 1 - 0
plugins/fileservices/fileservices.hpp

@@ -69,6 +69,7 @@ FILESERVICES_API void FILESERVICES_CALL fsAddSuperFile(IGlobalCodeContext *ctx,
 FILESERVICES_API void FILESERVICES_CALL fsRemoveSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *_lfn,bool del=false,bool remcontents=false);
 FILESERVICES_API void FILESERVICES_CALL fsRemoveSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *_lfn,bool del=false,bool remcontents=false);
 FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
+FILESERVICES_API void FILESERVICES_CALL fsDeleteOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn); // Obsolete
 FILESERVICES_API void FILESERVICES_CALL fsSwapSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn1,const char *lsuperfn2);
 FILESERVICES_API void FILESERVICES_CALL fsSwapSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn1,const char *lsuperfn2);
 FILESERVICES_API void FILESERVICES_CALL fsReplaceSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *lfn,const char *bylfn);
 FILESERVICES_API void FILESERVICES_CALL fsReplaceSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *lfn,const char *bylfn);
 FILESERVICES_API void FILESERVICES_CALL fsFinishSuperFileTransaction(IGlobalCodeContext *ctx, bool rollback=false);
 FILESERVICES_API void FILESERVICES_CALL fsFinishSuperFileTransaction(IGlobalCodeContext *ctx, bool rollback=false);

+ 9 - 3
roxie/ccd/ccdfile.cpp

@@ -1598,7 +1598,7 @@ CRoxieFileCache * fileCache;
 class CResolvedFile : public CInterface, implements IResolvedFileCreator
 class CResolvedFile : public CInterface, implements IResolvedFileCreator
 {
 {
 protected:
 protected:
-    const IRoxiePackage *cached;
+    IResolvedFileCache *cached;
     StringAttr lfn;
     StringAttr lfn;
     StringAttr physicalName;
     StringAttr physicalName;
     Owned<IDistributedFile> dFile; // NULL on copies serialized to slaves. Note that this implies we keep a lock on dali file for the lifetime of this object.
     Owned<IDistributedFile> dFile; // NULL on copies serialized to slaves. Note that this implies we keep a lock on dali file for the lifetime of this object.
@@ -2063,9 +2063,15 @@ public:
         addSubFile(fdesc.getClear(), NULL);
         addSubFile(fdesc.getClear(), NULL);
     }
     }
 
 
-    virtual void setCache(const IRoxiePackage *cache)
+    virtual void setCache(IResolvedFileCache *cache)
     {
     {
-        assertex (!cached);
+        if (cached)
+        {
+            if (cache==NULL)
+                cached->removeCache(this);
+            else
+                throwUnexpected();
+        }
         cached = cache;
         cached = cache;
     }
     }
 
 

+ 2 - 2
roxie/ccd/ccdfile.hpp

@@ -85,7 +85,7 @@ interface IDefRecordMeta;
 interface IFilePartMap;
 interface IFilePartMap;
 class TranslatorArray;
 class TranslatorArray;
 interface IInMemoryIndexManager ;
 interface IInMemoryIndexManager ;
-interface IRoxiePackage;
+interface IResolvedFileCache;
 
 
 interface IResolvedFile : extends ISimpleSuperFileEnquiry
 interface IResolvedFile : extends ISimpleSuperFileEnquiry
 {
 {
@@ -102,7 +102,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
 
 
     virtual const char *queryPhysicalName() const = 0; // Returns NULL unless in local file mode.
     virtual const char *queryPhysicalName() const = 0; // Returns NULL unless in local file mode.
     virtual const char *queryFileName() const = 0;
     virtual const char *queryFileName() const = 0;
-    virtual void setCache(const IRoxiePackage *cache) = 0;
+    virtual void setCache(IResolvedFileCache *cache) = 0;
     virtual bool isAlive() const = 0;
     virtual bool isAlive() const = 0;
     virtual const IPropertyTree *queryProperties() const = 0;
     virtual const IPropertyTree *queryProperties() const = 0;
 
 

+ 65 - 47
roxie/ccd/ccdstate.cpp

@@ -182,43 +182,42 @@ public:
  * </PackageMaps>
  * </PackageMaps>
  */
  */
 
 
-class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
+class CResolvedFileCache : implements IResolvedFileCache
 {
 {
-protected:
-    Owned<IRoxieDaliHelper> daliHelper;
-
-    mutable CriticalSection cacheLock;
-    mutable CopyMapStringToMyClass<IResolvedFile> fileCache;
-
-    virtual aindex_t getBaseCount() const = 0;
-    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+    CriticalSection cacheLock;
+    CopyMapStringToMyClass<IResolvedFile> files;
 
 
-    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+public:
+    // Retrieve number of files in cache
+    inline unsigned count() const
+    {
+        return files.count();
+    }
 
 
     // Add a filename and the corresponding IResolvedFile to the cache
     // Add a filename and the corresponding IResolvedFile to the cache
-    void addCache(const char *filename, const IResolvedFile *file) const
+    virtual void addCache(const char *filename, const IResolvedFile *file)
     {
     {
         CriticalBlock b(cacheLock);
         CriticalBlock b(cacheLock);
         IResolvedFile *add = const_cast<IResolvedFile *>(file);
         IResolvedFile *add = const_cast<IResolvedFile *>(file);
         add->setCache(this);
         add->setCache(this);
-        fileCache.setValue(filename, add);
+        files.setValue(filename, add);
     }
     }
     // Remove an IResolvedFile from the cache
     // Remove an IResolvedFile from the cache
-    void removeCache(const IResolvedFile *file) const
+    virtual void removeCache(const IResolvedFile *file)
     {
     {
         CriticalBlock b(cacheLock);
         CriticalBlock b(cacheLock);
         // NOTE: it's theoretically possible for the final release to happen after a replacement has been inserted into hash table. 
         // NOTE: it's theoretically possible for the final release to happen after a replacement has been inserted into hash table. 
         // So only remove from hash table if what we find there matches the item that is being deleted.
         // So only remove from hash table if what we find there matches the item that is being deleted.
-        IResolvedFile *goer = fileCache.getValue(file->queryFileName());
+        IResolvedFile *goer = files.getValue(file->queryFileName());
         if (goer == file)
         if (goer == file)
-            fileCache.remove(file->queryFileName());
+            files.remove(file->queryFileName());
         // You might want to remove files from the daliServer cache too, but it's not safe to do so here as there may be multiple package caches
         // You might want to remove files from the daliServer cache too, but it's not safe to do so here as there may be multiple package caches
     }
     }
     // Lookup a filename in the cache
     // Lookup a filename in the cache
-    IResolvedFile *lookupCache(const char *filename) const
+    virtual IResolvedFile *lookupCache(const char *filename)
     {
     {
         CriticalBlock b(cacheLock);
         CriticalBlock b(cacheLock);
-        IResolvedFile *cache = fileCache.getValue(filename);
+        IResolvedFile *cache = files.getValue(filename);
         if (cache)
         if (cache)
         {
         {
             LINK(cache);
             LINK(cache);
@@ -227,6 +226,18 @@ protected:
         }
         }
         return NULL;
         return NULL;
     }
     }
+};
+
+class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
+{
+protected:
+    static CResolvedFileCache daliFiles;
+    mutable CResolvedFileCache fileCache;
+
+    virtual aindex_t getBaseCount() const = 0;
+    virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
+
+    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
 
 
     // Use local package file only to resolve subfile into physical file info
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
@@ -246,16 +257,20 @@ protected:
     }
     }
 
 
     // Use dali to resolve subfile into physical file info
     // Use dali to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate) const
+    static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate)
     {
     {
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
+        IResolvedFile* result = daliFiles.lookupCache(fileName);
+        if (result)
+            return result;
+        Owned<IRoxieDaliHelper> daliHelper = connectToDali();
         if (daliHelper)
         if (daliHelper)
         {
         {
             if (daliHelper->connected())
             if (daliHelper->connected())
             {
             {
                 Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheIt, writeAccess);
                 Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheIt, writeAccess);
                 if (dFile)
                 if (dFile)
-                    return createResolvedFile(fileName, NULL, dFile.getClear(), daliHelper, cacheIt, writeAccess);
+                    result = createResolvedFile(fileName, NULL, dFile.getClear(), daliHelper, cacheIt, writeAccess);
             }
             }
             else if (!writeAccess)  // If we need write access and expect a dali, but don't have one, we should probably fail
             else if (!writeAccess)  // If we need write access and expect a dali, but don't have one, we should probably fail
             {
             {
@@ -263,19 +278,14 @@ protected:
                 Owned<IFileDescriptor> fd = daliHelper->resolveCachedLFN(fileName);
                 Owned<IFileDescriptor> fd = daliHelper->resolveCachedLFN(fileName);
                 if (fd)
                 if (fd)
                 {
                 {
-                    Owned <IResolvedFileCreator> result = createResolvedFile(fileName, NULL, false);
+                    Owned <IResolvedFileCreator> creator = createResolvedFile(fileName, NULL, false);
                     Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheIt);
                     Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheIt);
-                    result->addSubFile(fd.getClear(), remoteFDesc.getClear());
-                    return result.getClear();
+                    creator->addSubFile(fd.getClear(), remoteFDesc.getClear());
+                    result = creator.getClear();
                 }
                 }
             }
             }
         }
         }
-        return NULL;
-    }
-    // Use local package file's localFile info to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingLocal(const char *fileName, bool writeAccess, bool alwaysCreate) const
-    {
-        if (node && node->getPropBool("@localFiles"))
+        if (!result)
         {
         {
             StringBuffer useName;
             StringBuffer useName;
             if (strstr(fileName,"::"))
             if (strstr(fileName,"::"))
@@ -288,24 +298,33 @@ protected:
             bool exists = checkFileExists(useName);
             bool exists = checkFileExists(useName);
             if (exists || alwaysCreate)
             if (exists || alwaysCreate)
             {
             {
-                Owned <IResolvedFileCreator> result = createResolvedFile(fileName, useName, false);
+                Owned <IResolvedFileCreator> creator = createResolvedFile(fileName, useName, false);
                 if (exists)
                 if (exists)
-                    result->addSubFile(useName);
-                return result.getClear();
+                    creator->addSubFile(useName);
+                result = creator.getClear();
             }
             }
         }
         }
-        return NULL;
+        if (result && cacheIt)
+            daliFiles.addCache(fileName, result);
+        return result;
     }
     }
+
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
+    IResolvedFile *lookupExpandedFileName(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
+    {
+        IResolvedFile *result = lookupFile(fileName, cache, writeAccess, alwaysCreate);
+        if (!result)
+            result = resolveLFNusingDaliOrLocal(fileName, cache, writeAccess, alwaysCreate);
+        return result;
+    }
+
     IResolvedFile *lookupFile(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
     IResolvedFile *lookupFile(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
     {
     {
         // Order of resolution: 
         // Order of resolution: 
         // 1. Files named in package
         // 1. Files named in package
-        // 2. If dali lookup enabled, dali
-        // 3. If local file system lookup enabled, local file system?
-        // 4. Files named in bases
+        // 2. Files named in bases
 
 
-        IResolvedFile* result = lookupCache(fileName);
+        IResolvedFile* result = fileCache.lookupCache(fileName);
         if (result)
         if (result)
             return result;
             return result;
 
 
@@ -320,7 +339,7 @@ protected:
             {
             {
                 StringBuffer subFileName;
                 StringBuffer subFileName;
                 subFileInfo->getSubFileName(idx, subFileName);
                 subFileInfo->getSubFileName(idx, subFileName);
-                Owned<const IResolvedFile> subFileInfo = lookupFile(subFileName, cache, writeAccess, alwaysCreate);
+                Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, cache, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
                 if (subFileInfo)
                 if (subFileInfo)
                 {
                 {
                     if (!super)
                     if (!super)
@@ -329,18 +348,14 @@ protected:
                 }
                 }
             }
             }
             if (super && cache)
             if (super && cache)
-                addCache(fileName, super);
+                fileCache.addCache(fileName, super);
             return super.getClear();
             return super.getClear();
         }
         }
         result = resolveLFNusingPackage(fileName);
         result = resolveLFNusingPackage(fileName);
-        if (!result)
-            result = resolveLFNusingDali(fileName, cache, writeAccess, alwaysCreate);
-        if (!result)
-            result = resolveLFNusingLocal(fileName, writeAccess, alwaysCreate);
         if (result)
         if (result)
         {
         {
             if (cache)
             if (cache)
-                addCache(fileName, result);
+                fileCache.addCache(fileName, result);
             return result;
             return result;
         }
         }
         aindex_t count = getBaseCount();
         aindex_t count = getBaseCount();
@@ -366,7 +381,6 @@ public:
 
 
     CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     CRoxiePackageNode(IPropertyTree *p) : CPackageNode(p)
     {
     {
-        daliHelper.setown(connectToDali()); // MORE - should make this conditional?
         if (!fileNameServiceDali.length())
         if (!fileNameServiceDali.length())
             node->setPropBool("@localFiles", true);
             node->setPropBool("@localFiles", true);
     }
     }
@@ -392,7 +406,7 @@ public:
         if (traceLevel > 5)
         if (traceLevel > 5)
             DBGLOG("lookupFileName %s", fileName.str());
             DBGLOG("lookupFileName %s", fileName.str());
 
 
-        const IResolvedFile *result = lookupFile(fileName, cache, false, false);
+        const IResolvedFile *result = lookupExpandedFileName(fileName, cache, false, false);
         if (!result)
         if (!result)
         {
         {
             if (!opt)
             if (!opt)
@@ -408,6 +422,8 @@ public:
         StringBuffer fileName;
         StringBuffer fileName;
         expandLogicalFilename(fileName, _fileName, wu, false);
         expandLogicalFilename(fileName, _fileName, wu, false);
         Owned<IResolvedFile> resolved = lookupFile(fileName, false, true, true);
         Owned<IResolvedFile> resolved = lookupFile(fileName, false, true, true);
+        if (!resolved)
+            resolved.setown(resolveLFNusingDaliOrLocal(fileName, false, true, true));
         if (resolved)
         if (resolved)
         {
         {
             if (resolved->exists())
             if (resolved->exists())
@@ -416,13 +432,13 @@ public:
                     throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", resolved->queryFileName());
                     throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", resolved->queryFileName());
                 if (extend)
                 if (extend)
                     UNIMPLEMENTED; // How does extend fit in with the clusterwritemanager stuff? They can't specify cluster and extend together...
                     UNIMPLEMENTED; // How does extend fit in with the clusterwritemanager stuff? They can't specify cluster and extend together...
-                removeCache(resolved);
-                resolved->remove();
+                resolved->setCache(NULL);
             }
             }
             if (resolved->queryPhysicalName())
             if (resolved->queryPhysicalName())
                 fileName.clear().append(resolved->queryPhysicalName());
                 fileName.clear().append(resolved->queryPhysicalName());
             resolved.clear();
             resolved.clear();
         }
         }
+        Owned<IRoxieDaliHelper> daliHelper = connectToDali();
         bool disconnected = !daliHelper->connected();
         bool disconnected = !daliHelper->connected();
         // MORE - not sure this is really the right test. If there SHOULD be a dali but is's unavailable, we should fail.
         // MORE - not sure this is really the right test. If there SHOULD be a dali but is's unavailable, we should fail.
         Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, NULL, disconnected, !disconnected, true);
         Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, NULL, disconnected, !disconnected, true);
@@ -459,6 +475,8 @@ public:
     }
     }
 };
 };
 
 
+CResolvedFileCache CRoxiePackageNode::daliFiles;
+
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;
 
 
 IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages)
 IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages)

+ 10 - 2
roxie/ccd/ccdstate.hpp

@@ -60,8 +60,16 @@ interface IRoxiePackage : public IHpccPackage
     virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
-    // Remove a resolved file from the cache 
-    virtual void removeCache(const IResolvedFile *goer) const = 0; // note that this is const as cache is considered mutable
+};
+
+interface IResolvedFileCache
+{
+    // Add a filename and the corresponding IResolvedFile to the cache
+    virtual void addCache(const char *filename, const IResolvedFile *file) = 0;
+    // Lookup a filename in the cache
+    virtual IResolvedFile *lookupCache(const char *filename) = 0;
+    // Remove a resolved file from the cache
+    virtual void removeCache(const IResolvedFile *goer) = 0;
 };
 };
 
 
 extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages);
 extern IRoxiePackage *createRoxiePackage(IPropertyTree *p, IRoxiePackageMap *packages);