Merge pull request #3944 from richardkchapman/roxie-once-shared

HPCC-8727 ONCE dictionaries should not be serialized to the slaves

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hqlcpp/hqlinline.cpp

@@ -1330,6 +1330,8 @@ bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool h
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(0));
     case no_getgraphresult:
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(1));
+    case no_getresult:
+        return !matchesConstantValue(queryPropertyChild(expr, sequenceAtom, 0), ResultSequenceOnce);
     case no_failcode:
     case no_failmessage:
     case no_fail:

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -683,6 +683,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="preloadOnceData" type="xs:boolean" use="optional" default="true">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Evaluate : ONCE sections of queries at query load time</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="remoteFilesExpire" type="xs:integer" use="optional" default="3600000">
       <xs:annotation>
         <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -843,6 +843,7 @@
                 preabortKeyedJoinsThreshold="100"
                 preferredSubnet=""
                 preferredSubnetMask=""
+                preloadOnceData="true"
                 remoteFilesExpire="3600000"
                 resolveFilesInPackage="false"
                 roxieMulticastEnabled="true"

roxie/ccd/ccd.hpp

@@ -399,6 +399,7 @@ extern bool useTreeCopy;
 extern XmlReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves
+extern bool preloadOnceData;
 
 extern unsigned udpMulticastBufferSize;
 extern size32_t diskReadBufferSize;

roxie/ccd/ccdmain.cpp

@@ -112,6 +112,7 @@ bool debugPermitted = true;
 bool checkCompleted = true;
 unsigned preabortKeyedJoinsThreshold = 100;
 unsigned preabortIndexReadsThreshold = 100;
+bool preloadOnceData;
 
 unsigned memoryStatsInterval = 0;
 memsize_t defaultMemoryLimit;
@@ -683,6 +684,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         blindLogging = topology->getPropBool("@blindLogging", false);
         if (!blindLogging)
             logExcessiveSeeks = true;
+        preloadOnceData = topology->getPropBool("@preloadOnceData", true);
         linuxYield = topology->getPropBool("@linuxYield", false);
         traceSmartStepping = topology->getPropBool("@traceSmartStepping", false);
         useMemoryMappedIndexes = topology->getPropBool("@useMemoryMappedIndexes", false);

roxie/ccd/ccdquery.cpp

@@ -179,6 +179,73 @@ extern void addXrefLibraryInfo(IPropertyTree &reply, const char *libraryName)
 }
 
 //----------------------------------------------------------------------------------------------
+// Class CSharedOnceContext manages the context for a query's ONCE code, which is shared between
+// all slave and server contexts on a node
+//----------------------------------------------------------------------------------------------
+
+class CSharedOnceContext : public CInterfaceOf<ISharedOnceContext>
+{
+public:
+    CSharedOnceContext()
+    {
+    }
+
+    ~CSharedOnceContext()
+    {
+    }
+
+    virtual IDeserializedResultStore &queryOnceResultStore() const
+    {
+        assertex(onceResultStore!= NULL);
+        return *onceResultStore;
+    }
+
+    virtual IPropertyTree &queryOnceContext(const IQueryFactory *factory, const IRoxieContextLogger &logctx) const
+    {
+        checkOnceDone(factory, logctx);
+        assertex(onceContext != NULL);
+        return *onceContext;
+    }
+
+    virtual void checkOnceDone(const IQueryFactory *factory, const IRoxieContextLogger &logctx) const
+    {
+        CriticalBlock b(onceCrit);
+        if (!onceContext)
+        {
+            onceContext.setown(createPTree());
+            onceResultStore.setown(createDeserializedResultStore());
+            Owned <IRoxieServerContext> ctx = createOnceServerContext(factory, logctx);
+            onceManager.set(&ctx->queryRowManager());
+            try
+            {
+                ctx->process();
+                ctx->done(false);
+            }
+            catch (IException *E)
+            {
+                ctx->done(true);
+                onceException.setown(E);
+            }
+            catch (...)
+            {
+                ctx->done(true);
+                onceException.setown(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception in ONCE code"));
+            }
+        }
+        if (onceException)
+            throw onceException.getLink();
+    }
+
+protected:
+    mutable CriticalSection onceCrit;
+    mutable Owned<roxiemem::IRowManager> onceManager; // release AFTER resultStore
+    mutable Owned<IPropertyTree> onceContext;
+    mutable Owned<IDeserializedResultStore> onceResultStore;
+    mutable Owned<IException> onceException;
+
+};
+
+//----------------------------------------------------------------------------------------------
 // Class CQueryFactory is the main implementation of IQueryFactory, combining a IQueryDll and a
 // package context into an object that can quickly create a the query context that executes a specific
 // instance of a Roxie query. 
@@ -191,6 +258,7 @@ class CQueryFactory : public CInterface, implements IQueryFactory, implements IR
 protected:
     const IRoxiePackage &package;
     Owned<const IQueryDll> dll;
+    Linked<ISharedOnceContext> sharedOnceContext;
     MapStringToActivityArray graphMap;
     StringAttr id;
     StringBuffer errorMessage;
@@ -744,8 +812,8 @@ public:
     IMPLEMENT_IINTERFACE;
     unsigned channelNo;
 
-    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
-        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue)
+    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, ISharedOnceContext *_sharedOnceContext)
+        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue), sharedOnceContext(_sharedOnceContext)
     {
         package.Link();
         isSuspended = false;
@@ -872,6 +940,23 @@ public:
         return hashValue;
     }
 
+    virtual ISharedOnceContext *querySharedOnceContext() const
+    {
+        return sharedOnceContext;
+    }
+
+    virtual IDeserializedResultStore &queryOnceResultStore() const
+    {
+        assertex(sharedOnceContext);
+        return sharedOnceContext->queryOnceResultStore();
+    }
+
+    virtual IPropertyTree &queryOnceContext(const IRoxieContextLogger &logctx) const
+    {
+        assertex(sharedOnceContext);
+        return sharedOnceContext->queryOnceContext(this, logctx);
+    }
+
     virtual const char *loadResource(unsigned id)
     {
         return (const char *) queryDll()->getResource(id);
@@ -1111,14 +1196,6 @@ public:
         throwUnexpected();   // only implemented in derived slave class
     }
 
-    virtual IPropertyTree &queryOnceContext() const
-    {
-        throwUnexpected();   // only implemented in derived server class
-    }
-    virtual IDeserializedResultStore &queryOnceResultStore() const
-    {
-        throwUnexpected();   // only implemented in derived server class
-    }
     virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags) const
     {
         throwUnexpected();   // only implemented in derived server class
@@ -1147,6 +1224,21 @@ public:
     }
 
 protected:
+    IPropertyTree *queryWorkflowTree() const
+    {
+        assertex(dll->queryWorkUnit());
+        return dll->queryWorkUnit()->queryWorkflowTree();
+    }
+
+    bool hasOnceSection() const
+    {
+        IPropertyTree *workflow = queryWorkflowTree();
+        if (workflow)
+            return workflow->hasProp("Item[@mode='once']");
+        else
+            return false;
+    }
+
     void checkSuspended() const
     {
         if (isSuspended)
@@ -1171,35 +1263,14 @@ extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel)
 
 class CRoxieServerQueryFactory : public CQueryFactory
 {
-    // Parts of query factory is only interesting on the server - once support, workflow support, and tracking of total query times
+    // Parts of query factory is only interesting on the server - workflow support, and tracking of total query times
 
 protected:
-    mutable CriticalSection onceCrit;
-    mutable Owned<roxiemem::IRowManager> onceManager; // release AFTER resultStore
-    mutable Owned<IPropertyTree> onceContext;
-    mutable Owned<IDeserializedResultStore> onceResultStore;
-    mutable Owned<IException> onceException;
-
     Owned<IQueryStatsAggregator> queryStats;
 
-    IPropertyTree *queryWorkflowTree() const
-    {
-        assertex(dll->queryWorkUnit());
-        return dll->queryWorkUnit()->queryWorkflowTree();
-    }
-
-    bool hasOnceSection() const
-    {
-        IPropertyTree *workflow = queryWorkflowTree();
-        if (workflow)
-            return workflow->hasProp("Item[@mode='once']");
-        else
-            return false;
-    }
-
 public:
-    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue)
-        : CQueryFactory(_id, _dll, _package, _hashValue, 0)
+    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, ISharedOnceContext *_sharedOnceContext)
+        : CQueryFactory(_id, _dll, _package, _hashValue, 0, _sharedOnceContext)
     {
         queryStats.setown(createQueryStatsAggregator(id.get(), statsExpiryTime));
     }
@@ -1243,61 +1314,15 @@ public:
         return activities;
     }
 
-    virtual IDeserializedResultStore &queryOnceResultStore() const
-    {
-        assertex(onceResultStore!= NULL);
-        return *onceResultStore;
-    }
-
-    virtual IPropertyTree &queryOnceContext() const
-    {
-        assertex(onceContext != NULL);
-        return *onceContext;
-    }
-
-    virtual void checkOnceDone(const IRoxieContextLogger &_logctx) const
-    {
-        if (hasOnceSection())
-        {
-            CriticalBlock b(onceCrit);
-            if (!onceContext)
-            {
-                onceContext.setown(createPTree());
-                onceResultStore.setown(createDeserializedResultStore());
-                Owned <IRoxieServerContext> ctx = createOnceServerContext(this, _logctx);
-                onceManager.set(&ctx->queryRowManager());
-                try
-                {
-                    ctx->process();
-                    ctx->done(false);
-                }
-                catch (IException *E)
-                {
-                    ctx->done(true);
-                    onceException.setown(E);
-                }
-                catch (...)
-                {
-                    ctx->done(true);
-                    onceException.setown(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception in ONCE code"));
-                }
-            }
-            if (onceException)
-                throw onceException.getLink();
-        }
-    }
-
     virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions _xmlReadFlags) const
     {
         checkSuspended();
-        checkOnceDone(_logctx);
         return createRoxieServerContext(context, this, client, isXml, isRaw, isBlocked, httpHelper, trim, priority, _logctx, _xmlReadFlags);
     }
 
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const IRoxieContextLogger &_logctx) const
     {
         checkSuspended();
-        checkOnceDone(_logctx);
         return createWorkUnitServerContext(wu, this, _logctx);
     }
 
@@ -1328,8 +1353,18 @@ extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *
         ::Release(dll);
         return cached;
     }
-    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue);
+    Owned<ISharedOnceContext> sharedOnceContext;
+    assertex(dll->queryWorkUnit());
+    IPropertyTree *workflow = dll->queryWorkUnit()->queryWorkflowTree();
+    if (workflow && workflow->hasProp("Item[@mode='once']"))
+        sharedOnceContext.setown(new CSharedOnceContext);
+    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, sharedOnceContext);
     newFactory->load(stateInfo);
+    if (sharedOnceContext && preloadOnceData)
+    {
+        Owned<StringContextLogger> logctx = new StringContextLogger(id); // NB may get linked by the onceContext
+        sharedOnceContext->checkOnceDone(newFactory, *logctx);
+    }
     return newFactory.getClear();
 }
 
@@ -1517,8 +1552,8 @@ class CSlaveQueryFactory : public CQueryFactory
     }
 
 public:
-    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
-        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo)
+    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, ISharedOnceContext *_sharedOnceContext)
+        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo, _sharedOnceContext)
     {
     }
 
@@ -1578,7 +1613,8 @@ IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, con
         ::Release(dll);
         return cached;
     }
-    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel);
+    Owned<IQueryFactory> serverFactory = createServerQueryFactory(id, LINK(dll), package, stateInfo); // Should always find a cached one
+    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel, serverFactory->querySharedOnceContext());
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }

roxie/ccd/ccdquery.hpp

@@ -71,6 +71,13 @@ interface IActivityGraph : extends IInterface
 interface IRoxiePackage;
 interface IDeserializedResultStore;
 
+interface ISharedOnceContext : extends IInterface
+{
+    virtual IPropertyTree &queryOnceContext(const IQueryFactory *queryFactory, const IRoxieContextLogger &_logctx) const = 0;
+    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
+    virtual void checkOnceDone(const IQueryFactory *queryFactory, const IRoxieContextLogger &_logctx) const = 0;
+};
+
 interface IQueryFactory : extends IInterface
 {
     virtual IRoxieSlaveContext *createSlaveContext(const SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const = 0;
@@ -93,10 +100,11 @@ interface IQueryFactory : extends IInterface
     virtual ILoadedDllEntry *queryDll() const = 0;
     virtual bool getEnableFieldTranslation() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
+    virtual ISharedOnceContext *querySharedOnceContext() const = 0;
+    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
+    virtual IPropertyTree &queryOnceContext(const IRoxieContextLogger &logctx) const = 0;
 
     virtual const IRoxiePackage &queryPackage() const = 0;
-    virtual IPropertyTree &queryOnceContext() const = 0;
-    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
     virtual void getActivityMetrics(StringBuffer &reply) const = 0;
 
     virtual IPropertyTree *cloneQueryXGMML() const = 0;

roxie/ccd/ccdserver.hpp

@@ -155,6 +155,7 @@ interface IRoxieSlaveContext : extends IRoxieContextLogger
     virtual IWorkUnit *updateWorkUnit() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
     virtual IRoxieServerContext *queryServerContext() = 0;
+    virtual IWorkUnitRowReader *getWorkunitRowReader(const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) = 0;
 };
 
 interface IRoxieServerContext : extends IInterface
@@ -164,7 +165,6 @@ interface IRoxieServerContext : extends IInterface
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml) = 0;
     virtual void appendResultDeserialized(const char *name, unsigned sequence, size32_t count, byte **data, bool extend, IOutputMetaData *meta) = 0;
     virtual void appendResultRawContext(const char *name, unsigned sequence, int len, const void * data, int numRows, bool extend, bool saveInContext) = 0;
-    virtual IWorkUnitRowReader *getWorkunitRowReader(const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) = 0;
     virtual roxiemem::IRowManager &queryRowManager() = 0;
 
     virtual void process() = 0;

testing/ecl/roxie/key/remoteonce.xml

@@ -0,0 +1,80 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>5.0</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>5</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>5.0</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>5</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>5.0</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>5</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>5.0</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>5</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>

testing/ecl/roxie/remoteonce.ecl

@@ -0,0 +1,77 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT Std.system.thorlib as thorlib;
+
+//UseStandardFiles
+// NOTE - uses sequential as otherwise we use too many threads (allegedly)
+
+// Try some remote activities reading from normal indexes and local indexes
+onceFilterSet := 'A' : once;
+
+index_best := SORT(LIMIT(DG_FetchIndex1(LName[1]=onceFilterSet),1000000), LName, FName);
+index_best_all := allnodes(index_best);
+s1 := sort(index_best_all, LName, FName);
+count(s1) / thorlib.nodes();  // Every node should return all matching results
+o1 := output(DEDUP(s1,LName, FName, KEEP(1)), {LName, FName});
+
+index_best_all_local := allnodes(LOCAL(index_best));
+s2 := sort(index_best_all_local, LName, FName);
+count(s2); // Every node should return only local matching results
+o2 := output(DEDUP(s2,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with disk files
+
+disk_best := SORT(DG_FetchFile(LName[1]=onceFilterSet), LName, FName);
+disk_best_all := allnodes(disk_best);
+s3 := sort(disk_best_all, LName, FName);
+count(s3) / thorlib.nodes();    // Every node should return all matching results
+o3 := output(DEDUP(s3,LName, FName, KEEP(1)), {LName, FName});
+
+disk_best_all_local := allnodes(LOCAL(disk_best));
+s4 := sort(disk_best_all_local, LName, FName);
+count(s4);  // Every node should return only local matching results
+o4 := output(DEDUP(s4,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with in-memory files
+
+preload_best := sort(DG_FetchFilePreload(LName[1]=onceFilterSet), LName, FName);
+preload_best_all := allnodes(preload_best);
+s5 := sort(preload_best_all, LName, FName);
+count(s5) / thorlib.nodes();  // Every node should return all matching results
+o5 := output(DEDUP(s5,LName, FName, KEEP(1)), {LName, FName});
+
+preload_best_all_local := allnodes(LOCAL(preload_best));
+s6 := sort(preload_best_all_local, LName, FName);
+count(s6);  // Every node should return only local matching results
+o6 := output(DEDUP(s6,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with keyed in-memory files
+
+preload_indexed_best := SORT(DG_FetchFilePreloadIndexed(KEYED(LName[1]=onceFilterSet)), LName, FName);
+preload_indexed_best_all := allnodes(preload_indexed_best);
+s7 := sort(preload_indexed_best_all, LName, FName);
+count(s7) / thorlib.nodes();  // Every node should return all matching results
+o7 := output(DEDUP(s7,LName, FName, KEEP(1)), {LName, FName});
+
+preload_indexed_best_all_local := allnodes(LOCAL(preload_indexed_best));
+s8 := sort(preload_indexed_best_all_local, LName, FName);
+count(s8);   // Every node should return only local matching results
+o8 := output(DEDUP(s8,LName, FName, KEEP(1)), {LName, FName});
+
+
+SEQUENTIAL(o1,o2,o3,o4,o5,o6,o7,o8);