HPCC-8727 ONCE dictionaries should not be serialized to the slaves

Change how and when ONCE sections are handled, so that we can avoid
serializing data from a ONCE section to the slave nodes. This is particularly
useful when a DICTIONARY is createdin a ONCE section (for example by loading
data from a file) but may also apply elsewhere.

Refactored to ensure that the ONCE context is shared between the server and
any slave channels on a node. Also added the option (defaulted on) for ONCE
sections to be evaluated when queries are loaded rather than on first access,
to avoid prolonged 'cache warmup' times when slaves use ONCE data.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlinline.cpp

@@ -1330,6 +1330,8 @@ bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool h
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(0));
     case no_getgraphresult:
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(1));
+    case no_getresult:
+        return !matchesConstantValue(queryPropertyChild(expr, sequenceAtom, 0), ResultSequenceOnce);
     case no_failcode:
     case no_failmessage:
     case no_fail:

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -683,6 +683,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="preloadOnceData" type="xs:boolean" use="optional" default="true">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Evaluate : ONCE sections of queries at query load time</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="remoteFilesExpire" type="xs:integer" use="optional" default="3600000">
       <xs:annotation>
         <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -843,6 +843,7 @@
                 preabortKeyedJoinsThreshold="100"
                 preferredSubnet=""
                 preferredSubnetMask=""
+                preloadOnceData="true"
                 remoteFilesExpire="3600000"
                 resolveFilesInPackage="false"
                 roxieMulticastEnabled="true"

roxie/ccd/ccd.hpp

@@ -399,6 +399,7 @@ extern bool useTreeCopy;
 extern XmlReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves
+extern bool preloadOnceData;
 
 extern unsigned udpMulticastBufferSize;
 extern size32_t diskReadBufferSize;

roxie/ccd/ccdmain.cpp

@@ -112,6 +112,7 @@ bool debugPermitted = true;
 bool checkCompleted = true;
 unsigned preabortKeyedJoinsThreshold = 100;
 unsigned preabortIndexReadsThreshold = 100;
+bool preloadOnceData;
 
 unsigned memoryStatsInterval = 0;
 memsize_t defaultMemoryLimit;
@@ -676,6 +677,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         blindLogging = topology->getPropBool("@blindLogging", false);
         if (!blindLogging)
             logExcessiveSeeks = true;
+        preloadOnceData = topology->getPropBool("@preloadOnceData", true);
         linuxYield = topology->getPropBool("@linuxYield", false);
         traceSmartStepping = topology->getPropBool("@traceSmartStepping", false);
         useMemoryMappedIndexes = topology->getPropBool("@useMemoryMappedIndexes", false);

roxie/ccd/ccdquery.cpp

@@ -179,6 +179,73 @@ extern void addXrefLibraryInfo(IPropertyTree &reply, const char *libraryName)
 }
 
 //----------------------------------------------------------------------------------------------
+// Class CSharedOnceContext manages the context for a query's ONCE code, which is shared between
+// all slave and server contexts on a node
+//----------------------------------------------------------------------------------------------
+
+class CSharedOnceContext : public CInterfaceOf<ISharedOnceContext>
+{
+public:
+    CSharedOnceContext()
+    {
+    }
+
+    ~CSharedOnceContext()
+    {
+    }
+
+    virtual IDeserializedResultStore &queryOnceResultStore() const
+    {
+        assertex(onceResultStore!= NULL);
+        return *onceResultStore;
+    }
+
+    virtual IPropertyTree &queryOnceContext(const IQueryFactory *factory, const IRoxieContextLogger &logctx) const
+    {
+        checkOnceDone(factory, logctx);
+        assertex(onceContext != NULL);
+        return *onceContext;
+    }
+
+    virtual void checkOnceDone(const IQueryFactory *factory, const IRoxieContextLogger &logctx) const
+    {
+        CriticalBlock b(onceCrit);
+        if (!onceContext)
+        {
+            onceContext.setown(createPTree());
+            onceResultStore.setown(createDeserializedResultStore());
+            Owned <IRoxieServerContext> ctx = createOnceServerContext(factory, logctx);
+            onceManager.set(&ctx->queryRowManager());
+            try
+            {
+                ctx->process();
+                ctx->done(false);
+            }
+            catch (IException *E)
+            {
+                ctx->done(true);
+                onceException.setown(E);
+            }
+            catch (...)
+            {
+                ctx->done(true);
+                onceException.setown(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception in ONCE code"));
+            }
+        }
+        if (onceException)
+            throw onceException.getLink();
+    }
+
+protected:
+    mutable CriticalSection onceCrit;
+    mutable Owned<roxiemem::IRowManager> onceManager; // release AFTER resultStore
+    mutable Owned<IPropertyTree> onceContext;
+    mutable Owned<IDeserializedResultStore> onceResultStore;
+    mutable Owned<IException> onceException;
+
+};
+
+//----------------------------------------------------------------------------------------------
 // Class CQueryFactory is the main implementation of IQueryFactory, combining a IQueryDll and a
 // package context into an object that can quickly create a the query context that executes a specific
 // instance of a Roxie query. 
@@ -191,6 +258,7 @@ class CQueryFactory : public CInterface, implements IQueryFactory, implements IR
 protected:
     const IRoxiePackage &package;
     Owned<const IQueryDll> dll;
+    Linked<ISharedOnceContext> sharedOnceContext;
     MapStringToActivityArray graphMap;
     StringAttr id;
     StringBuffer errorMessage;
@@ -744,8 +812,8 @@ public:
     IMPLEMENT_IINTERFACE;
     unsigned channelNo;
 
-    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
-        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue)
+    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, ISharedOnceContext *_sharedOnceContext)
+        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue), sharedOnceContext(_sharedOnceContext)
     {
         package.Link();
         isSuspended = false;
@@ -872,6 +940,23 @@ public:
         return hashValue;
     }
 
+    virtual ISharedOnceContext *querySharedOnceContext() const
+    {
+        return sharedOnceContext;
+    }
+
+    virtual IDeserializedResultStore &queryOnceResultStore() const
+    {
+        assertex(sharedOnceContext);
+        return sharedOnceContext->queryOnceResultStore();
+    }
+
+    virtual IPropertyTree &queryOnceContext(const IRoxieContextLogger &logctx) const
+    {
+        assertex(sharedOnceContext);
+        return sharedOnceContext->queryOnceContext(this, logctx);
+    }
+
     virtual const char *loadResource(unsigned id)
     {
         return (const char *) queryDll()->getResource(id);
@@ -1111,14 +1196,6 @@ public:
         throwUnexpected();   // only implemented in derived slave class
     }
 
-    virtual IPropertyTree &queryOnceContext() const
-    {
-        throwUnexpected();   // only implemented in derived server class
-    }
-    virtual IDeserializedResultStore &queryOnceResultStore() const
-    {
-        throwUnexpected();   // only implemented in derived server class
-    }
     virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags) const
     {
         throwUnexpected();   // only implemented in derived server class
@@ -1147,6 +1224,21 @@ public:
     }
 
 protected:
+    IPropertyTree *queryWorkflowTree() const
+    {
+        assertex(dll->queryWorkUnit());
+        return dll->queryWorkUnit()->queryWorkflowTree();
+    }
+
+    bool hasOnceSection() const
+    {
+        IPropertyTree *workflow = queryWorkflowTree();
+        if (workflow)
+            return workflow->hasProp("Item[@mode='once']");
+        else
+            return false;
+    }
+
     void checkSuspended() const
     {
         if (isSuspended)
@@ -1171,35 +1263,14 @@ extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel)
 
 class CRoxieServerQueryFactory : public CQueryFactory
 {
-    // Parts of query factory is only interesting on the server - once support, workflow support, and tracking of total query times
+    // Parts of query factory is only interesting on the server - workflow support, and tracking of total query times
 
 protected:
-    mutable CriticalSection onceCrit;
-    mutable Owned<roxiemem::IRowManager> onceManager; // release AFTER resultStore
-    mutable Owned<IPropertyTree> onceContext;
-    mutable Owned<IDeserializedResultStore> onceResultStore;
-    mutable Owned<IException> onceException;
-
     Owned<IQueryStatsAggregator> queryStats;
 
-    IPropertyTree *queryWorkflowTree() const
-    {
-        assertex(dll->queryWorkUnit());
-        return dll->queryWorkUnit()->queryWorkflowTree();
-    }
-
-    bool hasOnceSection() const
-    {
-        IPropertyTree *workflow = queryWorkflowTree();
-        if (workflow)
-            return workflow->hasProp("Item[@mode='once']");
-        else
-            return false;
-    }
-
 public:
-    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue)
-        : CQueryFactory(_id, _dll, _package, _hashValue, 0)
+    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, ISharedOnceContext *_sharedOnceContext)
+        : CQueryFactory(_id, _dll, _package, _hashValue, 0, _sharedOnceContext)
     {
         queryStats.setown(createQueryStatsAggregator(id.get(), statsExpiryTime));
     }
@@ -1243,61 +1314,15 @@ public:
         return activities;
     }
 
-    virtual IDeserializedResultStore &queryOnceResultStore() const
-    {
-        assertex(onceResultStore!= NULL);
-        return *onceResultStore;
-    }
-
-    virtual IPropertyTree &queryOnceContext() const
-    {
-        assertex(onceContext != NULL);
-        return *onceContext;
-    }
-
-    virtual void checkOnceDone(const IRoxieContextLogger &_logctx) const
-    {
-        if (hasOnceSection())
-        {
-            CriticalBlock b(onceCrit);
-            if (!onceContext)
-            {
-                onceContext.setown(createPTree());
-                onceResultStore.setown(createDeserializedResultStore());
-                Owned <IRoxieServerContext> ctx = createOnceServerContext(this, _logctx);
-                onceManager.set(&ctx->queryRowManager());
-                try
-                {
-                    ctx->process();
-                    ctx->done(false);
-                }
-                catch (IException *E)
-                {
-                    ctx->done(true);
-                    onceException.setown(E);
-                }
-                catch (...)
-                {
-                    ctx->done(true);
-                    onceException.setown(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception in ONCE code"));
-                }
-            }
-            if (onceException)
-                throw onceException.getLink();
-        }
-    }
-
     virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions _xmlReadFlags) const
     {
         checkSuspended();
-        checkOnceDone(_logctx);
         return createRoxieServerContext(context, this, client, isXml, isRaw, isBlocked, httpHelper, trim, priority, _logctx, _xmlReadFlags);
     }
 
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const IRoxieContextLogger &_logctx) const
     {
         checkSuspended();
-        checkOnceDone(_logctx);
         return createWorkUnitServerContext(wu, this, _logctx);
     }
 
@@ -1328,8 +1353,18 @@ extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *
         ::Release(dll);
         return cached;
     }
-    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue);
+    Owned<ISharedOnceContext> sharedOnceContext;
+    assertex(dll->queryWorkUnit());
+    IPropertyTree *workflow = dll->queryWorkUnit()->queryWorkflowTree();
+    if (workflow && workflow->hasProp("Item[@mode='once']"))
+        sharedOnceContext.setown(new CSharedOnceContext);
+    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, sharedOnceContext);
     newFactory->load(stateInfo);
+    if (sharedOnceContext && preloadOnceData)
+    {
+        Owned<StringContextLogger> logctx = new StringContextLogger(id); // NB may get linked by the onceContext
+        sharedOnceContext->checkOnceDone(newFactory, *logctx);
+    }
     return newFactory.getClear();
 }
 
@@ -1517,8 +1552,8 @@ class CSlaveQueryFactory : public CQueryFactory
     }
 
 public:
-    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
-        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo)
+    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, ISharedOnceContext *_sharedOnceContext)
+        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo, _sharedOnceContext)
     {
     }
 
@@ -1578,7 +1613,8 @@ IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, con
         ::Release(dll);
         return cached;
     }
-    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel);
+    Owned<IQueryFactory> serverFactory = createServerQueryFactory(id, LINK(dll), package, stateInfo); // Should always find a cached one
+    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, dynamic_cast<const IRoxiePackage&>(package), hashValue, channel, serverFactory->querySharedOnceContext());
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }

roxie/ccd/ccdquery.hpp

@@ -71,6 +71,13 @@ interface IActivityGraph : extends IInterface
 interface IRoxiePackage;
 interface IDeserializedResultStore;
 
+interface ISharedOnceContext : extends IInterface
+{
+    virtual IPropertyTree &queryOnceContext(const IQueryFactory *queryFactory, const IRoxieContextLogger &_logctx) const = 0;
+    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
+    virtual void checkOnceDone(const IQueryFactory *queryFactory, const IRoxieContextLogger &_logctx) const = 0;
+};
+
 interface IQueryFactory : extends IInterface
 {
     virtual IRoxieSlaveContext *createSlaveContext(const SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const = 0;
@@ -93,10 +100,11 @@ interface IQueryFactory : extends IInterface
     virtual ILoadedDllEntry *queryDll() const = 0;
     virtual bool getEnableFieldTranslation() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
+    virtual ISharedOnceContext *querySharedOnceContext() const = 0;
+    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
+    virtual IPropertyTree &queryOnceContext(const IRoxieContextLogger &logctx) const = 0;
 
     virtual const IRoxiePackage &queryPackage() const = 0;
-    virtual IPropertyTree &queryOnceContext() const = 0;
-    virtual IDeserializedResultStore &queryOnceResultStore() const = 0;
     virtual void getActivityMetrics(StringBuffer &reply) const = 0;
 
     virtual IPropertyTree *cloneQueryXGMML() const = 0;

roxie/ccd/ccdserver.hpp

@@ -155,6 +155,7 @@ interface IRoxieSlaveContext : extends IRoxieContextLogger
     virtual IWorkUnit *updateWorkUnit() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
     virtual IRoxieServerContext *queryServerContext() = 0;
+    virtual IWorkUnitRowReader *getWorkunitRowReader(const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) = 0;
 };
 
 interface IRoxieServerContext : extends IInterface
@@ -164,7 +165,6 @@ interface IRoxieServerContext : extends IInterface
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml) = 0;
     virtual void appendResultDeserialized(const char *name, unsigned sequence, size32_t count, byte **data, bool extend, IOutputMetaData *meta) = 0;
     virtual void appendResultRawContext(const char *name, unsigned sequence, int len, const void * data, int numRows, bool extend, bool saveInContext) = 0;
-    virtual IWorkUnitRowReader *getWorkunitRowReader(const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) = 0;
     virtual roxiemem::IRowManager &queryRowManager() = 0;
 
     virtual void process() = 0;

testing/ecl/roxie/key/remoteonce.xml

@@ -0,0 +1,80 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>5.0</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>5</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>5.0</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>5</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>5.0</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>5</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>5.0</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>5</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><lname>Anderson                 </lname><fname>Jane           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Joe            </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>John           </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Larry          </fname></Row>
+ <Row><lname>Anderson                 </lname><fname>Sue            </fname></Row>
+</Dataset>

testing/ecl/roxie/remoteonce.ecl

@@ -0,0 +1,77 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT Std.system.thorlib as thorlib;
+
+//UseStandardFiles
+// NOTE - uses sequential as otherwise we use too many threads (allegedly)
+
+// Try some remote activities reading from normal indexes and local indexes
+onceFilterSet := 'A' : once;
+
+index_best := SORT(LIMIT(DG_FetchIndex1(LName[1]=onceFilterSet),1000000), LName, FName);
+index_best_all := allnodes(index_best);
+s1 := sort(index_best_all, LName, FName);
+count(s1) / thorlib.nodes();  // Every node should return all matching results
+o1 := output(DEDUP(s1,LName, FName, KEEP(1)), {LName, FName});
+
+index_best_all_local := allnodes(LOCAL(index_best));
+s2 := sort(index_best_all_local, LName, FName);
+count(s2); // Every node should return only local matching results
+o2 := output(DEDUP(s2,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with disk files
+
+disk_best := SORT(DG_FetchFile(LName[1]=onceFilterSet), LName, FName);
+disk_best_all := allnodes(disk_best);
+s3 := sort(disk_best_all, LName, FName);
+count(s3) / thorlib.nodes();    // Every node should return all matching results
+o3 := output(DEDUP(s3,LName, FName, KEEP(1)), {LName, FName});
+
+disk_best_all_local := allnodes(LOCAL(disk_best));
+s4 := sort(disk_best_all_local, LName, FName);
+count(s4);  // Every node should return only local matching results
+o4 := output(DEDUP(s4,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with in-memory files
+
+preload_best := sort(DG_FetchFilePreload(LName[1]=onceFilterSet), LName, FName);
+preload_best_all := allnodes(preload_best);
+s5 := sort(preload_best_all, LName, FName);
+count(s5) / thorlib.nodes();  // Every node should return all matching results
+o5 := output(DEDUP(s5,LName, FName, KEEP(1)), {LName, FName});
+
+preload_best_all_local := allnodes(LOCAL(preload_best));
+s6 := sort(preload_best_all_local, LName, FName);
+count(s6);  // Every node should return only local matching results
+o6 := output(DEDUP(s6,LName, FName, KEEP(1)), {LName, FName});
+
+// Now try with keyed in-memory files
+
+preload_indexed_best := SORT(DG_FetchFilePreloadIndexed(KEYED(LName[1]=onceFilterSet)), LName, FName);
+preload_indexed_best_all := allnodes(preload_indexed_best);
+s7 := sort(preload_indexed_best_all, LName, FName);
+count(s7) / thorlib.nodes();  // Every node should return all matching results
+o7 := output(DEDUP(s7,LName, FName, KEEP(1)), {LName, FName});
+
+preload_indexed_best_all_local := allnodes(LOCAL(preload_indexed_best));
+s8 := sort(preload_indexed_best_all_local, LName, FName);
+count(s8);   // Every node should return only local matching results
+o8 := output(DEDUP(s8,LName, FName, KEEP(1)), {LName, FName});
+
+
+SEQUENTIAL(o1,o2,o3,o4,o5,o6,o7,o8);