Merge branch 'candidate-8.0.x' into candidate-8.2.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/thorplugin.cpp

@@ -681,11 +681,20 @@ extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuf
 {
     size32_t len = 0;
     const void * data = NULL;
-    if (!dll->getResource(len, data, "WORKUNIT", 1000))
+    if (!dll->getResource(len, data, "WORKUNIT", 1000, false))
         return false;
     return decompressResource(len, data, xml);
 }
 
+extern DLLSERVER_API bool getEmbeddedWorkUnitBinary(ILoadedDllEntry *dll, MemoryBuffer &result)
+{
+    size32_t len = 0;
+    const void * data = NULL;
+    if (!dll->getResource(len, data, "BINWORKUNIT", 1000, false))
+        return false;
+    return decompressResource(len, data, result);
+}
+
 extern DLLSERVER_API bool getEmbeddedManifestXML(const ILoadedDllEntry *dll, StringBuffer &xml)
 {
     size32_t len = 0;
@@ -710,11 +719,12 @@ extern DLLSERVER_API IPropertyTree *getEmbeddedManifestPTree(const ILoadedDllEnt
     return getEmbeddedManifestXML(dll, xml) ? createPTreeFromXMLString(xml.str()) : createPTree();
 }
 
-extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll)
+extern DLLSERVER_API bool containsEmbeddedWorkUnit(ILoadedDllEntry *dll)
 {
     size32_t len = 0;
     const void * data = NULL;
-    return dll->getResource(len, data, "WORKUNIT", 1000, false);
+    return dll->getResource(len, data, "BINWORKUNIT", 1000, false) ||
+           dll->getResource(len, data, "WORKUNIT", 1000, false);
 }
 
 extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml)
@@ -740,6 +750,15 @@ extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuf
     return getResourceXMLFromFile(filename, "MANIFEST", 1000, xml);
 }
 
+extern DLLSERVER_API bool getWorkunitBinaryFromFile(const char *filename, MemoryBuffer &result)
+{
+    MemoryBuffer data;
+    if (!getResourceFromFile(filename, data, "BINWORKUNIT", 1000))
+        return false;
+    return decompressResource(data.length(), data.toByteArray(), result);
+}
+
+
 //-------------------------------------------------------------------------------------------------------------------
 
 extern DLLSERVER_API void getAdditionalPluginsPath(StringBuffer &pluginsPath, const char *_base)

common/dllserver/thorplugin.hpp

@@ -38,15 +38,18 @@ interface ILoadedDllEntry : extends IInterface
 extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile, bool resourcesOnly);
 extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *name);
 extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml);
+extern DLLSERVER_API bool getEmbeddedWorkUnitBinary(ILoadedDllEntry *dll, MemoryBuffer &result);
 extern DLLSERVER_API bool getEmbeddedManifestXML(const ILoadedDllEntry *dll, StringBuffer &xml);
 extern DLLSERVER_API bool getEmbeddedArchiveXML(ILoadedDllEntry *dll, StringBuffer &xml);
 
+
 extern DLLSERVER_API IPropertyTree *getEmbeddedManifestPTree(const ILoadedDllEntry *dll);
 
-extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll);
+extern DLLSERVER_API bool containsEmbeddedWorkUnit(ILoadedDllEntry *dll);
 extern DLLSERVER_API bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id);
 extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml);
 extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuffer &xml);
+extern DLLSERVER_API bool getWorkunitBinaryFromFile(const char *filename, MemoryBuffer &result);
 extern DLLSERVER_API bool getArchiveXMLFromFile(const char *filename, StringBuffer &xml);
 extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuffer &xml);
 

common/workunit/workunit.cpp

@@ -11678,20 +11678,67 @@ unsigned __int64 CLocalWUStatistic::getTimestamp() const
 
 //==========================================================================================
 
-extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit()
 {
     Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
-    if (xml)
-        cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
-    else
+    Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
+    p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
+    cw->loadPTree(p.getClear());
+
+    return QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
+}
+
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnitFromXml(const char *xml)
+{
+    if (!xml)
+        return createLocalWorkUnit();
+
+    Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
+    cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
+    return QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
+}
+
+static ILocalWorkUnit * createLocalWorkUnitFromBinary(MemoryBuffer & serialized)
+{
+    byte version;
+    serialized.read(version);
+
+    Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
+    switch (version)
     {
-        Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
-        p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
-        cw->loadPTree(p.getClear());
+    case 1:
+        cw->loadPTree(createPTree(serialized, ipt_lowmem));
+        break;
+    default:
+        throwUnexpectedX("Unsupported binary workunit format");
     }
+    return QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
+}
 
-    ILocalWorkUnit* ret = QUERYINTERFACE(&cw->lockRemote(false), ILocalWorkUnit);
-    return ret;
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(ILoadedDllEntry * dll)
+{
+    MemoryBuffer serialized;
+    if (getEmbeddedWorkUnitBinary(dll, serialized))
+        return createLocalWorkUnitFromBinary(serialized);
+
+    StringBuffer dllXML;
+    if (!getEmbeddedWorkUnitXML(dll, dllXML))
+        return nullptr;
+
+    return createLocalWorkUnitFromXml(dllXML.str());
+}
+
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnitFromFile(const char * filename)
+{
+    MemoryBuffer serialized;
+    if (getWorkunitBinaryFromFile(filename, serialized))
+        return createLocalWorkUnitFromBinary(serialized);
+
+    StringBuffer dllXML;
+    if (!getWorkunitXMLFromFile(filename, dllXML))
+        return nullptr;
+
+    return createLocalWorkUnitFromXml(dllXML.str());
 }
 
 void exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, IIOStream &out, unsigned extraXmlFlags)
@@ -11791,6 +11838,18 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
         return str.append("Unrecognized workunit format");
 }
 
+extern WORKUNIT_API void exportWorkUnitToBinary(const IConstWorkUnit *wu, MemoryBuffer & serialized)
+{
+    // MORE - queryPTree isn't really safe without holding CLocalWorkUnit::crit - really need to move these functions into CLocalWorkunit
+    const IExtendedWUInterface *ewu = queryExtendedWU(wu);
+    if (!ewu)
+        throwUnexpectedX("Unrecognized workunit format");
+
+    byte version = 1;
+    serialized.append(version);
+    ewu->queryPTree()->serialize(serialized);
+}
+
 extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool regressionTest)
 {
     const IExtendedWUInterface *ewu = queryExtendedWU(wu);

common/workunit/workunit.hpp

@@ -1579,6 +1579,8 @@ protected:
     const char * defaultWho;
 };
 
+interface ILoadedDllEntry;
+
 typedef IWorkUnitFactory * (* WorkUnitFactoryFactory)(const IPropertyTree *);
 
 extern WORKUNIT_API IStatisticGatherer * createGlobalStatisticGatherer(IWorkUnit * wu);
@@ -1596,9 +1598,13 @@ extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, ErrorSeverity se
 extern WORKUNIT_API void setWorkUnitFactory(IWorkUnitFactory *_factory);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
-extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit(const char *XML);
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit();
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnitFromXml(const char *XML);
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(ILoadedDllEntry * dll);
+extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnitFromFile(const char * filename);
 extern WORKUNIT_API IConstWorkUnitInfo *createConstWorkUnitInfo(IPropertyTree &p);
 extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API void exportWorkUnitToBinary(const IConstWorkUnit *wu, MemoryBuffer & serialized);
 extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool splitStats);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);

common/wuwebview/wuwebview.cpp

@@ -932,12 +932,11 @@ bool WuWebView::getEmbeddedArchive(StringBuffer &ret)
         return false;
     if (getEmbeddedArchiveXML(dll, ret))
         return true;
-    // Try the old way, in case it's an older dll
-    StringBuffer dllXML;
-    if (!getEmbeddedWorkUnitXML(dll, dllXML))
+
+    Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(dll);
+    if (!embeddedWU)
         return false;
 
-    Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(dllXML.str());
     Owned<IConstWUQuery> embeddedQuery = embeddedWU->getQuery();
     if (!embeddedQuery)
         return false;

ecl/eclagent/eclagent.cpp

@@ -3468,7 +3468,7 @@ hthor:
 )!!";
 
 
-extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer * wuXML, bool standAloneExe)
+extern int HTHOR_API eclagent_main(int argc, const char *argv[], Owned<ILocalWorkUnit> & standAloneWorkUnit, bool standAloneExe)
 {
 #ifdef _DEBUG
 #ifdef _WIN32
@@ -3602,14 +3602,6 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
 #ifdef MONITOR_ECLAGENT_STATUS
         std::unique_ptr<CSDSServerStatus> serverstatus;
 #endif
-        Owned<ILocalWorkUnit> standAloneWorkUnit;
-        if (wuXML)
-        {
-            //Create workunit from XML
-            standAloneWorkUnit.setown(createLocalWorkUnit(wuXML->str()));
-            wuXML->kill();  // free up text as soon as possible.
-        }
-
         Owned<IUserDescriptor> standAloneUDesc;
         if (daliServers)
         {
@@ -3836,6 +3828,12 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     return retcode;
 }
 
+extern int HTHOR_API eclagent_main(int argc, const char *argv[])
+{
+    Owned<ILocalWorkUnit> nullWu;
+    return eclagent_main(argc, argv, nullWu, false);
+}
+
 //=======================================================================================
 
 void standalone_usage(const char * exeName)
@@ -3879,11 +3877,11 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     try
     {
         Owned<ILoadedDllEntry> exeEntry = createExeDllEntry(argv[0]);
-        StringBuffer wuXML;
-        if (!getEmbeddedWorkUnitXML(exeEntry, wuXML))
+        Owned<ILocalWorkUnit> localWu = createLocalWorkUnit(exeEntry);
+        if (!localWu)
             throw MakeStringException(0, "Could not locate workunit resource");
 
-        ret = eclagent_main(argc, argv, &wuXML, true);
+        ret = eclagent_main(argc, argv, localWu, true);
     }
     catch (IException *E)
     {

ecl/eclagent/eclagentmain.cpp

@@ -24,7 +24,7 @@
 #include <cppunit/ui/text/TestRunner.h>
 #endif
 
-extern int STARTQUERY_API eclagent_main(int argc, const char *argv[], StringBuffer * embeddedWU, bool standAlone);
+extern int STARTQUERY_API eclagent_main(int argc, const char *argv[]);
 
 int main(int argc, const char *argv[])
 {
@@ -58,7 +58,7 @@ int main(int argc, const char *argv[])
     int ret = 0;
     try
     {
-        ret = eclagent_main(argc, argv, NULL, false);
+        ret = eclagent_main(argc, argv);
     }
     catch (IException *E)
     {

ecl/eclcc/eclcc.cpp

@@ -1739,7 +1739,7 @@ void EclCC::processFile(EclCompileInstance & instance)
     if (optArchive || optGenerateDepend || optSaveQueryArchive)
         instance.archive.setown(createAttributeArchive());
 
-    instance.wu.setown(createLocalWorkUnit(NULL));
+    instance.wu.setown(createLocalWorkUnit());
     //Record the version of the compiler in the workunit, but not when regression testing (to avoid spurious differences)
     if (!optBatchMode)
         instance.wu->setDebugValue("eclcc_compiler_version", LANGUAGE_VERSION, true);
@@ -1992,7 +1992,7 @@ void EclCC::processReference(EclCompileInstance & instance, const char * queryAt
 {
     const char * outputFilename = instance.outputFilename;
 
-    instance.wu.setown(createLocalWorkUnit(NULL));
+    instance.wu.setown(createLocalWorkUnit());
     if (optArchive || optGenerateDepend || optSaveQueryArchive)
         instance.archive.setown(createAttributeArchive());
 

ecl/eclccserver/eclccserver.cpp

@@ -582,11 +582,10 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 StringBuffer realdllfilename;
                 realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
 
-                StringBuffer wuXML;
-                if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
+                Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnitFromFile(realdllfilename);
+                if (!embeddedWU)
                     throw makeStringException(999, "Failed to extract workunit from query dll");
 
-                Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
                 queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
                 workunit->setIsClone(false);
                 const char *jobname = embeddedWU->queryJobName();
@@ -594,6 +593,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                     workunit->setJobName(jobname);
                 if (!workunit->getDebugValueBool("obfuscateOutput", false))
                 {
+                    StringBuffer wuXML;
                     Owned<IWUQuery> query = workunit->updateQuery();
                     if (getArchiveXMLFromFile(realdllfilename, wuXML.clear()))  // MORE - if what was submitted was an archive, this is probably pointless?
                         query->setQueryText(wuXML.str());

ecl/hqlcpp/hqlecl.cpp

@@ -603,9 +603,19 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
 
 void HqlDllGenerator::addWorkUnitAsResource()
 {
-    StringBuffer wuXML;
-    exportWorkUnitToXML(wu, wuXML, false, false, false);
-    code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
+    bool defaultBinaryWorkunit = false;
+    if (wu->getDebugValueInt("saveBinaryWorkunit", defaultBinaryWorkunit))
+    {
+        MemoryBuffer wuBinary;
+        exportWorkUnitToBinary(wu, wuBinary);
+        code->addCompressResource("BINWORKUNIT", wuBinary.length(), wuBinary.bytes(), NULL, 1000);
+    }
+    else
+    {
+        StringBuffer wuXML;
+        exportWorkUnitToXML(wu, wuXML, false, false, false);
+        code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
+    }
 }
 
 void HqlDllGenerator::addArchiveAsResource(StringBuffer &buf)

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -4721,11 +4721,10 @@ void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *fi
     wu->setClusterName(cluster);
     wu->commit();
 
-    StringBuffer dllXML;
-    if (getWorkunitXMLFromFile(dllpath.str(), dllXML))
     {
-        Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(dllXML.str());
-        queryExtendedWU(wu)->copyWorkUnit(embeddedWU, true, true);
+        Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnitFromFile(dllpath.str());
+        if (embeddedWU)
+            queryExtendedWU(wu)->copyWorkUnit(embeddedWU, true, true);
     }
 
     wu.associateDll(dllpath.str(), dllname.str());

roxie/ccd/ccddali.cpp

@@ -644,11 +644,10 @@ public:
             w->resetWorkflow();
         if (source)
         {
-            StringBuffer wuXML;
-            if (getEmbeddedWorkUnitXML(source, wuXML))
+            Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(source);
+            if (embeddedWU)
             {
-                Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(wuXML);
-                queryExtendedWU(w)->copyWorkUnit(localWU, true, true);
+                queryExtendedWU(w)->copyWorkUnit(embeddedWU, true, true);
             }
             else
                 throw MakeStringException(ROXIE_DALI_ERROR, "Failed to locate dll workunit info");

roxie/ccd/ccdlistener.cpp

@@ -1159,12 +1159,9 @@ public:
         if (standalone)
         {
             Owned<ILoadedDllEntry> standAloneDll = createExeDllEntry(wuid.get()+1);
-            StringBuffer wuXML;
-            if (getEmbeddedWorkUnitXML(standAloneDll, wuXML))
-            {
-                wu.setown(createLocalWorkUnit(wuXML));
+            wu.setown(createLocalWorkUnit(standAloneDll));
+            if (wu)
                 dll.setown(createExeQueryDll(wuid.get()+1));
-            }
         }
         else
         {

roxie/ccd/ccdmain.cpp

@@ -742,7 +742,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         else
         {
             Owned<ILoadedDllEntry> dll = createExeDllEntry(argv[0]);
-            if (checkEmbeddedWorkUnitXML(dll))
+            if (containsEmbeddedWorkUnit(dll))
                 standAloneDll.setown(createExeQueryDll(argv[0]));
         }
         if (standAloneDll || wuid)
@@ -1268,8 +1268,12 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         initializeTopology(topoValues, myRoles);
 #endif
         createDelayedReleaser();
+        CCycleTimer loadPackageTimer;
         globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
         globalPackageSetManager->load();
+        if (traceLevel)
+            DBGLOG("Loading all packages took %ums", loadPackageTimer.elapsedMs());
+
         ROQ = createOutputQueueManager(numAgentThreads, encryptInTransit);
         ROQ->setHeadRegionSize(headRegionSize);
         ROQ->start();

roxie/ccd/ccdquery.cpp

@@ -99,13 +99,16 @@ private:
         {
             try
             {
+                CCycleTimer elapsed;
                 dll.setown(isExe ? createExeDllEntry(dllName) : queryRoxieDllServer().loadDll(dllName, DllLocationDirectory));
-                StringBuffer wuXML;
-                if (!selfTestMode && getEmbeddedWorkUnitXML(dll, wuXML))
+                if (!selfTestMode)
                 {
-                    Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(wuXML);
-                    wu.setown(localWU->unlock());
+                    Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(dll);
+                    if (localWU)
+                        wu.setown(localWU->unlock());
                 }
+                if (traceLevel)
+                    DBGLOG("Loading dll %s took %" I64F "uus", dllName.str(), elapsed.elapsedNs()/1000);
             }
             catch (IException *E)
             {

testing/regress/ecl/indexlimit3.ecl

@@ -0,0 +1,96 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version optRemoteRead=false
+//version optRemoteRead=true
+
+#onwarning(4523, ignore);
+
+import $.setup;
+import ^ as root;
+prefix := setup.Files(false, false).QueryFilePrefix;
+optRemoteRead := #IFDEFINED(root.optRemoteRead, false);
+#option('forceRemoteRead', optRemoteRead);
+
+
+crec := RECORD
+ unsigned ckey;
+ string info := '';
+END;
+
+rec := RECORD
+  unsigned key;
+  unsigned key2;
+  unsigned v;
+  DATASET(crec) kkeys;
+  string info := '';
+END;
+
+
+ds := DATASET([
+ {1,2,1,[{991}]},
+ {1,2,6,[{992}]},
+ {1,2,7,[{993}]},
+ {1,2,8,[{994}]},
+ {1,2,9,[{995}]},
+ {4,2,2,[{996}]},
+ {4,2,3,[{997}]},
+ {4,2,4,[{998}]},
+ {4,2,5,[{999}]}
+], rec);
+
+i := INDEX(ds, {key, key2}, {ds}, prefix + 'index');
+
+// NB: filter designed so result is not 1st keyed match.
+fi := i(KEYED(key=1) AND v >= 6 AND v <= 8);
+
+SEQUENTIAL(
+ BUILD(i, FEW, OVERWRITE);
+
+ OUTPUT(fi);
+ OUTPUT(COUNT(fi));
+
+// Hit limits
+// indexread with LIMIT
+ OUTPUT(LIMIT(fi, 2, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**ROW LIMIT HIT IN INDEXREAD**'; SELF := []))));
+ OUTPUT(LIMIT(fi, 4, KEYED, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**KEYED LIMIT HIT IN INDEXREAD**'; SELF := []))));
+
+// indexcount with limits. NB: if exceeded, count will be 1
+ OUTPUT(COUNT(LIMIT(fi, 2, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**ROW LIMIT HIT IN INDEXCOUNT**'; SELF := [])))));
+ OUTPUT(COUNT(LIMIT(fi, 4, KEYED, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**KEYED LIMIT HIT IN INDEXCOUNT**'; SELF := [])))));
+
+// indexnormalize
+ OUTPUT(LIMIT(fi.kkeys(ckey>=992), 2, ONFAIL(TRANSFORM(RECORDOF(crec), SELF.info := '**ROW LIMIT HIT IN INDEXNORMALIZE**'; SELF := []))));
+
+
+// Under limits
+// indexread with LIMIT
+ OUTPUT(LIMIT(fi, 3, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**ROW LIMIT HIT IN INDEXREAD**'; SELF := []))));
+ OUTPUT(LIMIT(fi, 5, KEYED, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**KEYED LIMIT HIT IN INDEXREAD**'; SELF := []))));
+
+// indexcount with limits. NB: if exceeded, count will be 1
+ OUTPUT(COUNT(LIMIT(fi, 3, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**ROW LIMIT HIT IN INDEXCOUNT**'; SELF := [])))));
+ OUTPUT(COUNT(LIMIT(fi, 5, KEYED, ONFAIL(TRANSFORM(RECORDOF(fi), SELF.info := '**KEYED LIMIT HIT IN INDEXCOUNT**'; SELF := [])))));
+ OUTPUT(EXISTS(fi));
+
+// indexread with choosen
+ OUTPUT(CHOOSEN(fi, 1));
+
+// indexnormalize
+ OUTPUT(LIMIT(fi.kkeys(ckey>=992), 3, ONFAIL(TRANSFORM(RECORDOF(crec), SELF.info := '**ROW LIMIT HIT IN INDEXNORMALIZE**'; SELF := []))));
+ OUTPUT(CHOOSEN(fi.kkeys(ckey>=994), 1));
+);

testing/regress/ecl/key/indexlimit3.xml

@@ -0,0 +1,55 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><key>1</key><key2>2</key2><v>6</v><kkeys><Row><ckey>992</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>7</v><kkeys><Row><ckey>993</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>8</v><kkeys><Row><ckey>994</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>3</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><key>0</key><key2>0</key2><v>0</v><kkeys></kkeys><info>**ROW LIMIT HIT IN INDEXREAD**</info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><key>0</key><key2>0</key2><v>0</v><kkeys></kkeys><info>**KEYED LIMIT HIT IN INDEXREAD**</info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>1</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>1</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><ckey>0</ckey><info>**ROW LIMIT HIT IN INDEXNORMALIZE**</info></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><key>1</key><key2>2</key2><v>6</v><kkeys><Row><ckey>992</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>7</v><kkeys><Row><ckey>993</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>8</v><kkeys><Row><ckey>994</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><key>1</key><key2>2</key2><v>6</v><kkeys><Row><ckey>992</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>7</v><kkeys><Row><ckey>993</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><key>1</key><key2>2</key2><v>8</v><kkeys><Row><ckey>994</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>3</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>3</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>true</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><key>1</key><key2>2</key2><v>6</v><kkeys><Row><ckey>992</ckey><info></info></Row></kkeys><info></info><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><ckey>992</ckey><info></info></Row>
+ <Row><ckey>993</ckey><info></info></Row>
+ <Row><ckey>994</ckey><info></info></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><ckey>994</ckey><info></info></Row>
+</Dataset>

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -491,8 +491,11 @@ public:
 
         if ((RCMAX != keyedLimit) && (keyedLimit+1 < remoteLimit))
             remoteLimit = keyedLimit+1; // 1 more to ensure triggered when received back.
-        if ((RCMAX != rowLimit) && (rowLimit+1 < remoteLimit))
-            remoteLimit = rowLimit+1; // 1 more to ensure triggered when received back.
+        if (!mayFilter)
+        {
+            if ((RCMAX != rowLimit) && (rowLimit+1 < remoteLimit))
+                remoteLimit = rowLimit+1; // 1 more to ensure triggered when received back.
+        }
     }
     void calcKeyedLimitCount()
     {
@@ -1208,7 +1211,8 @@ public:
         ActivityTimer s(slaveTimerStats, timeActivities);
 
         // NB: initLimits sets up remoteLimit before base start() call, because if parts are remote PARENT::start() will use remoteLimit
-        initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), helper->hasMatchFilter());
+        // NB2: I'm not sure if hasMatchFilter() can ever actually be generated.
+        initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), helper->hasFilter() || helper->hasMatchFilter());
         calcKeyedLimitCount();
 
         PARENT::start();
@@ -1384,7 +1388,8 @@ public:
         ActivityTimer s(slaveTimerStats, timeActivities);
 
         // NB: initLimits sets up remoteLimit before base start() call, because if parts are remote PARENT::start() will use remoteLimit
-        initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), helper->hasMatchFilter());
+        // NB2: mayFilter=true assumed, because there is no flag to indicate post-filtering
+        initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), true);
         calcKeyedLimitCount();
 
         PARENT::start();

thorlcr/graph/thgraph.cpp

@@ -2727,10 +2727,9 @@ CJobBase::CJobBase(ILoadedDllEntry *_querySo, const char *_graphName) : querySo(
     jobSlaveChannelNum.allocateN(querySlaves()); // filled when channels are added.
     for (unsigned s=0; s<querySlaves(); s++)
         jobSlaveChannelNum[s] = NotFound;
-    StringBuffer wuXML;
-    if (!getEmbeddedWorkUnitXML(querySo, wuXML))
+    Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(querySo);
+    if (!localWU)
         throw MakeStringException(0, "Failed to locate workunit info in query : %s", querySo->queryName());
-    Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(wuXML);
     Owned<IConstWUGraph> graph = localWU->getGraph(graphName);
     graphXGMML.setown(graph->getXGMMLTree(false));
     if (!graphXGMML)

tools/wuget/wuget.cpp

@@ -68,14 +68,19 @@ int main(int argc, char **argv)
                 {
                     const char *filename = argv[i];
                     const char *ext = pathExtension(filename);
-                    StringBuffer xml;
+                    Owned<ILocalWorkUnit> wu;
                     if (strisame(ext, ".xml"))
+                    {
+                        StringBuffer xml;
                         xml.loadFile(filename, false);
+                        wu.setown(createLocalWorkUnitFromXml(xml));
+                    }
                     else
-                        getWorkunitXMLFromFile(filename, xml);
-                    if (xml.length())
+                        wu.setown(createLocalWorkUnitFromFile(filename));
+
+                    if (wu)
                     {
-                        Owned<ILocalWorkUnit> wu = createLocalWorkUnit(xml);
+                        StringBuffer xml;
                         if (doArchive && getArchiveXMLFromFile(filename, xml.clear()))
                         {
                             Owned<IWUQuery> q = wu->updateQuery();

tools/wutool/wutool.cpp

@@ -799,7 +799,7 @@ protected:
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
         Owned<IWorkUnit> createWu = factory->createWorkUnit("WuTest", NULL, NULL, NULL);
         StringBuffer wuid(createWu->queryWuid());
-        Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(
+        Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnitFromXml(
                 // Note - generated by compiling the following ecl:
                 //
                 //   integer one := 1 : stored('one');